From f61c4db9005d703d4c70652e7932d1b9b37767ec Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Wed, 20 Jan 2021 16:43:16 -0800 Subject: [PATCH 01/37] Kernel Environment Provisioning - begin implementation --- jupyter_client/kernelspec.py | 14 +- jupyter_client/manager.py | 124 +++++----------- jupyter_client/provisioning.py | 263 +++++++++++++++++++++++++++++++++ setup.py | 4 + 4 files changed, 312 insertions(+), 93 deletions(-) create mode 100644 jupyter_client/provisioning.py diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 772806f99..a65f1346e 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -19,7 +19,7 @@ from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH - +from .provisioning import EnvironmentProvisionerFactory NATIVE_KERNEL_NAME = 'python3' @@ -187,6 +187,7 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): """ Returns a :class:`KernelSpec` instance for a given kernel_name and resource_dir. """ + kspec = None if kernel_name == NATIVE_KERNEL_NAME: try: from ipykernel.kernelspec import RESOURCES, get_kernel_dict @@ -195,9 +196,16 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): pass else: if resource_dir == RESOURCES: - return self.kernel_spec_class(resource_dir=resource_dir, **get_kernel_dict()) + kspec = self.kernel_spec_class(resource_dir=resource_dir, **get_kernel_dict()) + if not kspec: + kspec = self.kernel_spec_class.from_resource_dir(resource_dir) + + if not EnvironmentProvisionerFactory.instance(parent=self.parent).is_provisioner_available(kspec.to_dict()): + self.log.warning(f"Kernel '{kernel_name}' is referencing an environment " # TODO should probably include + f"provisioner that is not available.") # provisioner name. + raise NoSuchKernel(kernel_name) - return self.kernel_spec_class.from_resource_dir(resource_dir) + return kspec def _find_spec_directory(self, kernel_name): """Find the resource directory of a named kernel spec""" diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 3f60dd099..1b379c23a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -15,19 +15,17 @@ import zmq from ipython_genutils.importstring import import_item -from .localinterfaces import is_local_ip, local_ips from traitlets import ( Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, default, observe, observe_compat ) -from jupyter_client import ( - launch_kernel, - kernelspec, -) +from typing import Optional +from jupyter_client import kernelspec + from .connect import ConnectionFileMixin -from .managerabc import ( - KernelManagerABC -) +from .localinterfaces import is_local_ip, local_ips +from .managerabc import KernelManagerABC +from .provisioning import EnvironmentProvisionerFactory, EnvironmentProvisionerBase class KernelManager(ConnectionFileMixin): @@ -54,9 +52,13 @@ def _client_factory_default(self): def _client_class_changed(self, change): self.client_factory = import_item(str(change['new'])) - # The kernel process with which the KernelManager is communicating. - # generally a Popen instance - kernel = Any() + # The kernel provisioner with which the KernelManager is communicating. + # This will generally be a ClientProvisioner instance unless the specification indicates otherwise. + # Note that we use two attributes, kernel and provisioner, that will point at the same provisioner instance. + # kernel will be non-None during the kernel's lifecycle, while provisioner will span that time, being set + # prior to launch and unset following the kernel's termination. + kernel: Optional[EnvironmentProvisionerBase] = None + provisioner: Optional[EnvironmentProvisionerBase] = None kernel_spec_manager = Instance(kernelspec.KernelSpecManager) @@ -215,11 +217,11 @@ def from_ns(match): return [ pat.sub(from_ns, arg) for arg in cmd ] def _launch_kernel(self, kernel_cmd, **kw): - """actually launch the kernel + """actually launch the kernel via the provisioner override in a subclass to launch kernel subprocesses differently """ - return launch_kernel(kernel_cmd, **kw) + return self.provisioner.launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown @@ -254,6 +256,9 @@ def pre_start_kernel(self, **kw): "Currently valid addresses are: %s" % (self.ip, local_ips()) ) + self.provisioner = EnvironmentProvisionerFactory.instance(parent=self.parent)\ + .create_provisioner_instance(self.kernel_spec.to_dict()) + # write connection file / get default ports self.write_connection_file() @@ -262,6 +267,10 @@ def pre_start_kernel(self, **kw): # build the Popen cmd extra_arguments = kw.pop('extra_arguments', []) kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) + + # Give provisioner a crack at the kernel cmd + kernel_cmd = self.provisioner.format_kernel_cmd(kernel_cmd, extra_arguments=extra_arguments, **kw) + env = kw.pop('env', os.environ).copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. @@ -357,6 +366,10 @@ def cleanup_resources(self, restart=False): if self._created_context and not restart: self.context.destroy(linger=100) + if self.provisioner: + self.provisioner.cleanup() + self.provisioner = self.kernel = None + def cleanup(self, connection_file=True): """Clean up resources when the kernel is shut down""" warnings.warn("Method cleanup(connection_file=True) is deprecated, use cleanup_resources(restart=False).", @@ -464,25 +477,7 @@ def _kill_kernel(self): This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - # Signal the kernel to terminate (sends SIGKILL on Unix and calls - # TerminateProcess() on Win32). - try: - if hasattr(signal, 'SIGKILL'): - self.signal_kernel(signal.SIGKILL) - else: - self.kernel.kill() - except OSError as e: - # In Windows, we will get an Access Denied error if the process - # has already terminated. Ignore it. - if sys.platform == 'win32': - if e.winerror != 5: - raise - # On Unix, we may get an ESRCH error if the process has already - # terminated. Ignore it. - else: - from errno import ESRCH - if e.errno != ESRCH: - raise + self.kernel.kill() # Block until the kernel terminates. self.kernel.wait() @@ -511,21 +506,9 @@ def interrupt_kernel(self): raise RuntimeError("Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): - """Sends a signal to the process group of the kernel (this - usually includes the kernel and any subprocesses spawned by - the kernel). - - Note that since only SIGTERM is supported on Windows, this function is - only useful on Unix systems. + """Sends signal (signum) to the kernel. """ if self.has_kernel: - if hasattr(os, "getpgid") and hasattr(os, "killpg"): - try: - pgid = os.getpgid(self.kernel.pid) - os.killpg(pgid, signum) - return - except OSError: - pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @@ -549,11 +532,11 @@ class AsyncKernelManager(KernelManager): client_factory = Type(klass='jupyter_client.asynchronous.AsyncKernelClient') async def _launch_kernel(self, kernel_cmd, **kw): - """actually launch the kernel + """actually launch the kernel via the provisioner override in a subclass to launch kernel subprocesses differently """ - res = launch_kernel(kernel_cmd, **kw) + res = self.provisioner.launch_kernel(kernel_cmd, **kw) return res async def start_kernel(self, **kw): @@ -679,25 +662,7 @@ async def _kill_kernel(self): This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - # Signal the kernel to terminate (sends SIGKILL on Unix and calls - # TerminateProcess() on Win32). - try: - if hasattr(signal, 'SIGKILL'): - await self.signal_kernel(signal.SIGKILL) - else: - self.kernel.kill() - except OSError as e: - # In Windows, we will get an Access Denied error if the process - # has already terminated. Ignore it. - if sys.platform == 'win32': - if e.winerror != 5: - raise - # On Unix, we may get an ESRCH error if the process has already - # terminated. Ignore it. - else: - from errno import ESRCH - if e.errno != ESRCH: - raise + self.kernel.kill() # Wait until the kernel terminates. try: @@ -721,12 +686,7 @@ async def interrupt_kernel(self): if self.has_kernel: interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == 'signal': - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - await self.signal_kernel(signal.SIGINT) - + await self.signal_kernel(signal.SIGINT) elif interrupt_mode == 'message': msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() @@ -735,24 +695,8 @@ async def interrupt_kernel(self): raise RuntimeError("Cannot interrupt kernel. No kernel is running!") async def signal_kernel(self, signum): - """Sends a signal to the process group of the kernel (this - usually includes the kernel and any subprocesses spawned by - the kernel). - - Note that since only SIGTERM is supported on Windows, this function is - only useful on Unix systems. - """ - if self.has_kernel: - if hasattr(os, "getpgid") and hasattr(os, "killpg"): - try: - pgid = os.getpgid(self.kernel.pid) - os.killpg(pgid, signum) - return - except OSError: - pass - self.kernel.send_signal(signum) - else: - raise RuntimeError("Cannot signal kernel. No kernel is running!") + """Sends signal (signum) to the kernel.""" + super(AsyncKernelManager, self).signal_kernel(signum) async def is_alive(self): """Is the kernel process still running?""" diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py new file mode 100644 index 000000000..2c21510a6 --- /dev/null +++ b/jupyter_client/provisioning.py @@ -0,0 +1,263 @@ +"""Kernel Environment Provisioner Classes""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import os +import signal +import sys + +from entrypoints import get_group_all, EntryPoint +from typing import Optional, Dict, List, Any +from traitlets.config import LoggingConfigurable, SingletonConfigurable + +from .launcher import launch_kernel + +DEFAULT_PROVISIONER = "ClientProvisioner" + + +class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name for base class + """Base class defining methods for EnvironmentProvisioner classes. + + Theses methods model those of the Subprocess Popen class: + https://docs.python.org/3/library/subprocess.html#popen-objects + """ + + def __init__(self, **kwargs): + super(EnvironmentProvisionerBase, self).__init__(**kwargs) + self.provisioner_config = kwargs.get('provisioner_config') + + def poll(self) -> [int, None]: + """Checks if kernel process is still running. + + If running, None is returned, otherwise the process's integer-valued exit code is returned. + """ + raise NotImplementedError() + + def wait(self, timeout: Optional[float] = None) -> [int, None]: + """Waits for kernel process to terminate. As a result, this method should be called with + a value for timeout. + + If the kernel process does not terminate following timeout seconds, a TimeoutException will + be raised - that can be caught and retried. If the kernel process has terminated, its + integer-valued exit code will be returned. + """ + raise NotImplementedError() + + def send_signal(self, signum: int) -> None: + """Sends signal identified by signum to the kernel process.""" + raise NotImplementedError() + + def kill(self) -> None: + """Kills the kernel process. This is typically accomplished via a SIGKILL signal, which + cannot be caught. + """ + raise NotImplementedError() + + def terminate(self) -> None: + """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, which + can be caught, allowing the kernel provisioner to perform possible cleanup of resources. + """ + raise NotImplementedError() + + def cleanup(self) -> None: + """Cleanup any resources allocated on behalf of the kernel provisioner.""" + raise NotImplementedError() + + def format_kernel_cmd(self, cmd: List[str], **kwargs: Dict[str, Any]) -> List[str]: + """Replace any kernel provisioner-specific templated arguments in the launch command.""" + return cmd + + def pre_launch(self, cmd: List[str], **kwargs: Dict[str, Any]) -> None: + """Perform any steps in preparation for kernel process launch.""" + pass + + def launch_kernel(self, cmd: List[str], **kwargs: Dict[str, Any]) -> 'EnvironmentProvisionerBase': + """Launch the kernel process returning the class instance.""" + raise NotImplementedError() + + def post_launch(self) -> None: + """Perform any steps following the kernel process launch.""" + pass + + def get_provisioner_info(self) -> Dict: + """Captures the base information necessary for kernel persistence relative to the provisioner. + + The superclass method must always be called first to ensure proper ordering. Since this is the + most base class, no call to `super()` is necessary. + """ + provisioner_info = {} + return provisioner_info + + def load_provisioner_info(self, provisioner_info: Dict) -> None: + """Loads the base information necessary for kernel persistence relative to the provisioner. + + The superclass method must always be called first to ensure proper ordering. Since this is the + most base class, no call to `super()` is necessary. + """ + pass + + +class ClientProvisioner(EnvironmentProvisionerBase): # TODO - determine name for default class + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.popen = None + self.pid = None + self.pgid = None + self.ip = None # TODO - assume local_ip? + + def launch_kernel(self, cmd, **kwargs): + self.popen = launch_kernel(cmd, **kwargs) + pgid = None + if hasattr(os, "getpgid"): + try: + pgid = os.getpgid(self.popen.pid) + except OSError: + pass + self.pid = self.popen.pid + self.pgid = pgid + return self + + def poll(self) -> [int, None]: + if self.popen: + return self.popen.poll() + + def wait(self, timeout: Optional[float] = None) -> [int, None]: + if self.popen: + return self.popen.wait(timeout=timeout) + + def send_signal(self, signum: int) -> None: + """Sends a signal to the process group of the kernel (this + usually includes the kernel and any subprocesses spawned by + the kernel). + + Note that since only SIGTERM is supported on Windows, we will + check if the desired signal is for interrupt and apply the + applicable code on Windows in that case. + """ + if self.popen: + if signum == signal.SIGINT and sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.popen.win32_interrupt_event) + return + + if self.pgid and hasattr(os, "killpg"): + try: + os.killpg(self.pgid, signum) + return + except OSError: + pass + return self.popen.send_signal(signum) + + def kill(self) -> None: + if self.popen: + try: + self.popen.kill() + except OSError as e: + # In Windows, we will get an Access Denied error if the process + # has already terminated. Ignore it. + if sys.platform == 'win32': + if e.winerror != 5: + raise + # On Unix, we may get an ESRCH error if the process has already + # terminated. Ignore it. + else: + from errno import ESRCH + if e.errno != ESRCH: + raise + + def terminate(self) -> None: + if self.popen: + return self.popen.terminate() + + def cleanup(self) -> None: + pass + + def get_provisioner_info(self) -> Dict: + """Captures the base information necessary for kernel persistence relative to the provisioner. + """ + provisioner_info = super(ClientProvisioner, self).get_provisioner_info() + provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) + return provisioner_info + + def load_provisioner_info(self, provisioner_info: Dict) -> None: + """Loads the base information necessary for kernel persistence relative to the provisioner. + """ + super(ClientProvisioner, self).load_provisioner_info(provisioner_info) + self.pid = provisioner_info['pid'] + self.pgid = provisioner_info['pgid'] + self.ip = provisioner_info['ip'] + + +class EnvironmentProvisionerFactory(SingletonConfigurable): + """EnvironmentProvisionerFactory is responsible for validating and initializing provisioner instances. + """ + + provisioners: Dict[str, EntryPoint] = {} + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + for ep in get_group_all('jupyter_client.environment_provisioners'): + self.provisioners[ep.name] = ep + + def is_provisioner_available(self, kernel_spec: Dict[str, Any]) -> bool: + """ + Reads the associated kernel_spec to determine the provisioner and returns whether it + exists as an entry_point (True) or not (False). + """ + provisioner_cfg = EnvironmentProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_name = provisioner_cfg.get('provisioner_name') + return provisioner_name in self.provisioners + + def create_provisioner_instance(self, kernel_spec: Dict[str, Any]) -> EnvironmentProvisionerBase: + """ + Reads the associated kernel_spec and to see if has an environment_provisioner stanza. + If one exists, it instantiates an instance. If an environment provisioner is not + specified in the kernelspec, a DEFAULT_PROVISIONER stanza is fabricated and instantiated. + The instantiated instance is returned. + + If the provisioner is found to not exist (not registered via entry_points), + ModuleNotFoundError is raised. + """ + provisioner_cfg = EnvironmentProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_name = provisioner_cfg.get('provisioner_name') + if provisioner_name not in self.provisioners: + raise ModuleNotFoundError(f"Environment provisioner '{provisioner_name}' has not been registered.") + + self.log.debug("Instantiating kernel '{}' with environment provisioner: {}". + format(kernel_spec.get('display_name'), provisioner_name)) + provisioner_class = self.provisioners[provisioner_name].load() + provisioner_config = provisioner_cfg.get('config') + return provisioner_class(parent=self.parent, **provisioner_config) + + @staticmethod + def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: + """ + Return the environment_provisioner stanza from the kernel_spec. + + Checks the kernel_spec's metadata dictionary for an environment_provisioner entry. + If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER and returned. + + Parameters + ---------- + kernel_spec : Dict + The kernel specification object from which the provisioner dictionary is derived. + + Returns + ------- + dict + The provisioner portion of the kernel_spec. If one does not exist, it will contain the default + information. If no `config` sub-dictionary exists, an empty `config` dictionary will be added. + + """ + if kernel_spec: + metadata = kernel_spec.get('metadata', {}) + if 'environment_provisioner' in metadata: + env_provisioner = metadata.get('environment_provisioner') + if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default + if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one + env_provisioner.update({"config": {}}) + return env_provisioner # Return what we found (plus config stanza if necessary) + return {"provisioner_name": DEFAULT_PROVISIONER, "config": {}} diff --git a/setup.py b/setup.py index f7ffc8dbb..d2f8072a9 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ def run(self): ], install_requires = [ 'traitlets', + 'entrypoints', 'jupyter_core>=4.6.0', 'pyzmq>=13', 'python-dateutil>=2.1', @@ -89,6 +90,9 @@ def run(self): 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', 'jupyter-kernel = jupyter_client.kernelapp:main', ], + 'jupyter_client.environment_provisioners': [ + 'ClientProvisioner = jupyter_client.provisioning:ClientProvisioner', + ] }, ) From bb13c98e51bf9a58ae42e09d0e768ac12835a25d Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sat, 23 Jan 2021 10:53:27 -0800 Subject: [PATCH 02/37] Add initial tests --- jupyter_client/provisioning.py | 8 +- jupyter_client/tests/conftest.py | 25 ++++ jupyter_client/tests/test_kernelmanager.py | 10 +- jupyter_client/tests/test_provisioning.py | 140 +++++++++++++++++++++ 4 files changed, 168 insertions(+), 15 deletions(-) create mode 100644 jupyter_client/tests/conftest.py create mode 100644 jupyter_client/tests/test_provisioning.py diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 2c21510a6..8afcf9f5e 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -9,7 +9,7 @@ from entrypoints import get_group_all, EntryPoint from typing import Optional, Dict, List, Any -from traitlets.config import LoggingConfigurable, SingletonConfigurable +from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable from .launcher import launch_kernel @@ -23,10 +23,6 @@ class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name https://docs.python.org/3/library/subprocess.html#popen-objects """ - def __init__(self, **kwargs): - super(EnvironmentProvisionerBase, self).__init__(**kwargs) - self.provisioner_config = kwargs.get('provisioner_config') - def poll(self) -> [int, None]: """Checks if kernel process is still running. @@ -230,7 +226,7 @@ def create_provisioner_instance(self, kernel_spec: Dict[str, Any]) -> Environmen format(kernel_spec.get('display_name'), provisioner_name)) provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class(parent=self.parent, **provisioner_config) + return provisioner_class(parent=self.parent, config=Config(provisioner_config)) @staticmethod def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py new file mode 100644 index 000000000..1a5d4e24d --- /dev/null +++ b/jupyter_client/tests/conftest.py @@ -0,0 +1,25 @@ +"""Pytest fixtures and configuration""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import os +import pytest + +from jupyter_core import paths +from .utils import test_env + +pjoin = os.path.join + + +@pytest.fixture(autouse=True) +def env(): + env_patch = test_env() + env_patch.start() + yield + env_patch.stop() + + +@pytest.fixture() +def kernel_dir(): + return pjoin(paths.jupyter_data_dir(), 'kernels') \ No newline at end of file diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index cc3f78feb..07903b58a 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -21,21 +21,13 @@ from subprocess import PIPE from ..manager import start_new_kernel, start_new_async_kernel -from .utils import test_env, skip_win32, AsyncKernelManagerSubclass, AsyncKernelManagerWithCleanup +from .utils import skip_win32, AsyncKernelManagerSubclass, AsyncKernelManagerWithCleanup pjoin = os.path.join TIMEOUT = 30 -@pytest.fixture(autouse=True) -def env(): - env_patch = test_env() - env_patch.start() - yield - env_patch.stop() - - @pytest.fixture(params=['tcp', 'ipc']) def transport(request): if sys.platform == 'win32' and request.param == 'ipc': # diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py new file mode 100644 index 000000000..21a31e4b9 --- /dev/null +++ b/jupyter_client/tests/test_provisioning.py @@ -0,0 +1,140 @@ +"""Test Provisionering""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import json +import os +import pytest +import sys + +from jupyter_core import paths +from ..kernelspec import KernelSpecManager, NoSuchKernel +from ..manager import KernelManager, AsyncKernelManager +from ..provisioning import ClientProvisioner + +pjoin = os.path.join + + +@pytest.fixture +def no_provisioner(): + kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'no_provisioner') + os.makedirs(kernel_dir) + with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: + f.write(json.dumps({ + 'argv': [sys.executable, + '-m', 'jupyter_client.tests.signalkernel', + '-f', '{connection_file}'], + 'display_name': "Signal Test Kernel Default", + 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'} + })) + + +@pytest.fixture +def default_provisioner(): + kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'default_provisioner') + os.makedirs(kernel_dir) + with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: + f.write(json.dumps({ + 'argv': [sys.executable, + '-m', 'jupyter_client.tests.signalkernel', + '-f', '{connection_file}'], + 'display_name': "Signal Test Kernel w Provisioner", + 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, + 'metadata': { + 'environment_provisioner': { + 'provisioner_name': 'ClientProvisioner', + 'config': {'config_var_1': 42, 'config_var_2': 'foo'} + } + } + })) + + +@pytest.fixture +def missing_provisioner(): + kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'missing_provisioner') + os.makedirs(kernel_dir) + with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: + f.write(json.dumps({ + 'argv': [sys.executable, + '-m', 'jupyter_client.tests.signalkernel', + '-f', '{connection_file}'], + 'display_name': "Signal Test Kernel Missing Provisioner", + 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, + 'metadata': { + 'environment_provisioner': { + 'provisioner_name': 'MissingProvisioner', + 'config': {'config_var_1': 42, 'config_var_2': 'foo'} + } + } + })) + + +@pytest.fixture +def ksm(): + return KernelSpecManager() + + +@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner']) +def km(request, no_provisioner, missing_provisioner, default_provisioner): + return KernelManager(kernel_name=request.param) + + +@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner']) +def akm(request, no_provisioner, missing_provisioner, default_provisioner): + return AsyncKernelManager(kernel_name=request.param) + + +class TestDiscovery: + def test_find_all_specs(self, no_provisioner, missing_provisioner, default_provisioner, ksm): + kernels = ksm.get_all_specs() + + # Ensure specs for no_provisioner and default_provisioner exist and missing_provisioner doesn't + assert 'no_provisioner' in kernels + assert 'default_provisioner' in kernels + assert 'missing_provisioner' not in kernels + + def test_get_missing(self, missing_provisioner, ksm): + with pytest.raises(NoSuchKernel): + ksm.get_kernel_spec('missing_provisioner') + + +class TestRuntime: + def test_lifecycle(self, km): + assert km.provisioner is None + if km.kernel_name == 'missing_provisioner': + with pytest.raises(NoSuchKernel): + km.start_kernel() + else: + km.start_kernel() + assert isinstance(km.provisioner, ClientProvisioner) + assert km.kernel is km.provisioner + if km.kernel_name == 'default_provisioner': + assert km.provisioner.config.get('config_var_1') == 42 + assert km.provisioner.config.get('config_var_2') == 'foo' + else: + assert 'config_var_1' not in km.provisioner.config + assert 'config_var_2' not in km.provisioner.config + km.shutdown_kernel() + assert km.kernel is None + assert km.provisioner is None + + @pytest.mark.asyncio + async def test_async_lifecycle(self, akm): + assert akm.provisioner is None + if akm.kernel_name == 'missing_provisioner': + with pytest.raises(NoSuchKernel): + await akm.start_kernel() + else: + await akm.start_kernel() + assert isinstance(akm.provisioner, ClientProvisioner) + assert akm.kernel is akm.provisioner + if akm.kernel_name == 'default_provisioner': + assert akm.provisioner.config.get('config_var_1') == 42 + assert akm.provisioner.config.get('config_var_2') == 'foo' + else: + assert 'config_var_1' not in akm.provisioner.config + assert 'config_var_2' not in akm.provisioner.config + await akm.shutdown_kernel() + assert akm.kernel is None + assert akm.provisioner is None From c9e5f0f1c0063fe96c0690103d5cc85a77fbbc18 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Mon, 25 Jan 2021 12:14:01 -0800 Subject: [PATCH 03/37] Minor refactoring, cleanup pre-start --- jupyter_client/kernelspec.py | 15 ++-- jupyter_client/manager.py | 47 +++------- jupyter_client/multikernelmanager.py | 1 + jupyter_client/provisioning.py | 130 ++++++++++++++++++++------- 4 files changed, 120 insertions(+), 73 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index a65f1346e..6e54e3e75 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -3,7 +3,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import errno import io import json import os @@ -11,15 +10,15 @@ import shutil import warnings -pjoin = os.path.join - from traitlets import ( HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum ) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH -from .provisioning import EnvironmentProvisionerFactory +from .provisioning import EnvironmentProvisionerFactory as EPF + +pjoin = os.path.join NATIVE_KERNEL_NAME = 'python3' @@ -67,6 +66,7 @@ def to_json(self): _kernel_name_pat = re.compile(r'^[a-z0-9._\-]+$', re.IGNORECASE) + def _is_valid_kernel_name(name): """Check that a kernel name is valid.""" # quote is not unicode-safe on Python 2 @@ -179,9 +179,8 @@ def find_kernel_specs(self): if self.whitelist: # filter if there's a whitelist - d = {name:spec for name,spec in d.items() if name in self.whitelist} + d = {name: spec for name, spec in d.items() if name in self.whitelist} return d - # TODO: Caching? def _get_kernel_spec_by_name(self, kernel_name, resource_dir): """ Returns a :class:`KernelSpec` instance for a given kernel_name @@ -200,9 +199,7 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): if not kspec: kspec = self.kernel_spec_class.from_resource_dir(resource_dir) - if not EnvironmentProvisionerFactory.instance(parent=self.parent).is_provisioner_available(kspec.to_dict()): - self.log.warning(f"Kernel '{kernel_name}' is referencing an environment " # TODO should probably include - f"provisioner that is not available.") # provisioner name. + if not EPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec.to_dict()): raise NoSuchKernel(kernel_name) return kspec diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 1b379c23a..59271b188 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -10,6 +10,7 @@ import signal import sys import time +import uuid import warnings import zmq @@ -25,7 +26,7 @@ from .connect import ConnectionFileMixin from .localinterfaces import is_local_ip, local_ips from .managerabc import KernelManagerABC -from .provisioning import EnvironmentProvisionerFactory, EnvironmentProvisionerBase +from .provisioning import EnvironmentProvisionerFactory as EPF, EnvironmentProvisionerBase class KernelManager(ConnectionFileMixin): @@ -91,6 +92,7 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec + # TODO - This has been deprecated since April 2014 (4.0 release) - time to remove? kernel_cmd = List(Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. @@ -248,6 +250,9 @@ def pre_start_kernel(self, **kw): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ + # TODO - clean this method up. Most should move into the provisioner base class, although + # attributes (e.g., transport, ip, _launch_args) need to remain here for b/c. + if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " "This one is not: %s." @@ -256,49 +261,23 @@ def pre_start_kernel(self, **kw): "Currently valid addresses are: %s" % (self.ip, local_ips()) ) - self.provisioner = EnvironmentProvisionerFactory.instance(parent=self.parent)\ - .create_provisioner_instance(self.kernel_spec.to_dict()) + self.provisioner = EPF.instance(parent=self.parent).\ + create_provisioner_instance(kw.get('kernel_id', str(uuid.uuid4())), self.kernel_spec.to_dict()) # write connection file / get default ports - self.write_connection_file() + self.write_connection_file() # TODO - this is created too soon # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd extra_arguments = kw.pop('extra_arguments', []) - kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) + + kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c # Give provisioner a crack at the kernel cmd - kernel_cmd = self.provisioner.format_kernel_cmd(kernel_cmd, extra_arguments=extra_arguments, **kw) - - env = kw.pop('env', os.environ).copy() - # Don't allow PYTHONEXECUTABLE to be passed to kernel process. - # If set, it can bork all the things. - env.pop('PYTHONEXECUTABLE', None) - if not self.kernel_cmd: - # If kernel_cmd has been set manually, don't refer to a kernel spec. - # Environment variables from kernel spec are added to os.environ. - env.update(self._get_env_substitutions(self.kernel_spec.env, env)) - - kw['env'] = env + kernel_cmd, kw = self.provisioner.pre_launch(kernel_cmd, extra_arguments=extra_arguments, **kw) return kernel_cmd, kw - def _get_env_substitutions(self, templated_env, substitution_values): - """ Walks env entries in templated_env and applies possible substitutions from current env - (represented by substitution_values). - Returns the substituted list of env entries. - """ - substituted_env = {} - if templated_env: - from string import Template - - # For each templated env entry, fill any templated references - # matching names of env variables with those values and build - # new dict with substitutions. - for k, v in templated_env.items(): - substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) - return substituted_env - def post_start_kernel(self, **kw): self.start_restarter() self._connect_control_socket() @@ -750,7 +729,7 @@ async def start_new_async_kernel(startup_timeout=60, kernel_name='python', **kwa await km.shutdown_kernel() raise - return (km, kc) + return km, kc @contextmanager diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index 105d2dc8b..a169f3a77 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -182,6 +182,7 @@ def start_kernel(self, kernel_name=None, **kwargs): The kernel ID for the newly started kernel is returned. """ km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) + kwargs['kernel_id'] = kernel_id # Make kernel_id available to manager and provisioner km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 8afcf9f5e..dae19745f 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -7,8 +7,8 @@ import signal import sys -from entrypoints import get_group_all, EntryPoint -from typing import Optional, Dict, List, Any +from entrypoints import EntryPoint, get_group_all, get_single, NoSuchEntryPoint +from typing import Optional, Dict, List, Any, Tuple from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable from .launcher import launch_kernel @@ -22,6 +22,15 @@ class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name Theses methods model those of the Subprocess Popen class: https://docs.python.org/3/library/subprocess.html#popen-objects """ + # The kernel specification associated with this provisioner + kernel_spec: Dict[str, Any] + kernel_id: str + + def __init__(self, **kwargs): + # Pop off expected arguments... + self.kernel_spec = kwargs.pop('kernel_spec', None) + self.kernel_id = kwargs.pop('kernel_id', None) + super().__init__(**kwargs) def poll(self) -> [int, None]: """Checks if kernel process is still running. @@ -60,15 +69,34 @@ def cleanup(self) -> None: """Cleanup any resources allocated on behalf of the kernel provisioner.""" raise NotImplementedError() - def format_kernel_cmd(self, cmd: List[str], **kwargs: Dict[str, Any]) -> List[str]: - """Replace any kernel provisioner-specific templated arguments in the launch command.""" - return cmd + def pre_launch(self, cmd: List[str], **kwargs: Any) -> Tuple[List[str], Dict[str, Any]]: + """Perform any steps in preparation for kernel process launch. - def pre_launch(self, cmd: List[str], **kwargs: Dict[str, Any]) -> None: - """Perform any steps in preparation for kernel process launch.""" - pass + This includes applying additional substitutions to the kernel launch command and env. + It also includes preparation of launch parameters. + + Returns the command list and potentially updated kwargs. + """ + + env = kwargs.pop('env', os.environ).copy() + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + # TODO: Potential B/C issue... Setting kernel_cmd has been deprecated since (April 2014). But, if set + # this will update its kernel process's env with those from the kernelspec and templated values + # filled with env when that didn't happen before. (Which I view as better anyway.) + + # if not self.kernel_cmd: + # # If kernel_cmd has been set manually, don't refer to a kernel spec. + # # Environment variables from kernel spec are added to os.environ. + # env.update(self._get_env_substitutions(self.kernel_spec.env, env)) + env.update(self._get_env_substitutions(env)) + + kwargs['env'] = env + + return cmd, kwargs - def launch_kernel(self, cmd: List[str], **kwargs: Dict[str, Any]) -> 'EnvironmentProvisionerBase': + def launch_kernel(self, cmd: List[str], **kwargs: Any) -> 'EnvironmentProvisionerBase': """Launch the kernel process returning the class instance.""" raise NotImplementedError() @@ -93,6 +121,22 @@ def load_provisioner_info(self, provisioner_info: Dict) -> None: """ pass + def _get_env_substitutions(self, substitution_values): + """ Walks env entries in templated_env and applies possible substitutions from current env + (represented by substitution_values). + Returns the substituted list of env entries. + """ + substituted_env = {} + if self.kernel_spec: + from string import Template + # For each templated env entry, fill any templated references + # matching names of env variables with those values and build + # new dict with substitutions. + templated_env = self.kernel_spec.get('env', {}) + for k, v in templated_env.items(): + substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) + return substituted_env + class ClientProvisioner(EnvironmentProvisionerBase): # TODO - determine name for default class @@ -103,18 +147,6 @@ def __init__(self, **kwargs): self.pgid = None self.ip = None # TODO - assume local_ip? - def launch_kernel(self, cmd, **kwargs): - self.popen = launch_kernel(cmd, **kwargs) - pgid = None - if hasattr(os, "getpgid"): - try: - pgid = os.getpgid(self.popen.pid) - except OSError: - pass - self.pid = self.popen.pid - self.pgid = pgid - return self - def poll(self) -> [int, None]: if self.popen: return self.popen.poll() @@ -170,6 +202,28 @@ def terminate(self) -> None: def cleanup(self) -> None: pass + def launch_kernel(self, cmd: List[str], **kwargs: Any): + scrubbed_kwargs = ClientProvisioner.scrub_kwargs(kwargs) + self.popen = launch_kernel(cmd, **scrubbed_kwargs) + pgid = None + if hasattr(os, "getpgid"): + try: + pgid = os.getpgid(self.popen.pid) + except OSError: + pass + self.pid = self.popen.pid + self.pgid = pgid + return self + + @staticmethod + def scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Remove any keyword arguments that Popen does not tolerate.""" + keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id'] + scrubbed_kwargs = kwargs.copy() + for kw in keywords_to_scrub: + scrubbed_kwargs.pop(kw, None) + return scrubbed_kwargs + def get_provisioner_info(self) -> Dict: """Captures the base information necessary for kernel persistence relative to the provisioner. """ @@ -190,24 +244,37 @@ class EnvironmentProvisionerFactory(SingletonConfigurable): """EnvironmentProvisionerFactory is responsible for validating and initializing provisioner instances. """ + GROUP_NAME = 'jupyter_client.environment_provisioners' provisioners: Dict[str, EntryPoint] = {} def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - for ep in get_group_all('jupyter_client.environment_provisioners'): + for ep in get_group_all(EnvironmentProvisionerFactory.GROUP_NAME): self.provisioners[ep.name] = ep - def is_provisioner_available(self, kernel_spec: Dict[str, Any]) -> bool: + def is_provisioner_available(self, kernel_name: str, kernel_spec: Dict[str, Any]) -> bool: """ Reads the associated kernel_spec to determine the provisioner and returns whether it - exists as an entry_point (True) or not (False). + exists as an entry_point (True) or not (False). If the referenced provisioner is not + in the current set of provisioners, attempt to retrieve its entrypoint. If found, add + to the list, else catch exception and return false. """ + is_available: bool = True provisioner_cfg = EnvironmentProvisionerFactory._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') - return provisioner_name in self.provisioners - - def create_provisioner_instance(self, kernel_spec: Dict[str, Any]) -> EnvironmentProvisionerBase: + if provisioner_name not in self.provisioners: + try: + ep = get_single(EnvironmentProvisionerFactory.GROUP_NAME, provisioner_name) + self.provisioners[provisioner_name] = ep # Update cache + except NoSuchEntryPoint: + is_available = False + self.log.warning( + f"Kernel '{kernel_name}' is referencing an environment provisioner ('{provisioner_name}') " + f"that is not available. Ensure the appropriate package has been installed and retry.") + return is_available + + def create_provisioner_instance(self, kernel_id: str, kernel_spec: Dict[str, Any]) -> EnvironmentProvisionerBase: """ Reads the associated kernel_spec and to see if has an environment_provisioner stanza. If one exists, it instantiates an instance. If an environment provisioner is not @@ -222,11 +289,14 @@ def create_provisioner_instance(self, kernel_spec: Dict[str, Any]) -> Environmen if provisioner_name not in self.provisioners: raise ModuleNotFoundError(f"Environment provisioner '{provisioner_name}' has not been registered.") - self.log.debug("Instantiating kernel '{}' with environment provisioner: {}". - format(kernel_spec.get('display_name'), provisioner_name)) + self.log.debug(f"Instantiating kernel '{kernel_spec.get('display_name')}' with " + f"environment provisioner: {provisioner_name}") provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class(parent=self.parent, config=Config(provisioner_config)) + return provisioner_class(parent=self.parent, + kernel_id=kernel_id, + kernel_spec=kernel_spec, + config=Config(provisioner_config)) @staticmethod def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: From 31398ffa71314031c0637ae8054a39e727d7d125 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Tue, 2 Feb 2021 17:12:11 -0800 Subject: [PATCH 04/37] Cleanup launch to better show responsibilities --- jupyter_client/kernelspec.py | 2 +- jupyter_client/manager.py | 42 +++++---------- jupyter_client/provisioning.py | 99 +++++++++++++++++++++++----------- 3 files changed, 82 insertions(+), 61 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 6e54e3e75..92ab086df 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -199,7 +199,7 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): if not kspec: kspec = self.kernel_spec_class.from_resource_dir(resource_dir) - if not EPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec.to_dict()): + if not EPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec): raise NoSuchKernel(kernel_name) return kspec diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 59271b188..4d5c69292 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -223,7 +223,11 @@ def _launch_kernel(self, kernel_cmd, **kw): override in a subclass to launch kernel subprocesses differently """ - return self.provisioner.launch_kernel(kernel_cmd, **kw) + kernel_proc, connection_info = self.provisioner.launch_kernel(kernel_cmd, **kw) + # Provisioner provides the connection information. Load into kernel manager and write file. + self.load_connection_info(connection_info) + self.write_connection_file() + return kernel_proc # Control socket used for polite kernel shutdown @@ -250,32 +254,11 @@ def pre_start_kernel(self, **kw): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - # TODO - clean this method up. Most should move into the provisioner base class, although - # attributes (e.g., transport, ip, _launch_args) need to remain here for b/c. - - if self.transport == 'tcp' and not is_local_ip(self.ip): - raise RuntimeError("Can only launch a kernel on a local interface. " - "This one is not: %s." - "Make sure that the '*_address' attributes are " - "configured properly. " - "Currently valid addresses are: %s" % (self.ip, local_ips()) - ) - self.provisioner = EPF.instance(parent=self.parent).\ - create_provisioner_instance(kw.get('kernel_id', str(uuid.uuid4())), self.kernel_spec.to_dict()) - - # write connection file / get default ports - self.write_connection_file() # TODO - this is created too soon - - # save kwargs for use in restart - self._launch_args = kw.copy() - # build the Popen cmd - extra_arguments = kw.pop('extra_arguments', []) - - kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c + create_provisioner_instance(kw.get('kernel_id', str(uuid.uuid4())), self.kernel_spec) - # Give provisioner a crack at the kernel cmd - kernel_cmd, kw = self.provisioner.pre_launch(kernel_cmd, extra_arguments=extra_arguments, **kw) + kw = self.provisioner.pre_launch(kernel_manager=self, **kw) + kernel_cmd = kw.pop('cmd') return kernel_cmd, kw def post_start_kernel(self, **kw): @@ -515,8 +498,11 @@ async def _launch_kernel(self, kernel_cmd, **kw): override in a subclass to launch kernel subprocesses differently """ - res = self.provisioner.launch_kernel(kernel_cmd, **kw) - return res + kernel_proc, connection_info = self.provisioner.launch_kernel(kernel_cmd, **kw) + # Provisioner provides the connection information. Load into kernel manager and write file. + self.load_connection_info(connection_info) + self.write_connection_file() + return kernel_proc async def start_kernel(self, **kw): """Starts a kernel in a separate process in an asynchronous manner. @@ -534,7 +520,7 @@ async def start_kernel(self, **kw): # launch the kernel subprocess self.log.debug("Starting kernel (async): %s", kernel_cmd) - self.kernel = await self._launch_kernel(kernel_cmd, **kw) + self.kernel = await self._launch_kernel(kernel_cmd, **kw) # TODO merge connection info self.post_start_kernel(**kw) async def finish_shutdown(self, waittime=None, pollinterval=0.1): diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index dae19745f..cb99a8642 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -11,7 +11,9 @@ from typing import Optional, Dict, List, Any, Tuple from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable +from .connect import write_connection_file from .launcher import launch_kernel +from .localinterfaces import is_local_ip, local_ips DEFAULT_PROVISIONER = "ClientProvisioner" @@ -23,7 +25,7 @@ class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name https://docs.python.org/3/library/subprocess.html#popen-objects """ # The kernel specification associated with this provisioner - kernel_spec: Dict[str, Any] + kernel_spec: Any kernel_id: str def __init__(self, **kwargs): @@ -69,35 +71,34 @@ def cleanup(self) -> None: """Cleanup any resources allocated on behalf of the kernel provisioner.""" raise NotImplementedError() - def pre_launch(self, cmd: List[str], **kwargs: Any) -> Tuple[List[str], Dict[str, Any]]: + def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """Perform any steps in preparation for kernel process launch. This includes applying additional substitutions to the kernel launch command and env. It also includes preparation of launch parameters. - Returns the command list and potentially updated kwargs. + Returns potentially updated kwargs. """ env = kwargs.pop('env', os.environ).copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) - # TODO: Potential B/C issue... Setting kernel_cmd has been deprecated since (April 2014). But, if set - # this will update its kernel process's env with those from the kernelspec and templated values - # filled with env when that didn't happen before. (Which I view as better anyway.) - - # if not self.kernel_cmd: - # # If kernel_cmd has been set manually, don't refer to a kernel spec. - # # Environment variables from kernel spec are added to os.environ. - # env.update(self._get_env_substitutions(self.kernel_spec.env, env)) + env.update(self._get_env_substitutions(env)) + self._validate_parameters(env, **kwargs) + kwargs['env'] = env - return cmd, kwargs + return kwargs + + def _validate_parameters(self, env: Dict[str, Any], **kwargs: Any) -> None: + """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" + pass - def launch_kernel(self, cmd: List[str], **kwargs: Any) -> 'EnvironmentProvisionerBase': - """Launch the kernel process returning the class instance.""" + def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: + """Launch the kernel process returning the class instance and connection info.""" raise NotImplementedError() def post_launch(self) -> None: @@ -132,7 +133,7 @@ def _get_env_substitutions(self, substitution_values): # For each templated env entry, fill any templated references # matching names of env variables with those values and build # new dict with substitutions. - templated_env = self.kernel_spec.get('env', {}) + templated_env = self.kernel_spec.env for k, v in templated_env.items(): substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) return substituted_env @@ -145,7 +146,7 @@ def __init__(self, **kwargs): self.popen = None self.pid = None self.pgid = None - self.ip = None # TODO - assume local_ip? + self.connection_info = {} def poll(self) -> [int, None]: if self.popen: @@ -202,7 +203,44 @@ def terminate(self) -> None: def cleanup(self) -> None: pass - def launch_kernel(self, cmd: List[str], **kwargs: Any): + def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + """Perform any steps in preparation for kernel process launch. + + This includes applying additional substitutions to the kernel launch command and env. + It also includes preparation of launch parameters. + + Returns the updated kwargs. + """ + + # If we have a kernel_manager pop it out of the args and use it to retain b/c. + # This should be considered temporary until a better division of labor can be defined. + km = kwargs.pop('kernel_manager') + if km: + if km.transport == 'tcp' and not is_local_ip(km.ip): + raise RuntimeError("Can only launch a kernel on a local interface. " + "This one is not: %s." + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % (km.ip, local_ips()) + ) + + # save kwargs for use in restart + km._launch_args = kwargs.copy() # TODO - stash these on provisioner? + # build the Popen cmd + extra_arguments = kwargs.pop('extra_arguments', []) + + # write connection file / get default ports + km.write_connection_file() # TODO - this will need to change when handshake pattern is adopted + self.connection_info = km.get_connection_info() + + kernel_cmd = km.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c + else: + extra_arguments = kwargs.pop('extra_arguments', []) + kernel_cmd = self.kernel_spec.argv + extra_arguments + + return super().pre_launch(cmd=kernel_cmd, **kwargs) + + def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['ClientProvisioner', Dict]: scrubbed_kwargs = ClientProvisioner.scrub_kwargs(kwargs) self.popen = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -213,12 +251,12 @@ def launch_kernel(self, cmd: List[str], **kwargs: Any): pass self.pid = self.popen.pid self.pgid = pgid - return self + return self, self.connection_info @staticmethod def scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: """Remove any keyword arguments that Popen does not tolerate.""" - keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id'] + keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id', 'kernel_manager'] scrubbed_kwargs = kwargs.copy() for kw in keywords_to_scrub: scrubbed_kwargs.pop(kw, None) @@ -253,7 +291,7 @@ def __init__(self, **kwargs) -> None: for ep in get_group_all(EnvironmentProvisionerFactory.GROUP_NAME): self.provisioners[ep.name] = ep - def is_provisioner_available(self, kernel_name: str, kernel_spec: Dict[str, Any]) -> bool: + def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: """ Reads the associated kernel_spec to determine the provisioner and returns whether it exists as an entry_point (True) or not (False). If the referenced provisioner is not @@ -274,7 +312,7 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Dict[str, Any] f"that is not available. Ensure the appropriate package has been installed and retry.") return is_available - def create_provisioner_instance(self, kernel_id: str, kernel_spec: Dict[str, Any]) -> EnvironmentProvisionerBase: + def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> EnvironmentProvisionerBase: """ Reads the associated kernel_spec and to see if has an environment_provisioner stanza. If one exists, it instantiates an instance. If an environment provisioner is not @@ -289,7 +327,7 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Dict[str, Any if provisioner_name not in self.provisioners: raise ModuleNotFoundError(f"Environment provisioner '{provisioner_name}' has not been registered.") - self.log.debug(f"Instantiating kernel '{kernel_spec.get('display_name')}' with " + self.log.debug(f"Instantiating kernel '{kernel_spec.display_name}' with " f"environment provisioner: {provisioner_name}") provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') @@ -299,7 +337,7 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Dict[str, Any config=Config(provisioner_config)) @staticmethod - def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: + def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: """ Return the environment_provisioner stanza from the kernel_spec. @@ -308,7 +346,7 @@ def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: Parameters ---------- - kernel_spec : Dict + kernel_spec : Any - this is a KernelSpec type but listed as Any to avoid circular import The kernel specification object from which the provisioner dictionary is derived. Returns @@ -318,12 +356,9 @@ def _get_provisioner_config(kernel_spec: Dict[str, Any]) -> Dict[str, Any]: information. If no `config` sub-dictionary exists, an empty `config` dictionary will be added. """ - if kernel_spec: - metadata = kernel_spec.get('metadata', {}) - if 'environment_provisioner' in metadata: - env_provisioner = metadata.get('environment_provisioner') - if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default - if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one - env_provisioner.update({"config": {}}) - return env_provisioner # Return what we found (plus config stanza if necessary) + env_provisioner = kernel_spec.metadata.get('environment_provisioner', {}) + if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default + if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one + env_provisioner.update({"config": {}}) + return env_provisioner # Return what we found (plus config stanza if necessary) return {"provisioner_name": DEFAULT_PROVISIONER, "config": {}} From 76f45e974d055239a696bc6236b7a40c41b2a6fd Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 19 Feb 2021 18:24:13 -0800 Subject: [PATCH 05/37] Switch to AsyncKM and use async subproc --- jupyter_client/launcher.py | 123 +++++++++-- jupyter_client/manager.py | 246 ++++++++++++++------- jupyter_client/multikernelmanager.py | 2 +- jupyter_client/provisioning.py | 135 ++++++----- jupyter_client/tests/test_kernelmanager.py | 7 - jupyter_client/tests/test_provisioning.py | 23 -- jupyter_client/tests/utils.py | 5 +- 7 files changed, 349 insertions(+), 192 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 0646a434a..30c69854f 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -2,10 +2,11 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - +import asyncio import os import sys from subprocess import Popen, PIPE +from typing import Any, Dict, Tuple from traitlets.log import get_logger @@ -40,9 +41,97 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, Returns ------- - Popen instance for the kernel subprocess + subprocess.Popen instance for the kernel subprocess """ + kwargs, interrupt_event = prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, + independent=independent, cwd=cwd, **kw) + + try: + # Allow to use ~/ in the command or its arguments + cmd = list(map(os.path.expanduser, cmd)) + + proc = Popen(cmd, **kwargs) + except Exception as exc: + msg = ( + "Failed to run command:\n{}\n" + " PATH={!r}\n" + " with kwargs:\n{!r}\n" + ) + # exclude environment variables, + # which may contain access tokens and the like. + without_env = {key:value for key, value in kwargs.items() if key != 'env'} + msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) + get_logger().error(msg) + raise + + finish_process_launch(proc, stdin, interrupt_event) + + return proc + + +async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, + independent=False, cwd=None, **kw): + """ Launches a localhost kernel, binding to the specified ports using async subprocess. + + Parameters + ---------- + cmd : list, + A string of Python code that imports and executes a kernel entry point. + + stdin, stdout, stderr : optional (default None) + Standards streams, as defined in subprocess.Popen. + + env: dict, optional + Environment variables passed to the kernel + + independent : bool, optional (default False) + If set, the kernel process is guaranteed to survive if this process + dies. If not set, an effort is made to ensure that the kernel is killed + when this process dies. Note that in this case it is still good practice + to kill kernels manually before exiting. + + cwd : path, optional + The working dir of the kernel process (default: cwd of this process). + + **kw: optional + Additional arguments for Popen + + Returns + ------- + + asyncio.subprocess.Process instance for the kernel subprocess + """ + + kwargs, interrupt_event = prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, + independent=independent, cwd=cwd, **kw) + + try: + # Allow to use ~/ in the command or its arguments + cmd = list(map(os.path.expanduser, cmd)) + + proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) + except Exception as exc: + msg = ( + "Failed to run command:\n{}\n" + " PATH={!r}\n" + " with kwargs:\n{!r}\n" + ) + # exclude environment variables, + # which may contain access tokens and the like. + without_env = {key:value for key, value in kwargs.items() if key != 'env'} + msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) + get_logger().error(msg) + raise + + finish_process_launch(proc, stdin, interrupt_event) + + return proc + + +def prepare_process_args(stdin=None, stdout=None, stderr=None, env=None, + independent=False, cwd=None, **kw) -> Tuple[Dict[str, Any], Any]: + # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr # are invalid. Unfortunately, there is in general no way to detect whether # they are valid. The following two blocks redirect them to (temporary) @@ -116,6 +205,7 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, # (we always capture stdin, so this is already False by default on <3.7) kwargs['close_fds'] = False else: + interrupt_event = None # N/A # Create a new session. # This makes it easier to interrupt the kernel, # because we want to interrupt the whole process group. @@ -125,35 +215,22 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, if not independent: env['JPY_PARENT_PID'] = str(os.getpid()) - try: - # Allow to use ~/ in the command or its arguments - cmd = list(map(os.path.expanduser, cmd)) + return kwargs, interrupt_event - proc = Popen(cmd, **kwargs) - except Exception as exc: - msg = ( - "Failed to run command:\n{}\n" - " PATH={!r}\n" - " with kwargs:\n{!r}\n" - ) - # exclude environment variables, - # which may contain access tokens and the like. - without_env = {key:value for key, value in kwargs.items() if key != 'env'} - msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) - get_logger().error(msg) - raise + +def finish_process_launch(subprocess, stdin, interrupt_event) -> None: + """Finishes the process launch by patching the interrupt event (windows) and closing stdin. """ if sys.platform == 'win32': # Attach the interrupt event to the Popen objet so it can be used later. - proc.win32_interrupt_event = interrupt_event + subprocess.win32_interrupt_event = interrupt_event # Clean up pipes created to work around Popen bug. - if redirect_in: - if stdin is None: - proc.stdin.close() + if stdin is None: + subprocess.stdin.close() - return proc __all__ = [ 'launch_kernel', + 'async_launch_kernel' ] diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 4d5c69292..32cf30655 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -24,6 +24,7 @@ from jupyter_client import kernelspec from .connect import ConnectionFileMixin +from .launcher import launch_kernel from .localinterfaces import is_local_ip, local_ips from .managerabc import KernelManagerABC from .provisioning import EnvironmentProvisionerFactory as EPF, EnvironmentProvisionerBase @@ -31,7 +32,6 @@ class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. - This version starts kernels with Popen. """ @@ -53,13 +53,9 @@ def _client_factory_default(self): def _client_class_changed(self, change): self.client_factory = import_item(str(change['new'])) - # The kernel provisioner with which the KernelManager is communicating. - # This will generally be a ClientProvisioner instance unless the specification indicates otherwise. - # Note that we use two attributes, kernel and provisioner, that will point at the same provisioner instance. - # kernel will be non-None during the kernel's lifecycle, while provisioner will span that time, being set - # prior to launch and unset following the kernel's termination. - kernel: Optional[EnvironmentProvisionerBase] = None - provisioner: Optional[EnvironmentProvisionerBase] = None + # The kernel process with which the KernelManager is communicating. + # generally a Popen instance + kernel = Any() kernel_spec_manager = Instance(kernelspec.KernelSpecManager) @@ -92,10 +88,8 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec - # TODO - This has been deprecated since April 2014 (4.0 release) - time to remove? kernel_cmd = List(Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. - The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, @@ -219,15 +213,10 @@ def from_ns(match): return [ pat.sub(from_ns, arg) for arg in cmd ] def _launch_kernel(self, kernel_cmd, **kw): - """actually launch the kernel via the provisioner - + """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ - kernel_proc, connection_info = self.provisioner.launch_kernel(kernel_cmd, **kw) - # Provisioner provides the connection information. Load into kernel manager and write file. - self.load_connection_info(connection_info) - self.write_connection_file() - return kernel_proc + return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown @@ -244,33 +233,73 @@ def _close_control_socket(self): def pre_start_kernel(self, **kw): """Prepares a kernel for startup in a separate process. - If random ports (port=0) are being used, this method must be called before the channels are created. - Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - self.provisioner = EPF.instance(parent=self.parent).\ - create_provisioner_instance(kw.get('kernel_id', str(uuid.uuid4())), self.kernel_spec) + if self.transport == 'tcp' and not is_local_ip(self.ip): + raise RuntimeError("Can only launch a kernel on a local interface. " + "This one is not: %s." + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % (self.ip, local_ips()) + ) + + # write connection file / get default ports + self.write_connection_file() - kw = self.provisioner.pre_launch(kernel_manager=self, **kw) - kernel_cmd = kw.pop('cmd') + # save kwargs for use in restart + self._launch_args = kw.copy() + # build the Popen cmd + extra_arguments = kw.pop('extra_arguments', []) + kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) + env = kw.pop('env', os.environ).copy() + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + if not self.kernel_cmd: + # If kernel_cmd has been set manually, don't refer to a kernel spec. + # Environment variables from kernel spec are added to os.environ. + env.update(self._get_env_substitutions(self.kernel_spec.env, env)) + + kw['env'] = env return kernel_cmd, kw + def _get_env_substitutions(self, templated_env, substitution_values): + """ Walks env entries in templated_env and applies possible substitutions from current env + (represented by substitution_values). + Returns the substituted list of env entries. + """ + substituted_env = {} + if templated_env: + from string import Template + + # For each templated env entry, fill any templated references + # matching names of env variables with those values and build + # new dict with substitutions. + for k, v in templated_env.items(): + substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) + return substituted_env + def post_start_kernel(self, **kw): + """Performs any post startup tasks relative to the kernel. + + Parameters + ---------- + `**kw` : optional + keyword arguments that were used in the kernel process's launch. + """ self.start_restarter() self._connect_control_socket() def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. - If random ports (port=0) are being used, this method must be called before the channels are created. - Parameters ---------- `**kw` : optional @@ -295,7 +324,6 @@ def request_shutdown(self, restart=False): def finish_shutdown(self, waittime=None, pollinterval=0.1): """Wait for kernel shutdown, then kill process if it doesn't shutdown. - This does not send shutdown requests - use :meth:`request_shutdown` first. """ @@ -328,10 +356,6 @@ def cleanup_resources(self, restart=False): if self._created_context and not restart: self.context.destroy(linger=100) - if self.provisioner: - self.provisioner.cleanup() - self.provisioner = self.kernel = None - def cleanup(self, connection_file=True): """Clean up resources when the kernel is shut down""" warnings.warn("Method cleanup(connection_file=True) is deprecated, use cleanup_resources(restart=False).", @@ -340,13 +364,10 @@ def cleanup(self, connection_file=True): def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. - This attempts to shutdown the kernels cleanly by: - 1. Sending it a shutdown message over the control channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. - Parameters ---------- now : bool @@ -391,17 +412,14 @@ def shutdown_kernel(self, now=False, restart=False): def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. - Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. - In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. - newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. @@ -409,7 +427,6 @@ def restart_kernel(self, now=False, newports=False, **kw): the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. - `**kw` : optional Any options specified here will overwrite those used to launch the kernel. @@ -435,11 +452,28 @@ def has_kernel(self): def _kill_kernel(self): """Kill the running kernel. - This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - self.kernel.kill() + # Signal the kernel to terminate (sends SIGKILL on Unix and calls + # TerminateProcess() on Win32). + try: + if hasattr(signal, 'SIGKILL'): + self.signal_kernel(signal.SIGKILL) + else: + self.kernel.kill() + except OSError as e: + # In Windows, we will get an Access Denied error if the process + # has already terminated. Ignore it. + if sys.platform == 'win32': + if e.winerror != 5: + raise + # On Unix, we may get an ESRCH error if the process has already + # terminated. Ignore it. + else: + from errno import ESRCH + if e.errno != ESRCH: + raise # Block until the kernel terminates. self.kernel.wait() @@ -447,7 +481,6 @@ def _kill_kernel(self): def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. - Unlike ``signal_kernel``, this operation is well supported on all platforms. """ @@ -468,9 +501,20 @@ def interrupt_kernel(self): raise RuntimeError("Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): - """Sends signal (signum) to the kernel. + """Sends a signal to the process group of the kernel (this + usually includes the kernel and any subprocesses spawned by + the kernel). + Note that since only SIGTERM is supported on Windows, this function is + only useful on Unix systems. """ if self.has_kernel: + if hasattr(os, "getpgid") and hasattr(os, "killpg"): + try: + pgid = os.getpgid(self.kernel.pid) + os.killpg(pgid, signum) + return + except OSError: + pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @@ -490,19 +534,40 @@ def is_alive(self): class AsyncKernelManager(KernelManager): """Manages kernels in an asynchronous manner """ + kernel_id = None + client_class = DottedObjectName('jupyter_client.asynchronous.AsyncKernelClient') client_factory = Type(klass='jupyter_client.asynchronous.AsyncKernelClient') - async def _launch_kernel(self, kernel_cmd, **kw): - """actually launch the kernel via the provisioner + # The kernel provisioner with which the KernelManager is communicating. + # This will generally be a ClientProvisioner instance unless the specification indicates otherwise. + # Note that we use two attributes, kernel and provisioner, that will point at the same provisioner instance. + # kernel will be non-None during the kernel's lifecycle, while provisioner will span that time, being set + # prior to launch and unset following the kernel's termination. + kernel: Optional[EnvironmentProvisionerBase] = None + provisioner: Optional[EnvironmentProvisionerBase] = None - override in a subclass to launch kernel subprocesses differently + async def pre_start_kernel(self, **kw): + """Prepares a kernel for startup in a separate process. + + If random ports (port=0) are being used, this method must be called + before the channels are created. + + Parameters + ---------- + `**kw` : optional + keyword arguments that are passed down to build the kernel_cmd + and launching the kernel (e.g. Popen kwargs). """ - kernel_proc, connection_info = self.provisioner.launch_kernel(kernel_cmd, **kw) - # Provisioner provides the connection information. Load into kernel manager and write file. - self.load_connection_info(connection_info) - self.write_connection_file() - return kernel_proc + self.kernel_id = kw.pop('kernel_id', str(uuid.uuid4())) + self.provisioner = EPF.instance(parent=self.parent).\ + create_provisioner_instance(self.kernel_id, self.kernel_spec) + + # save kwargs for use in restart + self._launch_args = kw.copy() + kw = await self.provisioner.pre_launch(kernel_manager=self, **kw) + kernel_cmd = kw.pop('cmd') + return kernel_cmd, kw async def start_kernel(self, **kw): """Starts a kernel in a separate process in an asynchronous manner. @@ -516,14 +581,41 @@ async def start_kernel(self, **kw): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - kernel_cmd, kw = self.pre_start_kernel(**kw) + kernel_cmd, kw = await self.pre_start_kernel(**kw) # launch the kernel subprocess self.log.debug("Starting kernel (async): %s", kernel_cmd) - self.kernel = await self._launch_kernel(kernel_cmd, **kw) # TODO merge connection info - self.post_start_kernel(**kw) + self.kernel = await self._launch_kernel(kernel_cmd, **kw) + await self.post_start_kernel(**kw) + + async def post_start_kernel(self, **kw): + """Performs any post startup tasks relative to the kernel. - async def finish_shutdown(self, waittime=None, pollinterval=0.1): + Parameters + ---------- + `**kw` : optional + keyword arguments that were used in the kernel process's launch. + """ + super().post_start_kernel(**kw) + await self.provisioner.post_launch(**kw) + + async def _launch_kernel(self, kernel_cmd, **kw): + """actually launch the kernel via the provisioner + + override in a subclass to launch kernel subprocesses differently + """ + kernel_proc, connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) + # Provisioner provides the connection information. Load into kernel manager and write file. + self.load_connection_info(connection_info) + self.write_connection_file() + return kernel_proc + + def request_shutdown(self, restart=False): + """Send a shutdown request via control channel + """ + super().request_shutdown(restart=restart) + + async def finish_shutdown(self, waittime=None, pollinterval=0.1, restart=False): """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` @@ -531,17 +623,28 @@ async def finish_shutdown(self, waittime=None, pollinterval=0.1): """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) + + if self.provisioner: # Allow provisioner to override + waittime = self.provisioner.get_shutdown_wait_time(recommended=waittime) try: await asyncio.wait_for(self._async_wait(pollinterval=pollinterval), timeout=waittime) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, killing") - await self._kill_kernel() + await self._kill_kernel(restart=restart) else: # Process is no longer alive, wait and clear if self.kernel is not None: - self.kernel.wait() + await self.kernel.wait() self.kernel = None + async def cleanup_resources(self, restart=False): + """Clean up resources when the kernel is shut down""" + super().cleanup_resources(restart=restart) + + if self.provisioner: + await self.provisioner.cleanup(restart=restart) + self.provisioner = self.kernel = None + async def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. @@ -564,22 +667,15 @@ async def shutdown_kernel(self, now=False, restart=False): self.stop_restarter() if now: - await self._kill_kernel() + await self._kill_kernel(restart=restart) else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. - await self.finish_shutdown() + await self.finish_shutdown(restart=restart) - # See comment in KernelManager.shutdown_kernel(). - overrides_cleanup = type(self).cleanup is not AsyncKernelManager.cleanup - overrides_cleanup_resources = type(self).cleanup_resources is not AsyncKernelManager.cleanup_resources - - if overrides_cleanup and not overrides_cleanup_resources: - self.cleanup(connection_file=not restart) - else: - self.cleanup_resources(restart=restart) + await self.cleanup_resources(restart=restart) async def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. @@ -621,13 +717,13 @@ async def restart_kernel(self, now=False, newports=False, **kw): await self.start_kernel(**self._launch_args) return None - async def _kill_kernel(self): + async def _kill_kernel(self, restart=False): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - self.kernel.kill() + await self.kernel.kill(restart=restart) # Wait until the kernel terminates. try: @@ -639,7 +735,7 @@ async def _kill_kernel(self): else: # Process is no longer alive, wait and clear if self.kernel is not None: - self.kernel.wait() + await self.kernel.wait() self.kernel = None async def interrupt_kernel(self): @@ -661,18 +757,18 @@ async def interrupt_kernel(self): async def signal_kernel(self, signum): """Sends signal (signum) to the kernel.""" - super(AsyncKernelManager, self).signal_kernel(signum) + if self.has_kernel: + await self.kernel.send_signal(signum) + else: + raise RuntimeError("Cannot signal kernel. No kernel is running!") async def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: - if self.kernel.poll() is None: + ret = await self.kernel.poll() + if ret is None: return True - else: - return False - else: - # we don't have a kernel - return False + return False async def _async_wait(self, pollinterval=0.1): # Use busy loop at 100ms intervals, polling until the process is diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index d4bd0b198..76d8e1105 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -183,7 +183,6 @@ def start_kernel(self, kernel_name=None, **kwargs): The kernel ID for the newly started kernel is returned. """ km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) - kwargs['kernel_id'] = kernel_id # Make kernel_id available to manager and provisioner km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id @@ -473,6 +472,7 @@ async def start_kernel(self, kernel_name=None, **kwargs): if not isinstance(km, AsyncKernelManager): self.log.warning("Kernel manager class ({km_class}) is not an instance of 'AsyncKernelManager'!". format(km_class=self.kernel_manager_class.__class__)) + kwargs['kernel_id'] = kernel_id # Make kernel_id available to manager and provisioner fut = asyncio.ensure_future( self._add_kernel_when_ready( kernel_id, diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index cb99a8642..43db480ec 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -2,7 +2,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - +import asyncio import os import signal import sys @@ -12,13 +12,13 @@ from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable from .connect import write_connection_file -from .launcher import launch_kernel +from .launcher import async_launch_kernel from .localinterfaces import is_local_ip, local_ips DEFAULT_PROVISIONER = "ClientProvisioner" -class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name for base class +class EnvironmentProvisionerBase(LoggingConfigurable): """Base class defining methods for EnvironmentProvisioner classes. Theses methods model those of the Subprocess Popen class: @@ -28,50 +28,50 @@ class EnvironmentProvisionerBase(LoggingConfigurable): # TODO - determine name kernel_spec: Any kernel_id: str - def __init__(self, **kwargs): - # Pop off expected arguments... - self.kernel_spec = kwargs.pop('kernel_spec', None) - self.kernel_id = kwargs.pop('kernel_id', None) + def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): + self.kernel_id = kernel_id + self.kernel_spec = kernel_spec super().__init__(**kwargs) - def poll(self) -> [int, None]: + async def poll(self) -> [int, None]: """Checks if kernel process is still running. If running, None is returned, otherwise the process's integer-valued exit code is returned. """ raise NotImplementedError() - def wait(self, timeout: Optional[float] = None) -> [int, None]: - """Waits for kernel process to terminate. As a result, this method should be called with - a value for timeout. - - If the kernel process does not terminate following timeout seconds, a TimeoutException will - be raised - that can be caught and retried. If the kernel process has terminated, its - integer-valued exit code will be returned. - """ + async def wait(self) -> [int, None]: + """Waits for kernel process to terminate.""" raise NotImplementedError() - def send_signal(self, signum: int) -> None: + async def send_signal(self, signum: int) -> None: """Sends signal identified by signum to the kernel process.""" raise NotImplementedError() - def kill(self) -> None: + async def kill(self, restart=False) -> None: """Kills the kernel process. This is typically accomplished via a SIGKILL signal, which cannot be caught. + + restart is True if this operation precedes a start launch_kernel request. """ raise NotImplementedError() - def terminate(self) -> None: + async def terminate(self, restart=False) -> None: """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, which can be caught, allowing the kernel provisioner to perform possible cleanup of resources. + + restart is True if this operation precedes a start launch_kernel request. """ raise NotImplementedError() - def cleanup(self) -> None: - """Cleanup any resources allocated on behalf of the kernel provisioner.""" - raise NotImplementedError() + async def cleanup(self, restart=False) -> None: + """Cleanup any resources allocated on behalf of the kernel provisioner. - def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + restart is True if this operation precedes a start launch_kernel request. + """ + pass + + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """Perform any steps in preparation for kernel process launch. This includes applying additional substitutions to the kernel launch command and env. @@ -93,19 +93,19 @@ def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return kwargs - def _validate_parameters(self, env: Dict[str, Any], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" - pass - - def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: """Launch the kernel process returning the class instance and connection info.""" raise NotImplementedError() - def post_launch(self) -> None: + async def post_launch(self, **kwargs: Any) -> None: """Perform any steps following the kernel process launch.""" pass - def get_provisioner_info(self) -> Dict: + def _validate_parameters(self, env: Dict[str, Any], **kwargs: Any) -> None: + """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" + pass + + async def get_provisioner_info(self) -> Dict: """Captures the base information necessary for kernel persistence relative to the provisioner. The superclass method must always be called first to ensure proper ordering. Since this is the @@ -114,7 +114,7 @@ def get_provisioner_info(self) -> Dict: provisioner_info = {} return provisioner_info - def load_provisioner_info(self, provisioner_info: Dict) -> None: + async def load_provisioner_info(self, provisioner_info: Dict) -> None: """Loads the base information necessary for kernel persistence relative to the provisioner. The superclass method must always be called first to ensure proper ordering. Since this is the @@ -122,6 +122,13 @@ def load_provisioner_info(self, provisioner_info: Dict) -> None: """ pass + def get_shutdown_wait_time(self, recommended: Optional[float] = 5.0) -> float: + """Returns the time allowed for a complete shutdown. This may vary by provisioner. + + The recommended value will typically be what is configured in the kernel manager. + """ + return recommended + def _get_env_substitutions(self, substitution_values): """ Walks env entries in templated_env and applies possible substitutions from current env (represented by substitution_values). @@ -141,22 +148,25 @@ def _get_env_substitutions(self, substitution_values): class ClientProvisioner(EnvironmentProvisionerBase): # TODO - determine name for default class - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.popen = None + def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): + super().__init__(kernel_id, kernel_spec, **kwargs) + self.process = None + self._exit_future = None self.pid = None self.pgid = None self.connection_info = {} - def poll(self) -> [int, None]: - if self.popen: - return self.popen.poll() + async def poll(self) -> [int, None]: + if self.process and self._exit_future: + if not self._exit_future.done(): # adhere to process returncode + return None + return False - def wait(self, timeout: Optional[float] = None) -> [int, None]: - if self.popen: - return self.popen.wait(timeout=timeout) + async def wait(self) -> [int, None]: + if self.process: + return await self.process.wait() - def send_signal(self, signum: int) -> None: + async def send_signal(self, signum: int) -> None: """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). @@ -165,24 +175,25 @@ def send_signal(self, signum: int) -> None: check if the desired signal is for interrupt and apply the applicable code on Windows in that case. """ - if self.popen: + if self.process: if signum == signal.SIGINT and sys.platform == 'win32': from .win_interrupt import send_interrupt - send_interrupt(self.popen.win32_interrupt_event) + send_interrupt(self.process.win32_interrupt_event) return + # Prefer process-group over process if self.pgid and hasattr(os, "killpg"): try: os.killpg(self.pgid, signum) return except OSError: pass - return self.popen.send_signal(signum) + return self.process.send_signal(signum) - def kill(self) -> None: - if self.popen: + async def kill(self, restart=False) -> None: + if self.process: try: - self.popen.kill() + self.process.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. @@ -196,14 +207,14 @@ def kill(self) -> None: if e.errno != ESRCH: raise - def terminate(self) -> None: - if self.popen: - return self.popen.terminate() + async def terminate(self, restart=False) -> None: + if self.process: + return self.process.terminate() - def cleanup(self) -> None: + async def cleanup(self, restart=False) -> None: pass - def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """Perform any steps in preparation for kernel process launch. This includes applying additional substitutions to the kernel launch command and env. @@ -238,18 +249,20 @@ def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: extra_arguments = kwargs.pop('extra_arguments', []) kernel_cmd = self.kernel_spec.argv + extra_arguments - return super().pre_launch(cmd=kernel_cmd, **kwargs) + return await super().pre_launch(cmd=kernel_cmd, **kwargs) - def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['ClientProvisioner', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['ClientProvisioner', Dict]: scrubbed_kwargs = ClientProvisioner.scrub_kwargs(kwargs) - self.popen = launch_kernel(cmd, **scrubbed_kwargs) + self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) + self._exit_future = asyncio.ensure_future(self.process.wait()) pgid = None if hasattr(os, "getpgid"): try: - pgid = os.getpgid(self.popen.pid) + pgid = os.getpgid(self.process.pid) except OSError: pass - self.pid = self.popen.pid + + self.pid = self.process.pid self.pgid = pgid return self, self.connection_info @@ -331,10 +344,10 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Envir f"environment provisioner: {provisioner_name}") provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class(parent=self.parent, - kernel_id=kernel_id, - kernel_spec=kernel_spec, - config=Config(provisioner_config)) + return provisioner_class(kernel_id, + kernel_spec, + config=Config(provisioner_config), + parent=self.parent) @staticmethod def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index 07903b58a..894ed64ea 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -331,13 +331,6 @@ async def test_subclasses(self, async_km): assert is_alive is False assert async_km.context.closed - if isinstance(async_km, AsyncKernelManagerWithCleanup): - assert async_km.which_cleanup == "cleanup" - elif isinstance(async_km, AsyncKernelManagerSubclass): - assert async_km.which_cleanup == "cleanup_resources" - else: - assert hasattr(async_km, "which_cleanup") is False - @pytest.mark.timeout(10) @pytest.mark.skipif(sys.platform == 'win32', reason="Windows doesn't support signals") async def test_signal_kernel_subprocesses(self, install_kernel, start_async_kernel): diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 21a31e4b9..4dd1efdd1 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -75,11 +75,6 @@ def ksm(): return KernelSpecManager() -@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner']) -def km(request, no_provisioner, missing_provisioner, default_provisioner): - return KernelManager(kernel_name=request.param) - - @pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner']) def akm(request, no_provisioner, missing_provisioner, default_provisioner): return AsyncKernelManager(kernel_name=request.param) @@ -100,24 +95,6 @@ def test_get_missing(self, missing_provisioner, ksm): class TestRuntime: - def test_lifecycle(self, km): - assert km.provisioner is None - if km.kernel_name == 'missing_provisioner': - with pytest.raises(NoSuchKernel): - km.start_kernel() - else: - km.start_kernel() - assert isinstance(km.provisioner, ClientProvisioner) - assert km.kernel is km.provisioner - if km.kernel_name == 'default_provisioner': - assert km.provisioner.config.get('config_var_1') == 42 - assert km.provisioner.config.get('config_var_2') == 'foo' - else: - assert 'config_var_1' not in km.provisioner.config - assert 'config_var_2' not in km.provisioner.config - km.shutdown_kernel() - assert km.kernel is None - assert km.provisioner is None @pytest.mark.asyncio async def test_async_lifecycle(self, akm): diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index 9d23f339b..66932bd6b 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -73,10 +73,11 @@ def cleanup(self, connection_file=True): super().cleanup(connection_file=connection_file) self.which_cleanup = 'cleanup' - def cleanup_resources(self, restart=False): - super().cleanup_resources(restart=restart) + async def cleanup_resources(self, restart=False): + await super().cleanup_resources(restart=restart) self.which_cleanup = 'cleanup_resources' + class AsyncKernelManagerWithCleanup(AsyncKernelManager): """Used to test deprecation "routes" that are determined by superclass' detection of methods. From cac893cf957db1dd06b9551ba38428c29f2bd306 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 19 Feb 2021 20:34:00 -0800 Subject: [PATCH 06/37] Use popen on Windows --- jupyter_client/launcher.py | 8 +++++++- jupyter_client/provisioning.py | 29 ++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 30c69854f..66a803e18 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -110,7 +110,13 @@ async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=Non # Allow to use ~/ in the command or its arguments cmd = list(map(os.path.expanduser, cmd)) - proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) + if sys.platform == 'win32': + # Windows is still not ready for async subprocs. When the SelectorEventLoop is used (3.6, 3.7), + # a NotImplementedError is raised for _make_subprocess_transport(). When the ProactorEventLoop + # is used (3.8, 3.9), a NotImplementedError is raised for _add_reader(). + proc = Popen(cmd, **kwargs) + else: + proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) except Exception as exc: msg = ( "Failed to run command:\n{}\n" diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 43db480ec..17512018f 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -151,20 +151,37 @@ class ClientProvisioner(EnvironmentProvisionerBase): # TODO - determine name fo def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): super().__init__(kernel_id, kernel_spec, **kwargs) self.process = None + self.async_subprocess = None self._exit_future = None self.pid = None self.pgid = None self.connection_info = {} async def poll(self) -> [int, None]: - if self.process and self._exit_future: - if not self._exit_future.done(): # adhere to process returncode - return None + if self.process: + if self.async_subprocess: + if not self._exit_future.done(): # adhere to process returncode + return None + else: + if self.process.poll() is None: + return None return False async def wait(self) -> [int, None]: if self.process: - return await self.process.wait() + if self.async_subprocess: + await self.process.wait() + else: + # Use busy loop at 100ms intervals, polling until the process is + # not alive. If we find the process is no longer alive, complete + # its cleanup via the blocking wait(). Callers are responsible for + # issuing calls to wait() using a timeout (see kill()). + while await self.poll() is None: + await asyncio.sleep(0.1) + + # Process is no longer alive, wait and clear + self.process.wait() + self.process = None async def send_signal(self, signum: int) -> None: """Sends a signal to the process group of the kernel (this @@ -254,7 +271,9 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['ClientProvisioner', Dict]: scrubbed_kwargs = ClientProvisioner.scrub_kwargs(kwargs) self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) - self._exit_future = asyncio.ensure_future(self.process.wait()) + self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) + if self.async_subprocess: + self._exit_future = asyncio.ensure_future(self.process.wait()) pgid = None if hasattr(os, "getpgid"): try: From 7db3d9784c6ecee579de0fb3cc844388c914f910 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sat, 20 Feb 2021 10:55:59 -0800 Subject: [PATCH 07/37] Use abstract base class --- jupyter_client/provisioning.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 17512018f..a16f3ed39 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -7,6 +7,7 @@ import signal import sys +from abc import ABCMeta, ABC, abstractmethod from entrypoints import EntryPoint, get_group_all, get_single, NoSuchEntryPoint from typing import Optional, Dict, List, Any, Tuple from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable @@ -18,7 +19,11 @@ DEFAULT_PROVISIONER = "ClientProvisioner" -class EnvironmentProvisionerBase(LoggingConfigurable): +class EnvironmentProvisionerMeta(ABCMeta, type(LoggingConfigurable)): + pass + + +class EnvironmentProvisionerBase(ABC, LoggingConfigurable, metaclass=EnvironmentProvisionerMeta): """Base class defining methods for EnvironmentProvisioner classes. Theses methods model those of the Subprocess Popen class: @@ -33,36 +38,46 @@ def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): self.kernel_spec = kernel_spec super().__init__(**kwargs) + @abstractmethod async def poll(self) -> [int, None]: """Checks if kernel process is still running. If running, None is returned, otherwise the process's integer-valued exit code is returned. """ - raise NotImplementedError() + pass + @abstractmethod async def wait(self) -> [int, None]: """Waits for kernel process to terminate.""" - raise NotImplementedError() + pass + @abstractmethod async def send_signal(self, signum: int) -> None: """Sends signal identified by signum to the kernel process.""" - raise NotImplementedError() + pass + @abstractmethod async def kill(self, restart=False) -> None: """Kills the kernel process. This is typically accomplished via a SIGKILL signal, which cannot be caught. restart is True if this operation precedes a start launch_kernel request. """ - raise NotImplementedError() + pass + @abstractmethod async def terminate(self, restart=False) -> None: """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, which can be caught, allowing the kernel provisioner to perform possible cleanup of resources. restart is True if this operation precedes a start launch_kernel request. """ - raise NotImplementedError() + pass + + @abstractmethod + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: + """Launch the kernel process returning the class instance and connection info.""" + pass async def cleanup(self, restart=False) -> None: """Cleanup any resources allocated on behalf of the kernel provisioner. @@ -93,10 +108,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return kwargs - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: - """Launch the kernel process returning the class instance and connection info.""" - raise NotImplementedError() - async def post_launch(self, **kwargs: Any) -> None: """Perform any steps following the kernel process launch.""" pass From e6e381ecea109f1564a27c15ac06a2427917f9cc Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sun, 21 Feb 2021 11:07:55 -0800 Subject: [PATCH 08/37] Rename to Kernel Provisioning, and default LocalProvisioner --- jupyter_client/kernelspec.py | 4 +- jupyter_client/manager.py | 10 ++--- jupyter_client/provisioning.py | 54 +++++++++++------------ jupyter_client/tests/test_provisioning.py | 10 ++--- setup.py | 4 +- 5 files changed, 41 insertions(+), 41 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 92ab086df..042c43d21 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -16,7 +16,7 @@ from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH -from .provisioning import EnvironmentProvisionerFactory as EPF +from .provisioning import KernelProvisionerFactory as KPF pjoin = os.path.join @@ -199,7 +199,7 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): if not kspec: kspec = self.kernel_spec_class.from_resource_dir(resource_dir) - if not EPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec): + if not KPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec): raise NoSuchKernel(kernel_name) return kspec diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 32cf30655..6a91a75d8 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -27,7 +27,7 @@ from .launcher import launch_kernel from .localinterfaces import is_local_ip, local_ips from .managerabc import KernelManagerABC -from .provisioning import EnvironmentProvisionerFactory as EPF, EnvironmentProvisionerBase +from .provisioning import KernelProvisionerFactory as KPF, KernelProvisionerBase class KernelManager(ConnectionFileMixin): @@ -540,12 +540,12 @@ class AsyncKernelManager(KernelManager): client_factory = Type(klass='jupyter_client.asynchronous.AsyncKernelClient') # The kernel provisioner with which the KernelManager is communicating. - # This will generally be a ClientProvisioner instance unless the specification indicates otherwise. + # This will generally be a LocalProvisioner instance unless the specification indicates otherwise. # Note that we use two attributes, kernel and provisioner, that will point at the same provisioner instance. # kernel will be non-None during the kernel's lifecycle, while provisioner will span that time, being set # prior to launch and unset following the kernel's termination. - kernel: Optional[EnvironmentProvisionerBase] = None - provisioner: Optional[EnvironmentProvisionerBase] = None + kernel: Optional[KernelProvisionerBase] = None + provisioner: Optional[KernelProvisionerBase] = None async def pre_start_kernel(self, **kw): """Prepares a kernel for startup in a separate process. @@ -560,7 +560,7 @@ async def pre_start_kernel(self, **kw): and launching the kernel (e.g. Popen kwargs). """ self.kernel_id = kw.pop('kernel_id', str(uuid.uuid4())) - self.provisioner = EPF.instance(parent=self.parent).\ + self.provisioner = KPF.instance(parent=self.parent).\ create_provisioner_instance(self.kernel_id, self.kernel_spec) # save kwargs for use in restart diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index a16f3ed39..e23efd094 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -1,4 +1,4 @@ -"""Kernel Environment Provisioner Classes""" +"""Kernel Provisioner Classes""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. @@ -16,15 +16,15 @@ from .launcher import async_launch_kernel from .localinterfaces import is_local_ip, local_ips -DEFAULT_PROVISIONER = "ClientProvisioner" +DEFAULT_PROVISIONER = "LocalProvisioner" -class EnvironmentProvisionerMeta(ABCMeta, type(LoggingConfigurable)): +class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): pass -class EnvironmentProvisionerBase(ABC, LoggingConfigurable, metaclass=EnvironmentProvisionerMeta): - """Base class defining methods for EnvironmentProvisioner classes. +class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisionerMeta): + """Base class defining methods for KernelProvisioner classes. Theses methods model those of the Subprocess Popen class: https://docs.python.org/3/library/subprocess.html#popen-objects @@ -75,7 +75,7 @@ async def terminate(self, restart=False) -> None: pass @abstractmethod - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['EnvironmentProvisionerBase', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['KernelProvisionerBase', Dict]: """Launch the kernel process returning the class instance and connection info.""" pass @@ -157,7 +157,7 @@ def _get_env_substitutions(self, substitution_values): return substituted_env -class ClientProvisioner(EnvironmentProvisionerBase): # TODO - determine name for default class +class LocalProvisioner(KernelProvisionerBase): def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): super().__init__(kernel_id, kernel_spec, **kwargs) @@ -279,8 +279,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['ClientProvisioner', Dict]: - scrubbed_kwargs = ClientProvisioner.scrub_kwargs(kwargs) + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: + scrubbed_kwargs = LocalProvisioner.scrub_kwargs(kwargs) self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) if self.async_subprocess: @@ -308,30 +308,30 @@ def scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: def get_provisioner_info(self) -> Dict: """Captures the base information necessary for kernel persistence relative to the provisioner. """ - provisioner_info = super(ClientProvisioner, self).get_provisioner_info() + provisioner_info = super(LocalProvisioner, self).get_provisioner_info() provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) return provisioner_info def load_provisioner_info(self, provisioner_info: Dict) -> None: """Loads the base information necessary for kernel persistence relative to the provisioner. """ - super(ClientProvisioner, self).load_provisioner_info(provisioner_info) + super(LocalProvisioner, self).load_provisioner_info(provisioner_info) self.pid = provisioner_info['pid'] self.pgid = provisioner_info['pgid'] self.ip = provisioner_info['ip'] -class EnvironmentProvisionerFactory(SingletonConfigurable): - """EnvironmentProvisionerFactory is responsible for validating and initializing provisioner instances. +class KernelProvisionerFactory(SingletonConfigurable): + """KernelProvisionerFactory is responsible for validating and initializing provisioner instances. """ - GROUP_NAME = 'jupyter_client.environment_provisioners' + GROUP_NAME = 'jupyter_client.kernel_provisioners' provisioners: Dict[str, EntryPoint] = {} def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - for ep in get_group_all(EnvironmentProvisionerFactory.GROUP_NAME): + for ep in get_group_all(KernelProvisionerFactory.GROUP_NAME): self.provisioners[ep.name] = ep def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: @@ -342,36 +342,36 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: to the list, else catch exception and return false. """ is_available: bool = True - provisioner_cfg = EnvironmentProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_cfg = KernelProvisionerFactory._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: try: - ep = get_single(EnvironmentProvisionerFactory.GROUP_NAME, provisioner_name) + ep = get_single(KernelProvisionerFactory.GROUP_NAME, provisioner_name) self.provisioners[provisioner_name] = ep # Update cache except NoSuchEntryPoint: is_available = False self.log.warning( - f"Kernel '{kernel_name}' is referencing an environment provisioner ('{provisioner_name}') " + f"Kernel '{kernel_name}' is referencing a kernel provisioner ('{provisioner_name}') " f"that is not available. Ensure the appropriate package has been installed and retry.") return is_available - def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> EnvironmentProvisionerBase: + def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> KernelProvisionerBase: """ - Reads the associated kernel_spec and to see if has an environment_provisioner stanza. - If one exists, it instantiates an instance. If an environment provisioner is not + Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. + If one exists, it instantiates an instance. If a kernel provisioner is not specified in the kernelspec, a DEFAULT_PROVISIONER stanza is fabricated and instantiated. The instantiated instance is returned. If the provisioner is found to not exist (not registered via entry_points), ModuleNotFoundError is raised. """ - provisioner_cfg = EnvironmentProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_cfg = KernelProvisionerFactory._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: - raise ModuleNotFoundError(f"Environment provisioner '{provisioner_name}' has not been registered.") + raise ModuleNotFoundError(f"Kernel provisioner '{provisioner_name}' has not been registered.") self.log.debug(f"Instantiating kernel '{kernel_spec.display_name}' with " - f"environment provisioner: {provisioner_name}") + f"kernel provisioner: {provisioner_name}") provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') return provisioner_class(kernel_id, @@ -382,9 +382,9 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Envir @staticmethod def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: """ - Return the environment_provisioner stanza from the kernel_spec. + Return the kernel_provisioner stanza from the kernel_spec. - Checks the kernel_spec's metadata dictionary for an environment_provisioner entry. + Checks the kernel_spec's metadata dictionary for a kernel_provisioner entry. If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER and returned. Parameters @@ -399,7 +399,7 @@ def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: information. If no `config` sub-dictionary exists, an empty `config` dictionary will be added. """ - env_provisioner = kernel_spec.metadata.get('environment_provisioner', {}) + env_provisioner = kernel_spec.metadata.get('kernel_provisioner', {}) if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one env_provisioner.update({"config": {}}) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 4dd1efdd1..5a673f59c 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -11,7 +11,7 @@ from jupyter_core import paths from ..kernelspec import KernelSpecManager, NoSuchKernel from ..manager import KernelManager, AsyncKernelManager -from ..provisioning import ClientProvisioner +from ..provisioning import LocalProvisioner pjoin = os.path.join @@ -42,8 +42,8 @@ def default_provisioner(): 'display_name': "Signal Test Kernel w Provisioner", 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, 'metadata': { - 'environment_provisioner': { - 'provisioner_name': 'ClientProvisioner', + 'kernel_provisioner': { + 'provisioner_name': 'LocalProvisioner', 'config': {'config_var_1': 42, 'config_var_2': 'foo'} } } @@ -62,7 +62,7 @@ def missing_provisioner(): 'display_name': "Signal Test Kernel Missing Provisioner", 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, 'metadata': { - 'environment_provisioner': { + 'kernel_provisioner': { 'provisioner_name': 'MissingProvisioner', 'config': {'config_var_1': 42, 'config_var_2': 'foo'} } @@ -104,7 +104,7 @@ async def test_async_lifecycle(self, akm): await akm.start_kernel() else: await akm.start_kernel() - assert isinstance(akm.provisioner, ClientProvisioner) + assert isinstance(akm.provisioner, LocalProvisioner) assert akm.kernel is akm.provisioner if akm.kernel_name == 'default_provisioner': assert akm.provisioner.config.get('config_var_1') == 42 diff --git a/setup.py b/setup.py index d2f8072a9..a69c547e1 100644 --- a/setup.py +++ b/setup.py @@ -90,8 +90,8 @@ def run(self): 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', 'jupyter-kernel = jupyter_client.kernelapp:main', ], - 'jupyter_client.environment_provisioners': [ - 'ClientProvisioner = jupyter_client.provisioning:ClientProvisioner', + 'jupyter_client.kernel_provisioners': [ + 'LocalProvisioner = jupyter_client.provisioning:LocalProvisioner', ] }, ) From f3692c3321de4af7939a913669950731df125db0 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Mon, 22 Feb 2021 17:19:19 -0800 Subject: [PATCH 09/37] Apply changes per review, add type hints, formalize default provisioner --- jupyter_client/launcher.py | 90 ++++++++++++++++++++++------------ jupyter_client/provisioning.py | 55 +++++++++++++-------- 2 files changed, 93 insertions(+), 52 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 66a803e18..056eb5cb3 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -6,13 +6,19 @@ import os import sys from subprocess import Popen, PIPE -from typing import Any, Dict, Tuple +from typing import Any, Dict, IO, List, NoReturn, Optional, Tuple, Union from traitlets.log import get_logger -def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, - independent=False, cwd=None, **kw): +def launch_kernel(cmd: List[str], + stdin: Optional[IO[str]] = None, + stdout: Optional[IO[str]] = None, + stderr: Optional[IO[str]] = None, + env: Optional[Dict[str, str]] = None, + independent: Optional[bool] = False, + cwd: Optional[str] = None, + **kw: Optional[Dict[str, Any]]) -> Popen: """ Launches a localhost kernel, binding to the specified ports. Parameters @@ -43,9 +49,10 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, subprocess.Popen instance for the kernel subprocess """ + proc = None - kwargs, interrupt_event = prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, - independent=independent, cwd=cwd, **kw) + kwargs, interrupt_event = _prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, + independent=independent, cwd=cwd, **kw) try: # Allow to use ~/ in the command or its arguments @@ -53,25 +60,21 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, proc = Popen(cmd, **kwargs) except Exception as exc: - msg = ( - "Failed to run command:\n{}\n" - " PATH={!r}\n" - " with kwargs:\n{!r}\n" - ) - # exclude environment variables, - # which may contain access tokens and the like. - without_env = {key:value for key, value in kwargs.items() if key != 'env'} - msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) - get_logger().error(msg) - raise + _handle_subprocess_exception(exc, cmd, env, kwargs) - finish_process_launch(proc, stdin, interrupt_event) + _finish_process_launch(proc, stdin, interrupt_event) return proc -async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, - independent=False, cwd=None, **kw): +async def async_launch_kernel(cmd: List[str], + stdin: Optional[IO[str]] = None, + stdout: Optional[IO[str]] = None, + stderr: Optional[IO[str]] = None, + env: Optional[Dict[str, str]] = None, + independent: Optional[bool] = False, + cwd: Optional[str] = None, + **kw: Optional[Dict[str, Any]]) -> Union[Popen, asyncio.subprocess.Process]: """ Launches a localhost kernel, binding to the specified ports using async subprocess. Parameters @@ -102,9 +105,10 @@ async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=Non asyncio.subprocess.Process instance for the kernel subprocess """ + proc = None - kwargs, interrupt_event = prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, - independent=independent, cwd=cwd, **kw) + kwargs, interrupt_event = _prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, + independent=independent, cwd=cwd, **kw) try: # Allow to use ~/ in the command or its arguments @@ -118,6 +122,19 @@ async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=Non else: proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) except Exception as exc: + _handle_subprocess_exception(exc, cmd, env, kwargs) + + _finish_process_launch(proc, stdin, interrupt_event) + + return proc + + +def _handle_subprocess_exception(exc: Exception, + cmd: List[str], + env: Dict[str, Any], + kwargs: Dict[str, Any]) -> NoReturn: + """ Log error message consisting of runtime arguments (sans env) prior to raising the exception. """ + try: msg = ( "Failed to run command:\n{}\n" " PATH={!r}\n" @@ -125,18 +142,25 @@ async def async_launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=Non ) # exclude environment variables, # which may contain access tokens and the like. - without_env = {key:value for key, value in kwargs.items() if key != 'env'} + without_env = {key: value for key, value in kwargs.items() if key != 'env'} msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) get_logger().error(msg) - raise + finally: + raise exc - finish_process_launch(proc, stdin, interrupt_event) - - return proc +def _prepare_process_args(stdin: Optional[IO[str]] = None, + stdout: Optional[IO[str]] = None, + stderr: Optional[IO[str]] = None, + env: Optional[Dict[str, str]] = None, + independent: Optional[bool] = False, + cwd: Optional[str] = None, + **kw: Optional[Dict[str, Any]]) -> Tuple[Dict[str, Any], Any]: + """ Prepares for process launch. -def prepare_process_args(stdin=None, stdout=None, stderr=None, env=None, - independent=False, cwd=None, **kw) -> Tuple[Dict[str, Any], Any]: + This consists of getting arguments into a form Popen/Process understands and creating the + necessary Windows structures to support interrupts. + """ # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr # are invalid. Unfortunately, there is in general no way to detect whether @@ -146,7 +170,7 @@ def prepare_process_args(stdin=None, stdout=None, stderr=None, env=None, # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. - redirect_in = True + # redirect_in = True _stdin = PIPE if stdin is None else stdin # If this process in running on pythonw, we know that stdin, stdout, and @@ -197,7 +221,7 @@ def prepare_process_args(stdin=None, stdout=None, stderr=None, env=None, else: pid = GetCurrentProcess() handle = DuplicateHandle(pid, pid, pid, 0, - True, # Inheritable by new processes. + True, # Inheritable by new processes. DUPLICATE_SAME_ACCESS) env['JPY_PARENT_PID'] = str(int(handle)) @@ -224,9 +248,13 @@ def prepare_process_args(stdin=None, stdout=None, stderr=None, env=None, return kwargs, interrupt_event -def finish_process_launch(subprocess, stdin, interrupt_event) -> None: +def _finish_process_launch(subprocess: Union[Popen, asyncio.subprocess.Process], + stdin: IO[str], + interrupt_event: Any) -> None: """Finishes the process launch by patching the interrupt event (windows) and closing stdin. """ + assert subprocess is not None + if sys.platform == 'win32': # Attach the interrupt event to the Popen objet so it can be used later. subprocess.win32_interrupt_event = interrupt_event diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index e23efd094..55af0d22d 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -10,14 +10,12 @@ from abc import ABCMeta, ABC, abstractmethod from entrypoints import EntryPoint, get_group_all, get_single, NoSuchEntryPoint from typing import Optional, Dict, List, Any, Tuple -from traitlets.config import Config, LoggingConfigurable, SingletonConfigurable +from traitlets.config import Config, default, LoggingConfigurable, SingletonConfigurable, Unicode from .connect import write_connection_file from .launcher import async_launch_kernel from .localinterfaces import is_local_ip, local_ips -DEFAULT_PROVISIONER = "LocalProvisioner" - class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): pass @@ -79,6 +77,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['KernelPro """Launch the kernel process returning the class instance and connection info.""" pass + @abstractmethod async def cleanup(self, restart=False) -> None: """Cleanup any resources allocated on behalf of the kernel provisioner. @@ -96,14 +95,13 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """ env = kwargs.pop('env', os.environ).copy() - # Don't allow PYTHONEXECUTABLE to be passed to kernel process. - # If set, it can bork all the things. - env.pop('PYTHONEXECUTABLE', None) - env.update(self._get_env_substitutions(env)) + env.update(self._apply_env_substitutions(env)) self._validate_parameters(env, **kwargs) + self._finalize_env(env) + kwargs['env'] = env return kwargs @@ -112,10 +110,6 @@ async def post_launch(self, **kwargs: Any) -> None: """Perform any steps following the kernel process launch.""" pass - def _validate_parameters(self, env: Dict[str, Any], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" - pass - async def get_provisioner_info(self) -> Dict: """Captures the base information necessary for kernel persistence relative to the provisioner. @@ -140,8 +134,8 @@ def get_shutdown_wait_time(self, recommended: Optional[float] = 5.0) -> float: """ return recommended - def _get_env_substitutions(self, substitution_values): - """ Walks env entries in templated_env and applies possible substitutions from current env + def _apply_env_substitutions(self, substitution_values: Dict[str, str]): + """ Walks env entries in the kernelspec's env stanza and applies possible substitutions from current env (represented by substitution_values). Returns the substituted list of env entries. """ @@ -156,6 +150,18 @@ def _get_env_substitutions(self, substitution_values): substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) return substituted_env + def _finalize_env(self, env: Dict[str, str]) -> None: + """ Ensures env is appropriate prior to launch. """ + + if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + + def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: + """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" + pass + class LocalProvisioner(KernelProvisionerBase): @@ -264,7 +270,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: ) # save kwargs for use in restart - km._launch_args = kwargs.copy() # TODO - stash these on provisioner? + km._launch_args = kwargs.copy() # build the Popen cmd extra_arguments = kwargs.pop('extra_arguments', []) @@ -322,12 +328,20 @@ def load_provisioner_info(self, provisioner_info: Dict) -> None: class KernelProvisionerFactory(SingletonConfigurable): - """KernelProvisionerFactory is responsible for validating and initializing provisioner instances. - """ + """KernelProvisionerFactory is responsible for validating and initializing provisioner instances.""" GROUP_NAME = 'jupyter_client.kernel_provisioners' provisioners: Dict[str, EntryPoint] = {} + default_provisioner_name_env = "JUPYTER_DEFAULT_PROVISIONER_NAME" + default_provisioner_name = Unicode(config=True, + help="""Indicates the name of the provisioner to use when no kernel_provisioner + entry is present in the kernelspec.""") + + @default('default_provisioner_name') + def default_provisioner_name_default(self): + return os.getenv(self.default_provisioner_name_env, "LocalProvisioner") + def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -342,7 +356,7 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: to the list, else catch exception and return false. """ is_available: bool = True - provisioner_cfg = KernelProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_cfg = self._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: try: @@ -365,7 +379,7 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Kerne If the provisioner is found to not exist (not registered via entry_points), ModuleNotFoundError is raised. """ - provisioner_cfg = KernelProvisionerFactory._get_provisioner_config(kernel_spec) + provisioner_cfg = self._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: raise ModuleNotFoundError(f"Kernel provisioner '{provisioner_name}' has not been registered.") @@ -379,8 +393,7 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Kerne config=Config(provisioner_config), parent=self.parent) - @staticmethod - def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: + def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: """ Return the kernel_provisioner stanza from the kernel_spec. @@ -404,4 +417,4 @@ def _get_provisioner_config(kernel_spec: Any) -> Dict[str, Any]: if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one env_provisioner.update({"config": {}}) return env_provisioner # Return what we found (plus config stanza if necessary) - return {"provisioner_name": DEFAULT_PROVISIONER, "config": {}} + return {"provisioner_name": self.default_provisioner_name, "config": {}} From e064f37b0e1d4b58685a8c45ecd372b43285d351 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 25 Feb 2021 17:22:43 -0800 Subject: [PATCH 10/37] Enhance tests --- jupyter_client/provisioning.py | 14 +- jupyter_client/tests/test_provisioning.py | 297 +++++++++++++++++----- 2 files changed, 242 insertions(+), 69 deletions(-) diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 55af0d22d..e0760732c 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -345,7 +345,7 @@ def default_provisioner_name_default(self): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - for ep in get_group_all(KernelProvisionerFactory.GROUP_NAME): + for ep in KernelProvisionerFactory._get_all_provisioners(): self.provisioners[ep.name] = ep def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: @@ -360,7 +360,7 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: try: - ep = get_single(KernelProvisionerFactory.GROUP_NAME, provisioner_name) + ep = KernelProvisionerFactory._get_provisioner(provisioner_name) self.provisioners[provisioner_name] = ep # Update cache except NoSuchEntryPoint: is_available = False @@ -418,3 +418,13 @@ def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: env_provisioner.update({"config": {}}) return env_provisioner # Return what we found (plus config stanza if necessary) return {"provisioner_name": self.default_provisioner_name, "config": {}} + + @staticmethod + def _get_all_provisioners() -> List[EntryPoint]: + """Wrapper around entrypoints.get_group_all() - primarily to facilitate testing.""" + return get_group_all(KernelProvisionerFactory.GROUP_NAME) + + @staticmethod + def _get_provisioner(name: str) -> EntryPoint: + """Wrapper around entrypoints.get_single() - primarily to facilitate testing.""" + return get_single(KernelProvisionerFactory.GROUP_NAME, name) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 5a673f59c..203024ac9 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -3,115 +3,278 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio import json import os import pytest +import signal import sys +from entrypoints import EntryPoint, NoSuchEntryPoint from jupyter_core import paths +from subprocess import PIPE +from typing import Any, Dict, List, Optional, Tuple + from ..kernelspec import KernelSpecManager, NoSuchKernel -from ..manager import KernelManager, AsyncKernelManager -from ..provisioning import LocalProvisioner +from ..launcher import launch_kernel +from ..manager import AsyncKernelManager +from ..provisioning import KernelProvisionerBase, LocalProvisioner, KernelProvisionerFactory pjoin = os.path.join -@pytest.fixture -def no_provisioner(): - kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'no_provisioner') - os.makedirs(kernel_dir) - with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: - f.write(json.dumps({ - 'argv': [sys.executable, - '-m', 'jupyter_client.tests.signalkernel', - '-f', '{connection_file}'], - 'display_name': "Signal Test Kernel Default", - 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'} - })) +class SubclassedTestProvisioner(LocalProvisioner): + pass -@pytest.fixture -def default_provisioner(): - kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'default_provisioner') - os.makedirs(kernel_dir) - with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: - f.write(json.dumps({ - 'argv': [sys.executable, +class CustomTestProvisioner(KernelProvisionerBase): + + def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): + super().__init__(kernel_id, kernel_spec, **kwargs) + self.process = None + self.pid = None + self.pgid = None + self.connection_info = None + + async def poll(self) -> [int, None]: + if self.process: + return self.process.poll() + return False + + async def wait(self) -> [int, None]: + if self.process: + while await self.poll() is None: + await asyncio.sleep(0.1) + + # Process is no longer alive, wait and clear + self.process.wait() + self.process = None + + async def send_signal(self, signum: int) -> None: + if self.process: + if signum == signal.SIGINT and sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.process.win32_interrupt_event) + return + + # Prefer process-group over process + if self.pgid and hasattr(os, "killpg"): + try: + os.killpg(self.pgid, signum) + return + except OSError: + pass + return self.process.send_signal(signum) + + async def kill(self, restart=False) -> None: + if self.process: + self.process.kill() + + async def terminate(self, restart=False) -> None: + if self.process: + self.process.terminate() + + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + km = kwargs.pop('kernel_manager') # TODO - this is temporary + if km: + # save kwargs for use in restart + km._launch_args = kwargs.copy() + # build the Popen cmd + extra_arguments = kwargs.pop('extra_arguments', []) + + # write connection file / get default ports + km.write_connection_file() + self.connection_info = km.get_connection_info() + + kernel_cmd = km.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c + + return await super().pre_launch(cmd=kernel_cmd, **kwargs) + + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['CustomTestProvisioner', Dict]: + scrubbed_kwargs = kwargs + self.process = launch_kernel(cmd, **scrubbed_kwargs) + pgid = None + if hasattr(os, "getpgid"): + try: + pgid = os.getpgid(self.process.pid) + except OSError: + pass + + self.pid = self.process.pid + self.pgid = pgid + return self, self.connection_info + + async def cleanup(self, restart=False) -> None: + pass + + +class NewTestProvisioner(CustomTestProvisioner): + pass + + +def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: + spec = {'argv': [sys.executable, '-m', 'jupyter_client.tests.signalkernel', '-f', '{connection_file}'], - 'display_name': "Signal Test Kernel w Provisioner", + 'display_name': f"Signal Test Kernel w {provisioner}", 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, - 'metadata': { - 'kernel_provisioner': { - 'provisioner_name': 'LocalProvisioner', - 'config': {'config_var_1': 42, 'config_var_2': 'foo'} - } - } - })) + 'metadata': {} + } + if provisioner: + kernel_provisioner = { + 'kernel_provisioner': { + 'provisioner_name': provisioner, + 'config': {'config_var_1': 42, 'config_var_2': name} + } + } + spec['metadata'].update(kernel_provisioner) -@pytest.fixture -def missing_provisioner(): - kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'missing_provisioner') + kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', name) os.makedirs(kernel_dir) with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: - f.write(json.dumps({ - 'argv': [sys.executable, - '-m', 'jupyter_client.tests.signalkernel', - '-f', '{connection_file}'], - 'display_name': "Signal Test Kernel Missing Provisioner", - 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, - 'metadata': { - 'kernel_provisioner': { - 'provisioner_name': 'MissingProvisioner', - 'config': {'config_var_1': 42, 'config_var_2': 'foo'} - } - } - })) + f.write(json.dumps(spec)) + +def new_provisioner(): + build_kernelspec('new_provisioner', 'NewTestProvisioner') + +def custom_provisioner(): + build_kernelspec('custom_provisioner', 'CustomTestProvisioner') @pytest.fixture -def ksm(): - return KernelSpecManager() +def all_provisioners(): + build_kernelspec('no_provisioner') + build_kernelspec('missing_provisioner', 'MissingProvisioner') + build_kernelspec('default_provisioner', 'LocalProvisioner') + build_kernelspec('subclassed_provisioner', 'SubclassedTestProvisioner') + custom_provisioner() -@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner']) -def akm(request, no_provisioner, missing_provisioner, default_provisioner): +@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner', + 'custom_provisioner', 'subclassed_provisioner']) +def akm(request, all_provisioners): return AsyncKernelManager(kernel_name=request.param) +def mock_get_all_provisioners(): + result = [] + for name, epstr in [('LocalProvisioner', 'jupyter_client.provisioning'), + ('SubclassedTestProvisioner', 'jupyter_client.tests.test_provisioning'), + ('CustomTestProvisioner', 'jupyter_client.tests.test_provisioning')]: + result.append(EntryPoint(name, epstr, name)) + return result + + +def mock_get_provisioner(name): + if name == 'NewTestProvisioner': + return EntryPoint('NewTestProvisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner') + # Since all other provisioners are added during the singleton's creation, we should + # always raise NoSuchEP for all others calling this method. + raise NoSuchEntryPoint(KernelProvisionerFactory.GROUP_NAME, name) + + +@pytest.fixture +def kpf(monkeypatch): + """Setup the Kernel Provisioner Factory, mocking the entypoint fetch calls.""" + monkeypatch.setattr(KernelProvisionerFactory, '_get_all_provisioners', mock_get_all_provisioners) + monkeypatch.setattr(KernelProvisionerFactory, '_get_provisioner', mock_get_provisioner) + factory = KernelProvisionerFactory.instance() + return factory + + class TestDiscovery: - def test_find_all_specs(self, no_provisioner, missing_provisioner, default_provisioner, ksm): + def test_find_all_specs(self, kpf, all_provisioners): + ksm = KernelSpecManager() kernels = ksm.get_all_specs() - # Ensure specs for no_provisioner and default_provisioner exist and missing_provisioner doesn't + # Ensure specs for initial provisiones exist and missing_provisioner (and new_provisioner) don't assert 'no_provisioner' in kernels assert 'default_provisioner' in kernels + assert 'subclassed_provisioner' in kernels + assert 'custom_provisioner' in kernels assert 'missing_provisioner' not in kernels + assert 'new_provisioner' not in kernels - def test_get_missing(self, missing_provisioner, ksm): + def test_get_missing(self, all_provisioners): + ksm = KernelSpecManager() with pytest.raises(NoSuchKernel): ksm.get_kernel_spec('missing_provisioner') + def test_get_new(self, kpf): + new_provisioner() # Introduce provisioner after initialization of KPF + ksm = KernelSpecManager() + kernel = ksm.get_kernel_spec('new_provisioner') + assert 'NewTestProvisioner' == kernel.metadata['kernel_provisioner']['provisioner_name'] + class TestRuntime: - @pytest.mark.asyncio - async def test_async_lifecycle(self, akm): - assert akm.provisioner is None - if akm.kernel_name == 'missing_provisioner': + async def akm_test(self, kernel_mgr): + """Starts a kernel, validates the associated provisioner's config, shuts down kernel """ + + assert kernel_mgr.provisioner is None + if kernel_mgr.kernel_name == 'missing_provisioner': with pytest.raises(NoSuchKernel): - await akm.start_kernel() + await kernel_mgr.start_kernel() else: - await akm.start_kernel() + await kernel_mgr.start_kernel() + + TestRuntime.validate_provisioner(kernel_mgr) + + await kernel_mgr.shutdown_kernel() + assert kernel_mgr.kernel is None + assert kernel_mgr.provisioner is None + + @pytest.mark.asyncio + async def test_existing(self, kpf, akm): + await self.akm_test(akm) + + @pytest.mark.asyncio + async def test_new(self, kpf): + new_provisioner() # Introduce provisioner after initialization of KPF + new_km = AsyncKernelManager(kernel_name='new_provisioner') + await self.akm_test(new_km) + + @pytest.mark.asyncio + async def test_custom_lifecycle(self, kpf): + custom_provisioner() + async_km = AsyncKernelManager(kernel_name='custom_provisioner') + await async_km.start_kernel(stdout=PIPE, stderr=PIPE) + is_alive = await async_km.is_alive() + assert is_alive + await async_km.restart_kernel(now=True) + is_alive = await async_km.is_alive() + assert is_alive + await async_km.interrupt_kernel() + assert isinstance(async_km, AsyncKernelManager) + await async_km.shutdown_kernel(now=True) + is_alive = await async_km.is_alive() + assert is_alive is False + assert async_km.context.closed + + @staticmethod + def validate_provisioner(akm: AsyncKernelManager): + # Ensure kernel attribute is the provisioner + assert akm.kernel is akm.provisioner + + # Validate provisioner config + if akm.kernel_name == 'no_provisioner': + assert 'config_var_1' not in akm.provisioner.config + assert 'config_var_2' not in akm.provisioner.config + else: + assert akm.provisioner.config.get('config_var_1') == 42 + assert akm.provisioner.config.get('config_var_2') == akm.kernel_name + + # Validate provisioner class + if akm.kernel_name in ['no_provisioner', 'default_provisioner', 'subclassed_provisioner']: assert isinstance(akm.provisioner, LocalProvisioner) - assert akm.kernel is akm.provisioner - if akm.kernel_name == 'default_provisioner': - assert akm.provisioner.config.get('config_var_1') == 42 - assert akm.provisioner.config.get('config_var_2') == 'foo' + if akm.kernel_name == 'subclassed_provisioner': + assert isinstance(akm.provisioner, SubclassedTestProvisioner) else: - assert 'config_var_1' not in akm.provisioner.config - assert 'config_var_2' not in akm.provisioner.config - await akm.shutdown_kernel() - assert akm.kernel is None - assert akm.provisioner is None + assert not isinstance(akm.provisioner, SubclassedTestProvisioner) + else: + assert isinstance(akm.provisioner, CustomTestProvisioner) + assert not isinstance(akm.provisioner, LocalProvisioner) + if akm.kernel_name == 'new_provisioner': + assert isinstance(akm.provisioner, NewTestProvisioner) From b0ee134cfb7e979e17474f8faae9c4e324ae7d9a Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 25 Feb 2021 17:43:31 -0800 Subject: [PATCH 11/37] Fix "enhanced" tests --- jupyter_client/tests/test_provisioning.py | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 203024ac9..99e15e223 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -53,7 +53,7 @@ async def wait(self) -> [int, None]: async def send_signal(self, signum: int) -> None: if self.process: if signum == signal.SIGINT and sys.platform == 'win32': - from .win_interrupt import send_interrupt + from ..win_interrupt import send_interrupt send_interrupt(self.process.win32_interrupt_event) return @@ -139,9 +139,11 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: def new_provisioner(): build_kernelspec('new_provisioner', 'NewTestProvisioner') + def custom_provisioner(): build_kernelspec('custom_provisioner', 'CustomTestProvisioner') + @pytest.fixture def all_provisioners(): build_kernelspec('no_provisioner') @@ -157,20 +159,27 @@ def akm(request, all_provisioners): return AsyncKernelManager(kernel_name=request.param) -def mock_get_all_provisioners(): +initial_provisioner_map = { + 'LocalProvisioner': 'jupyter_client.provisioning', + 'SubclassedTestProvisioner': 'jupyter_client.tests.test_provisioning', + 'CustomTestProvisioner': 'jupyter_client.tests.test_provisioning' +} + + +def mock_get_all_provisioners() -> List[EntryPoint]: result = [] - for name, epstr in [('LocalProvisioner', 'jupyter_client.provisioning'), - ('SubclassedTestProvisioner', 'jupyter_client.tests.test_provisioning'), - ('CustomTestProvisioner', 'jupyter_client.tests.test_provisioning')]: + for name, epstr in initial_provisioner_map.items(): result.append(EntryPoint(name, epstr, name)) return result -def mock_get_provisioner(name): +def mock_get_provisioner(name) -> EntryPoint: if name == 'NewTestProvisioner': return EntryPoint('NewTestProvisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner') - # Since all other provisioners are added during the singleton's creation, we should - # always raise NoSuchEP for all others calling this method. + + if name in initial_provisioner_map: + return EntryPoint(name, initial_provisioner_map[name], name) + raise NoSuchEntryPoint(KernelProvisionerFactory.GROUP_NAME, name) From 6bd9f56c5b90ca4f7e43a07b40686c2b1b247a1e Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Tue, 2 Mar 2021 10:22:15 -0800 Subject: [PATCH 12/37] Fix configurable behaviors This enables the ability to specify provisioner traits on command line or config file and override via kernelspec config stanza. --- jupyter_client/__init__.py | 1 + jupyter_client/provisioning.py | 33 ++++++--------- jupyter_client/tests/test_provisioning.py | 50 ++++++++++++++++------- 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/jupyter_client/__init__.py b/jupyter_client/__init__.py index f72c516d3..b4a8e193e 100644 --- a/jupyter_client/__init__.py +++ b/jupyter_client/__init__.py @@ -8,3 +8,4 @@ from .blocking import BlockingKernelClient from .asynchronous import AsyncKernelClient from .multikernelmanager import MultiKernelManager, AsyncMultiKernelManager +from .provisioning import KernelProvisionerBase, LocalProvisioner diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index e0760732c..7dd59d5ed 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -10,7 +10,7 @@ from abc import ABCMeta, ABC, abstractmethod from entrypoints import EntryPoint, get_group_all, get_single, NoSuchEntryPoint from typing import Optional, Dict, List, Any, Tuple -from traitlets.config import Config, default, LoggingConfigurable, SingletonConfigurable, Unicode +from traitlets.config import Config, default, Instance, LoggingConfigurable, SingletonConfigurable, Unicode from .connect import write_connection_file from .launcher import async_launch_kernel @@ -28,13 +28,9 @@ class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisione https://docs.python.org/3/library/subprocess.html#popen-objects """ # The kernel specification associated with this provisioner - kernel_spec: Any - kernel_id: str - - def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): - self.kernel_id = kernel_id - self.kernel_spec = kernel_spec - super().__init__(**kwargs) + kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) + kernel_id: str = Unicode(None, allow_none=True) + connection_info: dict = {} @abstractmethod async def poll(self) -> [int, None]: @@ -165,14 +161,11 @@ def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: class LocalProvisioner(KernelProvisionerBase): - def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): - super().__init__(kernel_id, kernel_spec, **kwargs) - self.process = None - self.async_subprocess = None - self._exit_future = None - self.pid = None - self.pgid = None - self.connection_info = {} + process = None + async_subprocess = None + _exit_future = None + pid = None + pgid = None async def poll(self) -> [int, None]: if self.process: @@ -388,10 +381,10 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Kerne f"kernel provisioner: {provisioner_name}") provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class(kernel_id, - kernel_spec, - config=Config(provisioner_config), - parent=self.parent) + return provisioner_class(kernel_id=kernel_id, + kernel_spec=kernel_spec, + parent=self.parent, + **provisioner_config) def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: """ diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 99e15e223..b27ad1eee 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -13,6 +13,7 @@ from entrypoints import EntryPoint, NoSuchEntryPoint from jupyter_core import paths from subprocess import PIPE +from traitlets import Int, Unicode from typing import Any, Dict, List, Optional, Tuple from ..kernelspec import KernelSpecManager, NoSuchKernel @@ -24,17 +25,21 @@ class SubclassedTestProvisioner(LocalProvisioner): + + config_var_1: int = Int(config=True) + config_var_2: str = Unicode(config=True) + pass class CustomTestProvisioner(KernelProvisionerBase): - def __init__(self, kernel_id: str, kernel_spec: Any, **kwargs): - super().__init__(kernel_id, kernel_spec, **kwargs) - self.process = None - self.pid = None - self.pgid = None - self.connection_info = None + process = None + pid = None + pgid = None + + config_var_1: int = Int(config=True) + config_var_2: str = Unicode(config=True) async def poll(self) -> [int, None]: if self.process: @@ -124,11 +129,12 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: if provisioner: kernel_provisioner = { 'kernel_provisioner': { - 'provisioner_name': provisioner, - 'config': {'config_var_1': 42, 'config_var_2': name} + 'provisioner_name': provisioner } } spec['metadata'].update(kernel_provisioner) + if provisioner != 'LocalProvisioner': + spec['metadata']['kernel_provisioner']['config'] = {'config_var_1': 42, 'config_var_2': name} kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', name) os.makedirs(kernel_dir) @@ -185,7 +191,7 @@ def mock_get_provisioner(name) -> EntryPoint: @pytest.fixture def kpf(monkeypatch): - """Setup the Kernel Provisioner Factory, mocking the entypoint fetch calls.""" + """Setup the Kernel Provisioner Factory, mocking the entrypoint fetch calls.""" monkeypatch.setattr(KernelProvisionerFactory, '_get_all_provisioners', mock_get_all_provisioners) monkeypatch.setattr(KernelProvisionerFactory, '_get_provisioner', mock_get_provisioner) factory = KernelProvisionerFactory.instance() @@ -262,18 +268,34 @@ async def test_custom_lifecycle(self, kpf): assert is_alive is False assert async_km.context.closed + @pytest.mark.asyncio + async def test_default_provisioner_config(self, kpf, all_provisioners): + kpf.default_provisioner_name = 'CustomTestProvisioner' + async_km = AsyncKernelManager(kernel_name='no_provisioner') + await async_km.start_kernel(stdout=PIPE, stderr=PIPE) + is_alive = await async_km.is_alive() + assert is_alive + + assert isinstance(async_km.provisioner, CustomTestProvisioner) + assert async_km.provisioner.config_var_1 == 0 # Not in kernelspec, so default of 0 exists + + await async_km.shutdown_kernel(now=True) + is_alive = await async_km.is_alive() + assert is_alive is False + assert async_km.context.closed + @staticmethod def validate_provisioner(akm: AsyncKernelManager): # Ensure kernel attribute is the provisioner assert akm.kernel is akm.provisioner # Validate provisioner config - if akm.kernel_name == 'no_provisioner': - assert 'config_var_1' not in akm.provisioner.config - assert 'config_var_2' not in akm.provisioner.config + if akm.kernel_name in ['no_provisioner', 'default_provisioner']: + assert not hasattr(akm.provisioner, 'config_var_1') + assert not hasattr(akm.provisioner, 'config_var_2') else: - assert akm.provisioner.config.get('config_var_1') == 42 - assert akm.provisioner.config.get('config_var_2') == akm.kernel_name + assert akm.provisioner.config_var_1 == 42 + assert akm.provisioner.config_var_2 == akm.kernel_name # Validate provisioner class if akm.kernel_name in ['no_provisioner', 'default_provisioner', 'subclassed_provisioner']: From 9856c9fe608319dcf71c3027e3b3660dd5945190 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 18 Mar 2021 06:46:47 -0700 Subject: [PATCH 13/37] Updates stemming from POC with other provisioners --- jupyter_client/connect.py | 13 +++++++ jupyter_client/manager.py | 7 ++-- jupyter_client/provisioning.py | 71 ++++++++++++++++++---------------- setup.py | 2 +- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 4a41af0ca..2ff52660e 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -520,6 +520,19 @@ def load_connection_info(self, info): if 'signature_scheme' in info: self.session.signature_scheme = info['signature_scheme'] + def _force_connection_info(self, info): + """Unconditionally loads connection info from a dict containing connection info. + + Overwrites connection info-based attributes, regardless of their current values + and writes this information to the connection file. + """ + # Reset current ports to 0 and indicate file has not been written to enable override + self._connection_file_written = False + for name in port_names: + setattr(self, name, 0) + self.load_connection_info(info) + self.write_connection_file() + #-------------------------------------------------------------------------- # Creating connected sockets #-------------------------------------------------------------------------- diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 6a91a75d8..2cc9f9581 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -559,7 +559,7 @@ async def pre_start_kernel(self, **kw): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - self.kernel_id = kw.pop('kernel_id', str(uuid.uuid4())) + self.kernel_id = self.kernel_id or kw.pop('kernel_id', str(uuid.uuid4())) self.provisioner = KPF.instance(parent=self.parent).\ create_provisioner_instance(self.kernel_id, self.kernel_spec) @@ -606,14 +606,15 @@ async def _launch_kernel(self, kernel_cmd, **kw): """ kernel_proc, connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) # Provisioner provides the connection information. Load into kernel manager and write file. - self.load_connection_info(connection_info) - self.write_connection_file() + self._force_connection_info(connection_info) return kernel_proc + # TODO - should this be async def? (I'd prefer that provisioner.shutdown_requested() is async def) def request_shutdown(self, restart=False): """Send a shutdown request via control channel """ super().request_shutdown(restart=restart) + self.provisioner.shutdown_requested(restart=restart) async def finish_shutdown(self, waittime=None, pollinterval=0.1, restart=False): """Wait for kernel shutdown, then kill process if it doesn't shutdown. diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 7dd59d5ed..08615a403 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -81,6 +81,15 @@ async def cleanup(self, restart=False) -> None: """ pass + # TODO - update to async def + def shutdown_requested(self, restart=False) -> None: + """Called after KernelManager sends a `shutdown_request` message to kernel. + + This method is optional and is primarily used in scenarios where the provisioner communicates + with a sibling (nanny) process to the kernel. + """ + pass + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """Perform any steps in preparation for kernel process launch. @@ -89,15 +98,10 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: Returns potentially updated kwargs. """ - env = kwargs.pop('env', os.environ).copy() - - env.update(self._apply_env_substitutions(env)) - + env.update(self.__apply_env_substitutions(env)) self._validate_parameters(env, **kwargs) - - self._finalize_env(env) - + self._finalize_env(env) # TODO: Should finalize be called first? kwargs['env'] = env return kwargs @@ -130,10 +134,25 @@ def get_shutdown_wait_time(self, recommended: Optional[float] = 5.0) -> float: """ return recommended - def _apply_env_substitutions(self, substitution_values: Dict[str, str]): - """ Walks env entries in the kernelspec's env stanza and applies possible substitutions from current env - (represented by substitution_values). + def _finalize_env(self, env: Dict[str, str]) -> None: + """Ensures env is appropriate prior to launch. + + Subclasses should be sure to call super()._finalize_env(env) + """ + if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + + def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: + """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" + pass + + def __apply_env_substitutions(self, substitution_values: Dict[str, str]): + """ Walks env entries in the kernelspec's env stanza and applies possible substitutions from current env. + Returns the substituted list of env entries. + Note: This method is private and is not intended to be overridden by provisioners. """ substituted_env = {} if self.kernel_spec: @@ -146,18 +165,6 @@ def _apply_env_substitutions(self, substitution_values: Dict[str, str]): substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) return substituted_env - def _finalize_env(self, env: Dict[str, str]) -> None: - """ Ensures env is appropriate prior to launch. """ - - if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): - # Don't allow PYTHONEXECUTABLE to be passed to kernel process. - # If set, it can bork all the things. - env.pop('PYTHONEXECUTABLE', None) - - def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" - pass - class LocalProvisioner(KernelProvisionerBase): @@ -166,6 +173,7 @@ class LocalProvisioner(KernelProvisionerBase): _exit_future = None pid = None pgid = None + ip = None async def poll(self) -> [int, None]: if self.process: @@ -252,7 +260,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # If we have a kernel_manager pop it out of the args and use it to retain b/c. # This should be considered temporary until a better division of labor can be defined. - km = kwargs.pop('kernel_manager') + km = kwargs.pop('kernel_manager', None) if km: if km.transport == 'tcp' and not is_local_ip(km.ip): raise RuntimeError("Can only launch a kernel on a local interface. " @@ -261,9 +269,6 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: "configured properly. " "Currently valid addresses are: %s" % (km.ip, local_ips()) ) - - # save kwargs for use in restart - km._launch_args = kwargs.copy() # build the Popen cmd extra_arguments = kwargs.pop('extra_arguments', []) @@ -279,7 +284,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: - scrubbed_kwargs = LocalProvisioner.scrub_kwargs(kwargs) + scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) if self.async_subprocess: @@ -296,7 +301,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProv return self, self.connection_info @staticmethod - def scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: """Remove any keyword arguments that Popen does not tolerate.""" keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id', 'kernel_manager'] scrubbed_kwargs = kwargs.copy() @@ -304,17 +309,17 @@ def scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: scrubbed_kwargs.pop(kw, None) return scrubbed_kwargs - def get_provisioner_info(self) -> Dict: + async def get_provisioner_info(self) -> Dict: """Captures the base information necessary for kernel persistence relative to the provisioner. """ - provisioner_info = super(LocalProvisioner, self).get_provisioner_info() + provisioner_info = await super().get_provisioner_info() provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) return provisioner_info - def load_provisioner_info(self, provisioner_info: Dict) -> None: + async def load_provisioner_info(self, provisioner_info: Dict) -> None: """Loads the base information necessary for kernel persistence relative to the provisioner. """ - super(LocalProvisioner, self).load_provisioner_info(provisioner_info) + await super().load_provisioner_info(provisioner_info) self.pid = provisioner_info['pid'] self.pgid = provisioner_info['pgid'] self.ip = provisioner_info['ip'] @@ -333,7 +338,7 @@ class KernelProvisionerFactory(SingletonConfigurable): @default('default_provisioner_name') def default_provisioner_name_default(self): - return os.getenv(self.default_provisioner_name_env, "LocalProvisioner") + return os.getenv(self.default_provisioner_name_env, "local-provisioner") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/setup.py b/setup.py index a69c547e1..8cba2a2d2 100644 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ def run(self): 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_provisioners': [ - 'LocalProvisioner = jupyter_client.provisioning:LocalProvisioner', + 'local-provisioner = jupyter_client.provisioning:LocalProvisioner', ] }, ) From cb68adeeae39f1341660b81f88f10f90732026bc Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 23 Apr 2021 15:07:39 -0700 Subject: [PATCH 14/37] Address pre-commit and mypy issues --- .github/workflows/main.yml | 2 +- jupyter_client/__init__.py | 3 +- jupyter_client/connect.py | 15 +- jupyter_client/launcher.py | 95 +++++----- jupyter_client/manager.py | 28 ++- jupyter_client/provisioning.py | 203 ++++++++++++--------- jupyter_client/tests/conftest.py | 2 +- jupyter_client/tests/test_kernelmanager.py | 1 - jupyter_client/tests/test_provisioning.py | 89 +++++---- setup.py | 2 +- 10 files changed, 256 insertions(+), 184 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fa45a7cd7..1b947eebe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: pip install --upgrade --upgrade-strategy eager --pre -e .[test] pytest-cov codecov 'coverage<5' pip freeze - name: Check types - run: mypy jupyter_client/manager.py jupyter_client/multikernelmanager.py jupyter_client/client.py jupyter_client/blocking/client.py jupyter_client/asynchronous/client.py jupyter_client/channels.py jupyter_client/session.py jupyter_client/adapter.py jupyter_client/connect.py jupyter_client/consoleapp.py jupyter_client/jsonutil.py jupyter_client/kernelapp.py jupyter_client/launcher.py jupyter_client/threaded.py + run: mypy jupyter_client/manager.py jupyter_client/multikernelmanager.py jupyter_client/client.py jupyter_client/blocking/client.py jupyter_client/asynchronous/client.py jupyter_client/channels.py jupyter_client/session.py jupyter_client/adapter.py jupyter_client/connect.py jupyter_client/consoleapp.py jupyter_client/jsonutil.py jupyter_client/kernelapp.py jupyter_client/launcher.py jupyter_client/threaded.py jupyter_client/provisioning.py - name: Run the tests run: py.test --cov jupyter_client -v jupyter_client - name: Code coverage diff --git a/jupyter_client/__init__.py b/jupyter_client/__init__.py index fb9265c8f..1aa6feadb 100644 --- a/jupyter_client/__init__.py +++ b/jupyter_client/__init__.py @@ -13,4 +13,5 @@ from .manager import run_kernel # noqa from .multikernelmanager import AsyncMultiKernelManager # noqa from .multikernelmanager import MultiKernelManager # noqa -from .provisioning import KernelProvisionerBase, LocalProvisioner # noqa +from .provisioning import KernelProvisionerBase # noqa +from .provisioning import LocalProvisioner # noqa diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index f51f4404b..096b1b5bd 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -38,6 +38,9 @@ from .localinterfaces import localhost from .utils import _filefind +# Define custom type for kernel connection info +KernelConnectionInfo = Dict[str, Union[int, str, bytes]] + def write_connection_file( fname: Optional[str] = None, @@ -51,7 +54,7 @@ def write_connection_file( transport: str = "tcp", signature_scheme: str = "hmac-sha256", kernel_name: str = "", -) -> Tuple[str, Dict[str, Union[int, str]]]: +) -> Tuple[str, KernelConnectionInfo]: """Generates a JSON config file, including the selection of random ports. Parameters @@ -139,7 +142,7 @@ def write_connection_file( if hb_port <= 0: hb_port = ports.pop(0) - cfg: Dict[str, Union[int, str]] = dict( + cfg: KernelConnectionInfo = dict( shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, @@ -250,7 +253,7 @@ def find_connection_file( def tunnel_to_kernel( - connection_info: Union[str, Dict[str, Any]], + connection_info: Union[str, KernelConnectionInfo], sshserver: str, sshkey: Optional[str] = None, ) -> Tuple[Any, ...]: @@ -398,7 +401,7 @@ def _session_default(self): # Connection and ipc file management # -------------------------------------------------------------------------- - def get_connection_info(self, session: bool = False) -> Dict[str, Any]: + def get_connection_info(self, session: bool = False) -> KernelConnectionInfo: """Return the connection info as a dict Parameters @@ -543,7 +546,7 @@ def load_connection_file(self, connection_file: Optional[str] = None) -> None: info = json.load(f) self.load_connection_info(info) - def load_connection_info(self, info: Dict[str, Union[int, str]]) -> None: + def load_connection_info(self, info: KernelConnectionInfo) -> None: """Load connection info from a dict containing connection info. Typically this data comes from a connection file @@ -574,7 +577,7 @@ def load_connection_info(self, info: Dict[str, Union[int, str]]) -> None: if "signature_scheme" in info: self.session.signature_scheme = info["signature_scheme"] - def _force_connection_info(self, info: Dict[str, Union[int, str]]) -> None: + def _force_connection_info(self, info: KernelConnectionInfo) -> None: """Unconditionally loads connection info from a dict containing connection info. Overwrites connection info-based attributes, regardless of their current values diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 18d2e0328..529a10f58 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -8,7 +8,6 @@ from subprocess import Popen from typing import Any from typing import Dict -from typing import IO from typing import List from typing import NoReturn from typing import Optional @@ -23,7 +22,7 @@ def launch_kernel( stdin: Optional[int] = None, stdout: Optional[int] = None, stderr: Optional[int] = None, - env: Optional[Dict[str, str]] = None, + env: Optional[Dict[str, Any]] = None, independent: bool = False, cwd: Optional[str] = None, **kw, @@ -60,31 +59,34 @@ def launch_kernel( """ proc = None - kwargs, interrupt_event = _prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, - independent=independent, cwd=cwd, **kw) + kwargs, interrupt_event = _prepare_process_args( + stdin=stdin, stdout=stdout, stderr=stderr, env=env, independent=independent, cwd=cwd, **kw + ) try: # Allow to use ~/ in the command or its arguments - cmd = list(map(os.path.expanduser, cmd)) + cmd = [os.path.expanduser(s) for s in cmd] proc = Popen(cmd, **kwargs) except Exception as exc: - _handle_subprocess_exception(exc, cmd, env, kwargs) + _handle_subprocess_exception(exc, cmd, env, **kwargs) _finish_process_launch(proc, stdin, interrupt_event) return proc -async def async_launch_kernel(cmd: List[str], - stdin: Optional[IO[str]] = None, - stdout: Optional[IO[str]] = None, - stderr: Optional[IO[str]] = None, - env: Optional[Dict[str, str]] = None, - independent: Optional[bool] = False, - cwd: Optional[str] = None, - **kw: Optional[Dict[str, Any]]) -> Union[Popen, asyncio.subprocess.Process]: - """ Launches a localhost kernel, binding to the specified ports using async subprocess. +async def async_launch_kernel( + cmd: List[str], + stdin: Optional[int] = None, + stdout: Optional[int] = None, + stderr: Optional[int] = None, + env: Optional[Dict[str, Any]] = None, + independent: Optional[bool] = False, + cwd: Optional[str] = None, + **kw, +) -> Union[Popen, asyncio.subprocess.Process]: + """Launches a localhost kernel, binding to the specified ports using async subprocess. Parameters ---------- @@ -116,56 +118,56 @@ async def async_launch_kernel(cmd: List[str], """ proc = None - kwargs, interrupt_event = _prepare_process_args(stdin=stdin, stdout=stdout, stderr=stderr, env=env, - independent=independent, cwd=cwd, **kw) + kwargs, interrupt_event = _prepare_process_args( + stdin=stdin, stdout=stdout, stderr=stderr, env=env, independent=independent, cwd=cwd, **kw + ) try: # Allow to use ~/ in the command or its arguments - cmd = list(map(os.path.expanduser, cmd)) + cmd = [os.path.expanduser(s) for s in cmd] if sys.platform == 'win32': - # Windows is still not ready for async subprocs. When the SelectorEventLoop is used (3.6, 3.7), - # a NotImplementedError is raised for _make_subprocess_transport(). When the ProactorEventLoop - # is used (3.8, 3.9), a NotImplementedError is raised for _add_reader(). + # Windows is still not ready for async subprocs. When the SelectorEventLoop is + # used (3.6, 3.7), a NotImplementedError is raised for _make_subprocess_transport(). + # When the ProactorEventLoop is used (3.8, 3.9), a NotImplementedError is raised + # for _add_reader(). proc = Popen(cmd, **kwargs) else: proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) except Exception as exc: - _handle_subprocess_exception(exc, cmd, env, kwargs) + _handle_subprocess_exception(exc, cmd, env, **kwargs) _finish_process_launch(proc, stdin, interrupt_event) return proc -def _handle_subprocess_exception(exc: Exception, - cmd: List[str], - env: Dict[str, Any], - kwargs: Dict[str, Any]) -> NoReturn: - """ Log error message consisting of runtime arguments (sans env) prior to raising the exception. """ +def _handle_subprocess_exception( + exc: Exception, cmd: List[str], env: Optional[Dict[str, Any]], **kwargs +) -> NoReturn: + """ Log error message consisting of runtime arguments prior to raising the exception. """ try: - msg = ( - "Failed to run command:\n{}\n" - " PATH={!r}\n" - " with kwargs:\n{!r}\n" - ) + msg = "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" # exclude environment variables, # which may contain access tokens and the like. without_env = {key: value for key, value in kwargs.items() if key != 'env'} + assert env is not None msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) get_logger().error(msg) finally: raise exc -def _prepare_process_args(stdin: Optional[IO[str]] = None, - stdout: Optional[IO[str]] = None, - stderr: Optional[IO[str]] = None, - env: Optional[Dict[str, str]] = None, - independent: Optional[bool] = False, - cwd: Optional[str] = None, - **kw: Optional[Dict[str, Any]]) -> Tuple[Dict[str, Any], Any]: - """ Prepares for process launch. +def _prepare_process_args( + stdin: Optional[int] = None, + stdout: Optional[int] = None, + stderr: Optional[int] = None, + env: Optional[Dict[str, str]] = None, + independent: Optional[bool] = False, + cwd: Optional[str] = None, + **kw, +) -> Tuple[Dict[str, Any], Any]: + """Prepares for process launch. This consists of getting arguments into a form Popen/Process understands and creating the necessary Windows structures to support interrupts. @@ -273,9 +275,9 @@ def _prepare_process_args(stdin: Optional[IO[str]] = None, return kwargs, interrupt_event -def _finish_process_launch(subprocess: Union[Popen, asyncio.subprocess.Process], - stdin: IO[str], - interrupt_event: Any) -> None: +def _finish_process_launch( + subprocess: Union[Popen, asyncio.subprocess.Process], stdin: Optional[int], interrupt_event: Any +) -> None: """Finishes the process launch by patching the interrupt event (windows) and closing stdin. """ assert subprocess is not None @@ -286,11 +288,8 @@ def _finish_process_launch(subprocess: Union[Popen, asyncio.subprocess.Process], # Clean up pipes created to work around Popen bug. if stdin is None: + assert subprocess.stdin is not None subprocess.stdin.close() - -__all__ = [ - "launch_kernel", - "async_launch_kernel" -] +__all__ = ["launch_kernel", "async_launch_kernel"] diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 65fe8687b..49e52ee89 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -25,16 +25,13 @@ from traitlets.utils.importstring import import_item # type: ignore from .connect import ConnectionFileMixin -from .localinterfaces import is_local_ip -from .localinterfaces import local_ips from .managerabc import KernelManagerABC -from .provisioning import KernelProvisionerFactory as KPF from .provisioning import KernelProvisionerBase +from .provisioning import KernelProvisionerFactory as KPF from .utils import ensure_async from .utils import run_sync from jupyter_client import KernelClient from jupyter_client import kernelspec -from jupyter_client import launch_kernel class _ShutdownStatus(Enum): @@ -89,10 +86,12 @@ def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: kernel_id = None # The kernel provisioner with which the KernelManager is communicating. - # This will generally be a LocalProvisioner instance unless the kernelspec indicates otherwise. - # Note that we use two attributes, kernel and provisioner, that will point at the same provisioner instance. - # kernel will be non-None during the kernel's lifecycle, while provisioner will span that time, being set - # prior to launch and unset following the kernel's termination. + # This will generally be a LocalProvisioner instance unless the kernelspec + # indicates otherwise. + # Note that we use two attributes, kernel and provisioner, both of which + # will point at the same provisioner instance. `kernel` will be non-None + # during the kernel's lifecycle, while `provisioner` will span that time, + # being set prior to launch and unset following the kernel's termination. kernel: t.Optional[KernelProvisionerBase] = None provisioner: t.Optional[KernelProvisionerBase] = None @@ -258,6 +257,7 @@ async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> KernelPro override in a subclass to launch kernel subprocesses differently """ + assert self.provisioner is not None kernel_proc, connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) # Provisioner provides the connection information. Load into kernel manager and write file. self._force_connection_info(connection_info) @@ -291,11 +291,13 @@ async def pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str, t.Any and launching the kernel (e.g. Popen kwargs). """ self.kernel_id = self.kernel_id or kw.pop('kernel_id', str(uuid.uuid4())) - self.provisioner = KPF.instance(parent=self.parent). \ - create_provisioner_instance(self.kernel_id, self.kernel_spec) + self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( + self.kernel_id, self.kernel_spec + ) # save kwargs for use in restart self._launch_args = kw.copy() + assert self.provisioner is not None kw = await self.provisioner.pre_launch(kernel_manager=self, **kw) kernel_cmd = kw.pop('cmd') return kernel_cmd, kw @@ -310,6 +312,7 @@ async def post_start_kernel(self, **kw) -> None: """ self.start_restarter() self._connect_control_socket() + assert self.provisioner is not None await self.provisioner.post_launch(**kw) async def _async_start_kernel(self, **kw): @@ -340,6 +343,7 @@ async def request_shutdown(self, restart: bool = False) -> None: # ensure control socket is connected self._connect_control_socket() self.session.send(self._control_socket, msg) + assert self.provisioner is not None await self.provisioner.shutdown_requested(restart=restart) async def _async_finish_shutdown( @@ -481,6 +485,7 @@ def has_kernel(self) -> bool: async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: + assert self.kernel is not None await self.kernel.terminate(restart=restart) _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm) @@ -491,6 +496,7 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: + assert self.kernel is not None await self.kernel.kill(restart=restart) # Wait until the kernel terminates. @@ -538,6 +544,7 @@ async def _async_signal_kernel(self, signum: int) -> None: only useful on Unix systems. """ if self.has_kernel: + assert self.kernel is not None await self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @@ -547,6 +554,7 @@ async def _async_signal_kernel(self, signum: int) -> None: async def _async_is_alive(self) -> bool: """Is the kernel process still running?""" if self.has_kernel: + assert self.kernel is not None ret = await self.kernel.poll() if ret is None: return True diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 091c9b392..11a6be14e 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -1,47 +1,61 @@ """Kernel Provisioner Classes""" - # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import asyncio import os import signal import sys +from abc import ABC +from abc import ABCMeta +from abc import abstractmethod +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +from entrypoints import EntryPoint # type: ignore +from entrypoints import get_group_all +from entrypoints import get_single +from entrypoints import NoSuchEntryPoint +from traitlets.config import default # type: ignore +from traitlets.config import Instance +from traitlets.config import LoggingConfigurable +from traitlets.config import SingletonConfigurable +from traitlets.config import Unicode -from abc import ABCMeta, ABC, abstractmethod -from entrypoints import EntryPoint, get_group_all, get_single, NoSuchEntryPoint -from typing import Optional, Dict, List, Any, Tuple -from traitlets.config import Config, default, Instance, LoggingConfigurable, SingletonConfigurable, Unicode - -from .connect import write_connection_file from .launcher import async_launch_kernel -from .localinterfaces import is_local_ip, local_ips +from .localinterfaces import is_local_ip +from .localinterfaces import local_ips +from .utils import ensure_async -class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): +class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): # type: ignore pass class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisionerMeta): """Base class defining methods for KernelProvisioner classes. - Theses methods model those of the Subprocess Popen class: - https://docs.python.org/3/library/subprocess.html#popen-objects + Theses methods model those of the Subprocess Popen class: + https://docs.python.org/3/library/subprocess.html#popen-objects """ + # The kernel specification associated with this provisioner kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) kernel_id: str = Unicode(None, allow_none=True) connection_info: dict = {} @abstractmethod - async def poll(self) -> [int, None]: + async def poll(self) -> Optional[int]: """Checks if kernel process is still running. - If running, None is returned, otherwise the process's integer-valued exit code is returned. - """ + If running, None is returned, otherwise the process's integer-valued exit code is returned. + """ pass @abstractmethod - async def wait(self) -> [int, None]: + async def wait(self) -> Optional[int]: """Waits for kernel process to terminate.""" pass @@ -51,41 +65,44 @@ async def send_signal(self, signum: int) -> None: pass @abstractmethod - async def kill(self, restart=False) -> None: - """Kills the kernel process. This is typically accomplished via a SIGKILL signal, which - cannot be caught. + async def kill(self, restart: bool = False) -> None: + """Kills the kernel process. This is typically accomplished via a SIGKILL signal, + which cannot be caught. restart is True if this operation precedes a start launch_kernel request. """ pass @abstractmethod - async def terminate(self, restart=False) -> None: - """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, which - can be caught, allowing the kernel provisioner to perform possible cleanup of resources. + async def terminate(self, restart: bool = False) -> None: + """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, + which can be caught, allowing the kernel provisioner to perform possible cleanup + of resources. restart is True if this operation precedes a start launch_kernel request. """ pass @abstractmethod - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['KernelProvisionerBase', Dict]: + async def launch_kernel( + self, cmd: List[str], **kwargs: Any + ) -> Tuple['KernelProvisionerBase', Dict]: """Launch the kernel process returning the class instance and connection info.""" pass @abstractmethod - async def cleanup(self, restart=False) -> None: + async def cleanup(self, restart: bool = False) -> None: """Cleanup any resources allocated on behalf of the kernel provisioner. restart is True if this operation precedes a start launch_kernel request. """ pass - async def shutdown_requested(self, restart=False) -> None: + async def shutdown_requested(self, restart: bool = False) -> None: """Called after KernelManager sends a `shutdown_request` message to kernel. - This method is optional and is primarily used in scenarios where the provisioner communicates - with a sibling (nanny) process to the kernel. + This method is optional and is primarily used in scenarios where the provisioner + communicates with a sibling (nanny) process to the kernel. """ pass @@ -109,24 +126,22 @@ async def post_launch(self, **kwargs: Any) -> None: """Perform any steps following the kernel process launch.""" pass - async def get_provisioner_info(self) -> Dict: - """Captures the base information necessary for kernel persistence relative to the provisioner. + async def get_provisioner_info(self) -> Dict[str, Any]: + """Captures the base information necessary for persistence relative to this instance. - The superclass method must always be called first to ensure proper ordering. Since this is the - most base class, no call to `super()` is necessary. + The superclass method must always be called first to ensure proper ordering. """ - provisioner_info = {} + provisioner_info: Dict[str, Any] = {} return provisioner_info async def load_provisioner_info(self, provisioner_info: Dict) -> None: - """Loads the base information necessary for kernel persistence relative to the provisioner. + """Loads the base information necessary for persistence relative to this instance. - The superclass method must always be called first to ensure proper ordering. Since this is the - most base class, no call to `super()` is necessary. + The superclass method must always be called first to ensure proper ordering. """ pass - def get_shutdown_wait_time(self, recommended: Optional[float] = 5.0) -> float: + def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: """Returns the time allowed for a complete shutdown. This may vary by provisioner. The recommended value will typically be what is configured in the kernel manager. @@ -144,18 +159,19 @@ def _finalize_env(self, env: Dict[str, str]) -> None: env.pop('PYTHONEXECUTABLE', None) def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel specification.""" + """Future: Validates that launch parameters adhere to schema specified in kernel spec.""" pass def __apply_env_substitutions(self, substitution_values: Dict[str, str]): - """ Walks env entries in the kernelspec's env stanza and applies possible substitutions from current env. + """Walks entries in the kernelspec's env stanza and applies substitutions from current env. - Returns the substituted list of env entries. - Note: This method is private and is not intended to be overridden by provisioners. + Returns the substituted list of env entries. + Note: This method is private and is not intended to be overridden by provisioners. """ substituted_env = {} if self.kernel_spec: from string import Template + # For each templated env entry, fill any templated references # matching names of env variables with those values and build # new dict with substitutions. @@ -174,20 +190,23 @@ class LocalProvisioner(KernelProvisionerBase): pgid = None ip = None - async def poll(self) -> [int, None]: + async def poll(self) -> Optional[int]: + ret = None if self.process: if self.async_subprocess: if not self._exit_future.done(): # adhere to process returncode - return None + ret = None + else: + ret = self._exit_future.result() else: - if self.process.poll() is None: - return None - return False + ret = self.process.poll() + return ret - async def wait(self) -> [int, None]: + async def wait(self) -> Optional[int]: + ret = None if self.process: if self.async_subprocess: - await self.process.wait() + ret = await self.process.wait() else: # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete @@ -197,8 +216,9 @@ async def wait(self) -> [int, None]: await asyncio.sleep(0.1) # Process is no longer alive, wait and clear - self.process.wait() + ret = self.process.wait() self.process = None + return ret async def send_signal(self, signum: int) -> None: """Sends a signal to the process group of the kernel (this @@ -212,6 +232,7 @@ async def send_signal(self, signum: int) -> None: if self.process: if signum == signal.SIGINT and sys.platform == 'win32': from .win_interrupt import send_interrupt + send_interrupt(self.process.win32_interrupt_event) return @@ -224,7 +245,7 @@ async def send_signal(self, signum: int) -> None: pass return self.process.send_signal(signum) - async def kill(self, restart=False) -> None: + async def kill(self, restart: bool = False) -> None: if self.process: try: self.process.kill() @@ -238,14 +259,15 @@ async def kill(self, restart=False) -> None: # terminated. Ignore it. else: from errno import ESRCH + if e.errno != ESRCH: raise - async def terminate(self, restart=False) -> None: + async def terminate(self, restart: bool = False) -> None: if self.process: return self.process.terminate() - async def cleanup(self, restart=False) -> None: + async def cleanup(self, restart: bool = False) -> None: pass async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: @@ -262,20 +284,23 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: km = kwargs.pop('kernel_manager', None) if km: if km.transport == 'tcp' and not is_local_ip(km.ip): - raise RuntimeError("Can only launch a kernel on a local interface. " - "This one is not: %s." - "Make sure that the '*_address' attributes are " - "configured properly. " - "Currently valid addresses are: %s" % (km.ip, local_ips()) - ) + raise RuntimeError( + "Can only launch a kernel on a local interface. " + "This one is not: %s." + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % (km.ip, local_ips()) + ) # build the Popen cmd extra_arguments = kwargs.pop('extra_arguments', []) # write connection file / get default ports - km.write_connection_file() # TODO - this will need to change when handshake pattern is adopted + km.write_connection_file() # TODO - change when handshake pattern is adopted self.connection_info = km.get_connection_info() - kernel_cmd = km.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c + kernel_cmd = km.format_kernel_cmd( + extra_arguments=extra_arguments + ) # This needs to remain here for b/c else: extra_arguments = kwargs.pop('extra_arguments', []) kernel_cmd = self.kernel_spec.argv + extra_arguments @@ -287,7 +312,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProv self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) if self.async_subprocess: - self._exit_future = asyncio.ensure_future(self.process.wait()) + self._exit_future = asyncio.ensure_future(ensure_async(self.process.wait())) pgid = None if hasattr(os, "getpgid"): try: @@ -309,15 +334,13 @@ def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: return scrubbed_kwargs async def get_provisioner_info(self) -> Dict: - """Captures the base information necessary for kernel persistence relative to the provisioner. - """ + """Captures the base information necessary for persistence relative to this instance.""" provisioner_info = await super().get_provisioner_info() provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) return provisioner_info async def load_provisioner_info(self, provisioner_info: Dict) -> None: - """Loads the base information necessary for kernel persistence relative to the provisioner. - """ + """Loads the base information necessary for persistence relative to this instance.""" await super().load_provisioner_info(provisioner_info) self.pid = provisioner_info['pid'] self.pgid = provisioner_info['pgid'] @@ -325,15 +348,17 @@ async def load_provisioner_info(self, provisioner_info: Dict) -> None: class KernelProvisionerFactory(SingletonConfigurable): - """KernelProvisionerFactory is responsible for validating and initializing provisioner instances.""" + """KernelProvisionerFactory is responsible for creating provisioner instances.""" GROUP_NAME = 'jupyter_client.kernel_provisioners' provisioners: Dict[str, EntryPoint] = {} default_provisioner_name_env = "JUPYTER_DEFAULT_PROVISIONER_NAME" - default_provisioner_name = Unicode(config=True, - help="""Indicates the name of the provisioner to use when no kernel_provisioner - entry is present in the kernelspec.""") + default_provisioner_name = Unicode( + config=True, + help="""Indicates the name of the provisioner to use when no kernel_provisioner + entry is present in the kernelspec.""", + ) @default('default_provisioner_name') def default_provisioner_name_default(self): @@ -354,7 +379,7 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: """ is_available: bool = True provisioner_cfg = self._get_provisioner_config(kernel_spec) - provisioner_name = provisioner_cfg.get('provisioner_name') + provisioner_name = str(provisioner_cfg.get('provisioner_name')) if provisioner_name not in self.provisioners: try: ep = KernelProvisionerFactory._get_provisioner(provisioner_name) @@ -362,11 +387,15 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: except NoSuchEntryPoint: is_available = False self.log.warning( - f"Kernel '{kernel_name}' is referencing a kernel provisioner ('{provisioner_name}') " - f"that is not available. Ensure the appropriate package has been installed and retry.") + f"Kernel '{kernel_name}' is referencing a kernel provisioner " + f"('{provisioner_name}') that is not available. Ensure the " + f"appropriate package has been installed and retry." + ) return is_available - def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> KernelProvisionerBase: + def create_provisioner_instance( + self, kernel_id: str, kernel_spec: Any + ) -> KernelProvisionerBase: """ Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. If one exists, it instantiates an instance. If a kernel provisioner is not @@ -379,23 +408,27 @@ def create_provisioner_instance(self, kernel_id: str, kernel_spec: Any) -> Kerne provisioner_cfg = self._get_provisioner_config(kernel_spec) provisioner_name = provisioner_cfg.get('provisioner_name') if provisioner_name not in self.provisioners: - raise ModuleNotFoundError(f"Kernel provisioner '{provisioner_name}' has not been registered.") - - self.log.debug(f"Instantiating kernel '{kernel_spec.display_name}' with " - f"kernel provisioner: {provisioner_name}") + raise ModuleNotFoundError( + f"Kernel provisioner '{provisioner_name}' has not been registered." + ) + + self.log.debug( + f"Instantiating kernel '{kernel_spec.display_name}' with " + f"kernel provisioner: {provisioner_name}" + ) provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class(kernel_id=kernel_id, - kernel_spec=kernel_spec, - parent=self.parent, - **provisioner_config) + return provisioner_class( + kernel_id=kernel_id, kernel_spec=kernel_spec, parent=self.parent, **provisioner_config + ) def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: """ Return the kernel_provisioner stanza from the kernel_spec. Checks the kernel_spec's metadata dictionary for a kernel_provisioner entry. - If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER and returned. + If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER + and returned. Parameters ---------- @@ -405,13 +438,15 @@ def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: Returns ------- dict - The provisioner portion of the kernel_spec. If one does not exist, it will contain the default - information. If no `config` sub-dictionary exists, an empty `config` dictionary will be added. - + The provisioner portion of the kernel_spec. If one does not exist, it will contain + the default information. If no `config` sub-dictionary exists, an empty `config` + dictionary will be added. """ env_provisioner = kernel_spec.metadata.get('kernel_provisioner', {}) if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default - if 'config' not in env_provisioner: # if provisioner_name, but no config stanza, add one + if ( + 'config' not in env_provisioner + ): # if provisioner_name, but no config stanza, add one env_provisioner.update({"config": {}}) return env_provisioner # Return what we found (plus config stanza if necessary) return {"provisioner_name": self.default_provisioner_name, "config": {}} diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index eedbbb39a..8f9ad7378 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -3,8 +3,8 @@ import sys import pytest - from jupyter_core import paths + from .utils import test_env pjoin = os.path.join diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index 563acfa07..d2c0b1550 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -21,7 +21,6 @@ from ..manager import start_new_kernel from .utils import AsyncKMSubclass from .utils import SyncKMSubclass -from .utils import test_env from jupyter_client import AsyncKernelManager from jupyter_client import KernelManager diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index b27ad1eee..e9a0b92b1 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -1,25 +1,32 @@ """Test Provisionering""" - # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - import asyncio import json import os -import pytest import signal import sys +from subprocess import PIPE +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple -from entrypoints import EntryPoint, NoSuchEntryPoint +import pytest +from entrypoints import EntryPoint +from entrypoints import NoSuchEntryPoint from jupyter_core import paths -from subprocess import PIPE -from traitlets import Int, Unicode -from typing import Any, Dict, List, Optional, Tuple +from traitlets import Int +from traitlets import Unicode -from ..kernelspec import KernelSpecManager, NoSuchKernel +from ..kernelspec import KernelSpecManager +from ..kernelspec import NoSuchKernel from ..launcher import launch_kernel from ..manager import AsyncKernelManager -from ..provisioning import KernelProvisionerBase, LocalProvisioner, KernelProvisionerFactory +from ..provisioning import KernelProvisionerBase +from ..provisioning import KernelProvisionerFactory +from ..provisioning import LocalProvisioner pjoin = os.path.join @@ -59,6 +66,7 @@ async def send_signal(self, signum: int) -> None: if self.process: if signum == signal.SIGINT and sys.platform == 'win32': from ..win_interrupt import send_interrupt + send_interrupt(self.process.win32_interrupt_event) return @@ -91,11 +99,15 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: km.write_connection_file() self.connection_info = km.get_connection_info() - kernel_cmd = km.format_kernel_cmd(extra_arguments=extra_arguments) # This needs to remain here for b/c + kernel_cmd = km.format_kernel_cmd( + extra_arguments=extra_arguments + ) # This needs to remain here for b/c return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['CustomTestProvisioner', Dict]: + async def launch_kernel( + self, cmd: List[str], **kwargs: Any + ) -> Tuple['CustomTestProvisioner', Dict]: scrubbed_kwargs = kwargs self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -118,23 +130,27 @@ class NewTestProvisioner(CustomTestProvisioner): def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: - spec = {'argv': [sys.executable, - '-m', 'jupyter_client.tests.signalkernel', - '-f', '{connection_file}'], - 'display_name': f"Signal Test Kernel w {provisioner}", - 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, - 'metadata': {} - } + spec = { + 'argv': [ + sys.executable, + '-m', + 'jupyter_client.tests.signalkernel', + '-f', + '{connection_file}', + ], + 'display_name': f"Signal Test Kernel w {provisioner}", + 'env': {'TEST_VARS': '${TEST_VARS}:test_var_2'}, + 'metadata': {}, + } if provisioner: - kernel_provisioner = { - 'kernel_provisioner': { - 'provisioner_name': provisioner - } - } + kernel_provisioner = {'kernel_provisioner': {'provisioner_name': provisioner}} spec['metadata'].update(kernel_provisioner) if provisioner != 'LocalProvisioner': - spec['metadata']['kernel_provisioner']['config'] = {'config_var_1': 42, 'config_var_2': name} + spec['metadata']['kernel_provisioner']['config'] = { + 'config_var_1': 42, + 'config_var_2': name, + } kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', name) os.makedirs(kernel_dir) @@ -159,8 +175,15 @@ def all_provisioners(): custom_provisioner() -@pytest.fixture(params=['no_provisioner', 'default_provisioner', 'missing_provisioner', - 'custom_provisioner', 'subclassed_provisioner']) +@pytest.fixture( + params=[ + 'no_provisioner', + 'default_provisioner', + 'missing_provisioner', + 'custom_provisioner', + 'subclassed_provisioner', + ] +) def akm(request, all_provisioners): return AsyncKernelManager(kernel_name=request.param) @@ -168,7 +191,7 @@ def akm(request, all_provisioners): initial_provisioner_map = { 'LocalProvisioner': 'jupyter_client.provisioning', 'SubclassedTestProvisioner': 'jupyter_client.tests.test_provisioning', - 'CustomTestProvisioner': 'jupyter_client.tests.test_provisioning' + 'CustomTestProvisioner': 'jupyter_client.tests.test_provisioning', } @@ -181,7 +204,9 @@ def mock_get_all_provisioners() -> List[EntryPoint]: def mock_get_provisioner(name) -> EntryPoint: if name == 'NewTestProvisioner': - return EntryPoint('NewTestProvisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner') + return EntryPoint( + 'NewTestProvisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner' + ) if name in initial_provisioner_map: return EntryPoint(name, initial_provisioner_map[name], name) @@ -192,7 +217,9 @@ def mock_get_provisioner(name) -> EntryPoint: @pytest.fixture def kpf(monkeypatch): """Setup the Kernel Provisioner Factory, mocking the entrypoint fetch calls.""" - monkeypatch.setattr(KernelProvisionerFactory, '_get_all_provisioners', mock_get_all_provisioners) + monkeypatch.setattr( + KernelProvisionerFactory, '_get_all_provisioners', mock_get_all_provisioners + ) monkeypatch.setattr(KernelProvisionerFactory, '_get_provisioner', mock_get_provisioner) factory = KernelProvisionerFactory.instance() return factory @@ -203,7 +230,8 @@ def test_find_all_specs(self, kpf, all_provisioners): ksm = KernelSpecManager() kernels = ksm.get_all_specs() - # Ensure specs for initial provisiones exist and missing_provisioner (and new_provisioner) don't + # Ensure specs for initial provisioners exist, + # and missing_provisioner & new_provisioner don't assert 'no_provisioner' in kernels assert 'default_provisioner' in kernels assert 'subclassed_provisioner' in kernels @@ -224,7 +252,6 @@ def test_get_new(self, kpf): class TestRuntime: - async def akm_test(self, kernel_mgr): """Starts a kernel, validates the associated provisioner's config, shuts down kernel """ diff --git a/setup.py b/setup.py index 3854dbf4d..1b035bd2b 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ def run(self): ], 'jupyter_client.kernel_provisioners': [ 'local-provisioner = jupyter_client.provisioning:LocalProvisioner', - ] + ], }, ) From 654549d36fd573c0bc8db979457cb76ed8e0e2a9 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sun, 25 Apr 2021 07:25:01 -0700 Subject: [PATCH 15/37] Get tests passing again --- jupyter_client/manager.py | 13 ++++++++----- jupyter_client/provisioning.py | 2 +- jupyter_client/tests/utils.py | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 49e52ee89..810d57dd1 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -347,7 +347,7 @@ async def request_shutdown(self, restart: bool = False) -> None: await self.provisioner.shutdown_requested(restart=restart) async def _async_finish_shutdown( - self, waittime: t.Optional[float] = None, pollinterval: float = 0.1, restart: bool = False + self, waittime: t.Optional[float] = None, pollinterval: float = 0.1, restart: t.Optional[bool] = False ) -> None: """Wait for kernel shutdown, then kill process if it doesn't shutdown. @@ -375,7 +375,7 @@ async def _async_finish_shutdown( except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, killing") self._shutdown_status = _ShutdownStatus.SigkillRequest - await ensure_async(self._kill_kernel()) + await ensure_async(self._kill_kernel(restart=restart)) else: # Process is no longer alive, wait and clear if self.kernel is not None: @@ -384,7 +384,7 @@ async def _async_finish_shutdown( finish_shutdown = run_sync(_async_finish_shutdown) - async def cleanup_resources(self, restart: bool = False) -> None: + async def _async_cleanup_resources(self, restart: bool = False) -> None: """Clean up resources when the kernel is shut down""" if not restart: self.cleanup_connection_file() @@ -400,6 +400,8 @@ async def cleanup_resources(self, restart: bool = False) -> None: await self.provisioner.cleanup(restart=restart) self.provisioner = self.kernel = None + cleanup_resources = run_sync(_async_cleanup_resources) + async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. @@ -433,7 +435,7 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) # most 1s, checking every 0.1s. await ensure_async(self.finish_shutdown(restart=restart)) - await self.cleanup_resources(restart=restart) + await self._async_cleanup_resources(restart=restart) shutdown_kernel = run_sync(_async_shutdown_kernel) @@ -524,7 +526,7 @@ async def _async_interrupt_kernel(self) -> None: assert self.kernel_spec is not None interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == "signal": - await self.signal_kernel(signal.SIGINT) + await self._async_signal_kernel(signal.SIGINT) elif interrupt_mode == "message": msg = self.session.msg("interrupt_request", content={}) @@ -581,6 +583,7 @@ class AsyncKernelManager(KernelManager): _launch_kernel = KernelManager._async_launch_kernel start_kernel = KernelManager._async_start_kernel finish_shutdown = KernelManager._async_finish_shutdown + cleanup_resources = KernelManager._async_cleanup_resources shutdown_kernel = KernelManager._async_shutdown_kernel restart_kernel = KernelManager._async_restart_kernel _send_kernel_sigterm = KernelManager._async_send_kernel_sigterm diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 11a6be14e..830c1a6d2 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -362,7 +362,7 @@ class KernelProvisionerFactory(SingletonConfigurable): @default('default_provisioner_name') def default_provisioner_name_default(self): - return os.getenv(self.default_provisioner_name_env, "LocalProvisioner") + return os.getenv(self.default_provisioner_name_env, "local-provisioner") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index 252c368de..a1ef0a6cf 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -130,7 +130,7 @@ def request_shutdown(self, restart=False): """ Record call and defer to superclass """ @subclass_recorder - def finish_shutdown(self, waittime=None, pollinterval=0.1): + def finish_shutdown(self, waittime=None, pollinterval=0.1, restart=False): """ Record call and defer to superclass """ @subclass_recorder @@ -191,7 +191,7 @@ def request_shutdown(self, kernel_id, restart=False): """ Record call and defer to superclass """ @subclass_recorder - def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): + def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1, restart=False): """ Record call and defer to superclass """ @subclass_recorder From 6097f8a8a702d00c7ac00cf05fab10ff71042389 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Mon, 26 Apr 2021 16:11:45 -0700 Subject: [PATCH 16/37] Use naming convention for provisioners --- jupyter_client/provisioning.py | 2 +- jupyter_client/tests/test_provisioning.py | 28 +++++++++++------------ setup.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 830c1a6d2..084437c04 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -362,7 +362,7 @@ class KernelProvisionerFactory(SingletonConfigurable): @default('default_provisioner_name') def default_provisioner_name_default(self): - return os.getenv(self.default_provisioner_name_env, "local-provisioner") + return os.getenv(self.default_provisioner_name_env, "Local-Provisioner") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index e9a0b92b1..4b5dd5f35 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -146,7 +146,7 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: if provisioner: kernel_provisioner = {'kernel_provisioner': {'provisioner_name': provisioner}} spec['metadata'].update(kernel_provisioner) - if provisioner != 'LocalProvisioner': + if provisioner != 'Local-Provisioner': spec['metadata']['kernel_provisioner']['config'] = { 'config_var_1': 42, 'config_var_2': name, @@ -159,19 +159,19 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: def new_provisioner(): - build_kernelspec('new_provisioner', 'NewTestProvisioner') + build_kernelspec('new_provisioner', 'New-Test-Provisioner') def custom_provisioner(): - build_kernelspec('custom_provisioner', 'CustomTestProvisioner') + build_kernelspec('custom_provisioner', 'Custom-Test-Provisioner') @pytest.fixture def all_provisioners(): build_kernelspec('no_provisioner') - build_kernelspec('missing_provisioner', 'MissingProvisioner') - build_kernelspec('default_provisioner', 'LocalProvisioner') - build_kernelspec('subclassed_provisioner', 'SubclassedTestProvisioner') + build_kernelspec('missing_provisioner', 'Missing-Provisioner') + build_kernelspec('default_provisioner', 'Local-Provisioner') + build_kernelspec('subclassed_provisioner', 'Subclassed-Test-Provisioner') custom_provisioner() @@ -189,23 +189,23 @@ def akm(request, all_provisioners): initial_provisioner_map = { - 'LocalProvisioner': 'jupyter_client.provisioning', - 'SubclassedTestProvisioner': 'jupyter_client.tests.test_provisioning', - 'CustomTestProvisioner': 'jupyter_client.tests.test_provisioning', + 'Local-Provisioner': ('jupyter_client.provisioning', 'LocalProvisioner'), + 'Subclassed-Test-Provisioner': ('jupyter_client.tests.test_provisioning', 'SubclassedTestProvisioner'), + 'Custom-Test-Provisioner': ('jupyter_client.tests.test_provisioning', 'CustomTestProvisioner'), } def mock_get_all_provisioners() -> List[EntryPoint]: result = [] for name, epstr in initial_provisioner_map.items(): - result.append(EntryPoint(name, epstr, name)) + result.append(EntryPoint(name, epstr[0], epstr[1])) return result def mock_get_provisioner(name) -> EntryPoint: - if name == 'NewTestProvisioner': + if name == 'New-Test-Provisioner': return EntryPoint( - 'NewTestProvisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner' + 'New-Test-Provisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner' ) if name in initial_provisioner_map: @@ -248,7 +248,7 @@ def test_get_new(self, kpf): new_provisioner() # Introduce provisioner after initialization of KPF ksm = KernelSpecManager() kernel = ksm.get_kernel_spec('new_provisioner') - assert 'NewTestProvisioner' == kernel.metadata['kernel_provisioner']['provisioner_name'] + assert 'New-Test-Provisioner' == kernel.metadata['kernel_provisioner']['provisioner_name'] class TestRuntime: @@ -297,7 +297,7 @@ async def test_custom_lifecycle(self, kpf): @pytest.mark.asyncio async def test_default_provisioner_config(self, kpf, all_provisioners): - kpf.default_provisioner_name = 'CustomTestProvisioner' + kpf.default_provisioner_name = 'Custom-Test-Provisioner' async_km = AsyncKernelManager(kernel_name='no_provisioner') await async_km.start_kernel(stdout=PIPE, stderr=PIPE) is_alive = await async_km.is_alive() diff --git a/setup.py b/setup.py index 1b035bd2b..bc76a4880 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ def run(self): 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_provisioners': [ - 'local-provisioner = jupyter_client.provisioning:LocalProvisioner', + 'Local-Provisioner = jupyter_client.provisioning:LocalProvisioner', ], }, ) From c2c3ccb42b48c96c8f1753d3c346fe1f1df48edd Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Mon, 26 Apr 2021 19:06:46 -0700 Subject: [PATCH 17/37] Fix tests, remove async_generator dependency --- jupyter_client/launcher.py | 2 +- jupyter_client/manager.py | 9 ++++++--- jupyter_client/provisioning.py | 8 ++++++-- jupyter_client/tests/signalkernel.py | 6 ++---- jupyter_client/tests/test_kernelmanager.py | 14 ++++++++------ jupyter_client/tests/test_provisioning.py | 7 +++++-- setup.py | 1 - 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 529a10f58..02e89b6a4 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -284,7 +284,7 @@ def _finish_process_launch( if sys.platform == "win32": # Attach the interrupt event to the Popen objet so it can be used later. - subprocess.win32_interrupt_event = interrupt_event + subprocess.win32_interrupt_event = interrupt_event # type: ignore # Clean up pipes created to work around Popen bug. if stdin is None: diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 810d57dd1..2d34eff68 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -347,7 +347,10 @@ async def request_shutdown(self, restart: bool = False) -> None: await self.provisioner.shutdown_requested(restart=restart) async def _async_finish_shutdown( - self, waittime: t.Optional[float] = None, pollinterval: float = 0.1, restart: t.Optional[bool] = False + self, + waittime: t.Optional[float] = None, + pollinterval: float = 0.1, + restart: t.Optional[bool] = False, ) -> None: """Wait for kernel shutdown, then kill process if it doesn't shutdown. @@ -435,7 +438,7 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) # most 1s, checking every 0.1s. await ensure_async(self.finish_shutdown(restart=restart)) - await self._async_cleanup_resources(restart=restart) + await ensure_async(self.cleanup_resources(restart=restart)) shutdown_kernel = run_sync(_async_shutdown_kernel) @@ -512,7 +515,7 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: # Process is no longer alive, wait and clear if self.kernel is not None: await self.kernel.wait() - self.kernel = None + self.kernel = None _kill_kernel = run_sync(_async_kill_kernel) diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index 084437c04..cfc1604af 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -243,7 +243,11 @@ async def send_signal(self, signum: int) -> None: return except OSError: pass - return self.process.send_signal(signum) + try: + self.process.send_signal(signum) + except OSError: + pass + return async def kill(self, restart: bool = False) -> None: if self.process: @@ -316,7 +320,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProv pgid = None if hasattr(os, "getpgid"): try: - pgid = os.getpgid(self.process.pid) + pgid = os.getpgid(self.process.pid) # type: ignore except OSError: pass diff --git a/jupyter_client/tests/signalkernel.py b/jupyter_client/tests/signalkernel.py index e26731ff8..52679f6d5 100644 --- a/jupyter_client/tests/signalkernel.py +++ b/jupyter_client/tests/signalkernel.py @@ -10,7 +10,6 @@ from ipykernel.displayhook import ZMQDisplayHook from ipykernel.kernelapp import IPKernelApp from ipykernel.kernelbase import Kernel -from tornado.web import gen class SignalTestKernel(Kernel): @@ -28,10 +27,9 @@ def __init__(self, **kwargs): if os.environ.get("NO_SIGTERM_REPLY", None) == "1": signal.signal(signal.SIGTERM, signal.SIG_IGN) - @gen.coroutine - def shutdown_request(self, stream, ident, parent): + async def shutdown_request(self, stream, ident, parent): if os.environ.get("NO_SHUTDOWN_REPLY") != "1": - yield gen.maybe_future(super().shutdown_request(stream, ident, parent)) + await super().shutdown_request(stream, ident, parent) def do_execute( self, code, silent, store_history=True, user_expressions=None, allow_stdin=False diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index d2c0b1550..f90fbc6fd 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -11,8 +11,6 @@ from subprocess import PIPE import pytest -from async_generator import async_generator -from async_generator import yield_ from jupyter_core import paths from traitlets.config.loader import Config @@ -126,11 +124,9 @@ def async_km_subclass(config): @pytest.fixture -@async_generator # This is only necessary while Python 3.5 is support afterwhich both it and -# yield_() can be removed async def start_async_kernel(): km, kc = await start_new_async_kernel(kernel_name="signaltest") - await yield_((km, kc)) + yield km, kc kc.stop_channels() await km.shutdown_kernel() assert km.context.closed @@ -157,6 +153,9 @@ class TestKernelManagerShutDownGracefully: @pytest.mark.skipif(sys.platform == "win32", reason="Windows doesn't support signals") @pytest.mark.parametrize(*parameters) def test_signal_kernel_subprocesses(self, name, install, expected): + # ipykernel doesn't support 3.6 and this test uses async shutdown_request + if expected == _ShutdownStatus.ShutdownRequest and sys.version_info < (3, 7): + pytest.skip() install() km, kc = start_new_kernel(kernel_name=name) assert km._shutdown_status == _ShutdownStatus.Unset @@ -171,6 +170,9 @@ def test_signal_kernel_subprocesses(self, name, install, expected): @pytest.mark.skipif(sys.platform == "win32", reason="Windows doesn't support signals") @pytest.mark.parametrize(*parameters) async def test_async_signal_kernel_subprocesses(self, name, install, expected): + # ipykernel doesn't support 3.6 and this test uses async shutdown_request + if expected == _ShutdownStatus.ShutdownRequest and sys.version_info < (3, 7): + pytest.skip() install() km, kc = await start_new_async_kernel(kernel_name=name) assert km._shutdown_status == _ShutdownStatus.Unset @@ -342,7 +344,7 @@ def test_start_sequence_kernels(self, config, install_kernel): self._run_signaltest_lifecycle(config) self._run_signaltest_lifecycle(config) - @pytest.mark.timeout(TIMEOUT) + @pytest.mark.timeout(TIMEOUT + 10) def test_start_parallel_thread_kernels(self, config, install_kernel): if config.KernelManager.transport == "ipc": # FIXME pytest.skip("IPC transport is currently not working for this test!") diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 4b5dd5f35..2f72ec979 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -190,7 +190,10 @@ def akm(request, all_provisioners): initial_provisioner_map = { 'Local-Provisioner': ('jupyter_client.provisioning', 'LocalProvisioner'), - 'Subclassed-Test-Provisioner': ('jupyter_client.tests.test_provisioning', 'SubclassedTestProvisioner'), + 'Subclassed-Test-Provisioner': ( + 'jupyter_client.tests.test_provisioning', + 'SubclassedTestProvisioner', + ), 'Custom-Test-Provisioner': ('jupyter_client.tests.test_provisioning', 'CustomTestProvisioner'), } @@ -209,7 +212,7 @@ def mock_get_provisioner(name) -> EntryPoint: ) if name in initial_provisioner_map: - return EntryPoint(name, initial_provisioner_map[name], name) + return EntryPoint(name, initial_provisioner_map[name][0], initial_provisioner_map[name][1]) raise NoSuchEntryPoint(KernelProvisionerFactory.GROUP_NAME, name) diff --git a/setup.py b/setup.py index bc76a4880..272bf91e8 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ def run(self): python_requires='>=3.6.1', extras_require={ 'test': [ - 'async_generator', 'ipykernel', 'ipython', 'jedi<0.18; python_version<="3.6"', From ccce1d01f9e928228a2391d4e4e9e60b1dd234dc Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Wed, 28 Apr 2021 07:50:53 -0700 Subject: [PATCH 18/37] Use lowercase convention for provisioner names --- jupyter_client/provisioning.py | 2 +- jupyter_client/tests/test_provisioning.py | 26 +++++++++++------------ setup.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py index cfc1604af..cedca5ebd 100644 --- a/jupyter_client/provisioning.py +++ b/jupyter_client/provisioning.py @@ -366,7 +366,7 @@ class KernelProvisionerFactory(SingletonConfigurable): @default('default_provisioner_name') def default_provisioner_name_default(self): - return os.getenv(self.default_provisioner_name_env, "Local-Provisioner") + return os.getenv(self.default_provisioner_name_env, "local-provisioner") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 2f72ec979..24d108a8a 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -146,7 +146,7 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: if provisioner: kernel_provisioner = {'kernel_provisioner': {'provisioner_name': provisioner}} spec['metadata'].update(kernel_provisioner) - if provisioner != 'Local-Provisioner': + if provisioner != 'local-provisioner': spec['metadata']['kernel_provisioner']['config'] = { 'config_var_1': 42, 'config_var_2': name, @@ -159,19 +159,19 @@ def build_kernelspec(name: str, provisioner: Optional[str] = None) -> None: def new_provisioner(): - build_kernelspec('new_provisioner', 'New-Test-Provisioner') + build_kernelspec('new_provisioner', 'new-test-provisioner') def custom_provisioner(): - build_kernelspec('custom_provisioner', 'Custom-Test-Provisioner') + build_kernelspec('custom_provisioner', 'custom-test-provisioner') @pytest.fixture def all_provisioners(): build_kernelspec('no_provisioner') - build_kernelspec('missing_provisioner', 'Missing-Provisioner') - build_kernelspec('default_provisioner', 'Local-Provisioner') - build_kernelspec('subclassed_provisioner', 'Subclassed-Test-Provisioner') + build_kernelspec('missing_provisioner', 'missing-provisioner') + build_kernelspec('default_provisioner', 'local-provisioner') + build_kernelspec('subclassed_provisioner', 'subclassed-test-provisioner') custom_provisioner() @@ -189,12 +189,12 @@ def akm(request, all_provisioners): initial_provisioner_map = { - 'Local-Provisioner': ('jupyter_client.provisioning', 'LocalProvisioner'), - 'Subclassed-Test-Provisioner': ( + 'local-provisioner': ('jupyter_client.provisioning', 'LocalProvisioner'), + 'subclassed-test-provisioner': ( 'jupyter_client.tests.test_provisioning', 'SubclassedTestProvisioner', ), - 'Custom-Test-Provisioner': ('jupyter_client.tests.test_provisioning', 'CustomTestProvisioner'), + 'custom-test-provisioner': ('jupyter_client.tests.test_provisioning', 'CustomTestProvisioner'), } @@ -206,9 +206,9 @@ def mock_get_all_provisioners() -> List[EntryPoint]: def mock_get_provisioner(name) -> EntryPoint: - if name == 'New-Test-Provisioner': + if name == 'new-test-provisioner': return EntryPoint( - 'New-Test-Provisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner' + 'new-test-provisioner', 'jupyter_client.tests.test_provisioning', 'NewTestProvisioner' ) if name in initial_provisioner_map: @@ -251,7 +251,7 @@ def test_get_new(self, kpf): new_provisioner() # Introduce provisioner after initialization of KPF ksm = KernelSpecManager() kernel = ksm.get_kernel_spec('new_provisioner') - assert 'New-Test-Provisioner' == kernel.metadata['kernel_provisioner']['provisioner_name'] + assert 'new-test-provisioner' == kernel.metadata['kernel_provisioner']['provisioner_name'] class TestRuntime: @@ -300,7 +300,7 @@ async def test_custom_lifecycle(self, kpf): @pytest.mark.asyncio async def test_default_provisioner_config(self, kpf, all_provisioners): - kpf.default_provisioner_name = 'Custom-Test-Provisioner' + kpf.default_provisioner_name = 'custom-test-provisioner' async_km = AsyncKernelManager(kernel_name='no_provisioner') await async_km.start_kernel(stdout=PIPE, stderr=PIPE) is_alive = await async_km.is_alive() diff --git a/setup.py b/setup.py index 272bf91e8..9c15f681c 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ def run(self): 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_provisioners': [ - 'Local-Provisioner = jupyter_client.provisioning:LocalProvisioner', + 'local-provisioner = jupyter_client.provisioning:LocalProvisioner', ], }, ) From 1c1f71a58ccea01e9ad530a47bb3d2ba04d5509b Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Wed, 28 Apr 2021 17:01:18 -0700 Subject: [PATCH 19/37] Move provisioning to sub-package --- .github/workflows/main.yml | 2 +- jupyter_client/provisioning.py | 466 ------------------ jupyter_client/provisioning/__init__.py | 3 + jupyter_client/provisioning/factory.py | 132 +++++ .../provisioning/local_provisioner.py | 188 +++++++ .../provisioning/provisioner_base.py | 167 +++++++ 6 files changed, 491 insertions(+), 467 deletions(-) delete mode 100644 jupyter_client/provisioning.py create mode 100644 jupyter_client/provisioning/__init__.py create mode 100644 jupyter_client/provisioning/factory.py create mode 100644 jupyter_client/provisioning/local_provisioner.py create mode 100644 jupyter_client/provisioning/provisioner_base.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1b947eebe..d877362d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: pip install --upgrade --upgrade-strategy eager --pre -e .[test] pytest-cov codecov 'coverage<5' pip freeze - name: Check types - run: mypy jupyter_client/manager.py jupyter_client/multikernelmanager.py jupyter_client/client.py jupyter_client/blocking/client.py jupyter_client/asynchronous/client.py jupyter_client/channels.py jupyter_client/session.py jupyter_client/adapter.py jupyter_client/connect.py jupyter_client/consoleapp.py jupyter_client/jsonutil.py jupyter_client/kernelapp.py jupyter_client/launcher.py jupyter_client/threaded.py jupyter_client/provisioning.py + run: mypy jupyter_client/manager.py jupyter_client/multikernelmanager.py jupyter_client/client.py jupyter_client/blocking/client.py jupyter_client/asynchronous/client.py jupyter_client/channels.py jupyter_client/session.py jupyter_client/adapter.py jupyter_client/connect.py jupyter_client/consoleapp.py jupyter_client/jsonutil.py jupyter_client/kernelapp.py jupyter_client/launcher.py jupyter_client/threaded.py jupyter_client/provisioning/factory.py jupyter_client/provisioning/local_provisioner.py jupyter_client/provisioning/provisioner_base.py - name: Run the tests run: py.test --cov jupyter_client -v jupyter_client - name: Code coverage diff --git a/jupyter_client/provisioning.py b/jupyter_client/provisioning.py deleted file mode 100644 index cedca5ebd..000000000 --- a/jupyter_client/provisioning.py +++ /dev/null @@ -1,466 +0,0 @@ -"""Kernel Provisioner Classes""" -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import asyncio -import os -import signal -import sys -from abc import ABC -from abc import ABCMeta -from abc import abstractmethod -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -from entrypoints import EntryPoint # type: ignore -from entrypoints import get_group_all -from entrypoints import get_single -from entrypoints import NoSuchEntryPoint -from traitlets.config import default # type: ignore -from traitlets.config import Instance -from traitlets.config import LoggingConfigurable -from traitlets.config import SingletonConfigurable -from traitlets.config import Unicode - -from .launcher import async_launch_kernel -from .localinterfaces import is_local_ip -from .localinterfaces import local_ips -from .utils import ensure_async - - -class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): # type: ignore - pass - - -class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisionerMeta): - """Base class defining methods for KernelProvisioner classes. - - Theses methods model those of the Subprocess Popen class: - https://docs.python.org/3/library/subprocess.html#popen-objects - """ - - # The kernel specification associated with this provisioner - kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) - kernel_id: str = Unicode(None, allow_none=True) - connection_info: dict = {} - - @abstractmethod - async def poll(self) -> Optional[int]: - """Checks if kernel process is still running. - - If running, None is returned, otherwise the process's integer-valued exit code is returned. - """ - pass - - @abstractmethod - async def wait(self) -> Optional[int]: - """Waits for kernel process to terminate.""" - pass - - @abstractmethod - async def send_signal(self, signum: int) -> None: - """Sends signal identified by signum to the kernel process.""" - pass - - @abstractmethod - async def kill(self, restart: bool = False) -> None: - """Kills the kernel process. This is typically accomplished via a SIGKILL signal, - which cannot be caught. - - restart is True if this operation precedes a start launch_kernel request. - """ - pass - - @abstractmethod - async def terminate(self, restart: bool = False) -> None: - """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, - which can be caught, allowing the kernel provisioner to perform possible cleanup - of resources. - - restart is True if this operation precedes a start launch_kernel request. - """ - pass - - @abstractmethod - async def launch_kernel( - self, cmd: List[str], **kwargs: Any - ) -> Tuple['KernelProvisionerBase', Dict]: - """Launch the kernel process returning the class instance and connection info.""" - pass - - @abstractmethod - async def cleanup(self, restart: bool = False) -> None: - """Cleanup any resources allocated on behalf of the kernel provisioner. - - restart is True if this operation precedes a start launch_kernel request. - """ - pass - - async def shutdown_requested(self, restart: bool = False) -> None: - """Called after KernelManager sends a `shutdown_request` message to kernel. - - This method is optional and is primarily used in scenarios where the provisioner - communicates with a sibling (nanny) process to the kernel. - """ - pass - - async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: - """Perform any steps in preparation for kernel process launch. - - This includes applying additional substitutions to the kernel launch command and env. - It also includes preparation of launch parameters. - - Returns potentially updated kwargs. - """ - env = kwargs.pop('env', os.environ).copy() - env.update(self.__apply_env_substitutions(env)) - self._validate_parameters(env, **kwargs) - self._finalize_env(env) # TODO: Should finalize be called first? - kwargs['env'] = env - - return kwargs - - async def post_launch(self, **kwargs: Any) -> None: - """Perform any steps following the kernel process launch.""" - pass - - async def get_provisioner_info(self) -> Dict[str, Any]: - """Captures the base information necessary for persistence relative to this instance. - - The superclass method must always be called first to ensure proper ordering. - """ - provisioner_info: Dict[str, Any] = {} - return provisioner_info - - async def load_provisioner_info(self, provisioner_info: Dict) -> None: - """Loads the base information necessary for persistence relative to this instance. - - The superclass method must always be called first to ensure proper ordering. - """ - pass - - def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: - """Returns the time allowed for a complete shutdown. This may vary by provisioner. - - The recommended value will typically be what is configured in the kernel manager. - """ - return recommended - - def _finalize_env(self, env: Dict[str, str]) -> None: - """Ensures env is appropriate prior to launch. - - Subclasses should be sure to call super()._finalize_env(env) - """ - if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): - # Don't allow PYTHONEXECUTABLE to be passed to kernel process. - # If set, it can bork all the things. - env.pop('PYTHONEXECUTABLE', None) - - def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel spec.""" - pass - - def __apply_env_substitutions(self, substitution_values: Dict[str, str]): - """Walks entries in the kernelspec's env stanza and applies substitutions from current env. - - Returns the substituted list of env entries. - Note: This method is private and is not intended to be overridden by provisioners. - """ - substituted_env = {} - if self.kernel_spec: - from string import Template - - # For each templated env entry, fill any templated references - # matching names of env variables with those values and build - # new dict with substitutions. - templated_env = self.kernel_spec.env - for k, v in templated_env.items(): - substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) - return substituted_env - - -class LocalProvisioner(KernelProvisionerBase): - - process = None - async_subprocess = None - _exit_future = None - pid = None - pgid = None - ip = None - - async def poll(self) -> Optional[int]: - ret = None - if self.process: - if self.async_subprocess: - if not self._exit_future.done(): # adhere to process returncode - ret = None - else: - ret = self._exit_future.result() - else: - ret = self.process.poll() - return ret - - async def wait(self) -> Optional[int]: - ret = None - if self.process: - if self.async_subprocess: - ret = await self.process.wait() - else: - # Use busy loop at 100ms intervals, polling until the process is - # not alive. If we find the process is no longer alive, complete - # its cleanup via the blocking wait(). Callers are responsible for - # issuing calls to wait() using a timeout (see kill()). - while await self.poll() is None: - await asyncio.sleep(0.1) - - # Process is no longer alive, wait and clear - ret = self.process.wait() - self.process = None - return ret - - async def send_signal(self, signum: int) -> None: - """Sends a signal to the process group of the kernel (this - usually includes the kernel and any subprocesses spawned by - the kernel). - - Note that since only SIGTERM is supported on Windows, we will - check if the desired signal is for interrupt and apply the - applicable code on Windows in that case. - """ - if self.process: - if signum == signal.SIGINT and sys.platform == 'win32': - from .win_interrupt import send_interrupt - - send_interrupt(self.process.win32_interrupt_event) - return - - # Prefer process-group over process - if self.pgid and hasattr(os, "killpg"): - try: - os.killpg(self.pgid, signum) - return - except OSError: - pass - try: - self.process.send_signal(signum) - except OSError: - pass - return - - async def kill(self, restart: bool = False) -> None: - if self.process: - try: - self.process.kill() - except OSError as e: - # In Windows, we will get an Access Denied error if the process - # has already terminated. Ignore it. - if sys.platform == 'win32': - if e.winerror != 5: - raise - # On Unix, we may get an ESRCH error if the process has already - # terminated. Ignore it. - else: - from errno import ESRCH - - if e.errno != ESRCH: - raise - - async def terminate(self, restart: bool = False) -> None: - if self.process: - return self.process.terminate() - - async def cleanup(self, restart: bool = False) -> None: - pass - - async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: - """Perform any steps in preparation for kernel process launch. - - This includes applying additional substitutions to the kernel launch command and env. - It also includes preparation of launch parameters. - - Returns the updated kwargs. - """ - - # If we have a kernel_manager pop it out of the args and use it to retain b/c. - # This should be considered temporary until a better division of labor can be defined. - km = kwargs.pop('kernel_manager', None) - if km: - if km.transport == 'tcp' and not is_local_ip(km.ip): - raise RuntimeError( - "Can only launch a kernel on a local interface. " - "This one is not: %s." - "Make sure that the '*_address' attributes are " - "configured properly. " - "Currently valid addresses are: %s" % (km.ip, local_ips()) - ) - # build the Popen cmd - extra_arguments = kwargs.pop('extra_arguments', []) - - # write connection file / get default ports - km.write_connection_file() # TODO - change when handshake pattern is adopted - self.connection_info = km.get_connection_info() - - kernel_cmd = km.format_kernel_cmd( - extra_arguments=extra_arguments - ) # This needs to remain here for b/c - else: - extra_arguments = kwargs.pop('extra_arguments', []) - kernel_cmd = self.kernel_spec.argv + extra_arguments - - return await super().pre_launch(cmd=kernel_cmd, **kwargs) - - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: - scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) - self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) - self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) - if self.async_subprocess: - self._exit_future = asyncio.ensure_future(ensure_async(self.process.wait())) - pgid = None - if hasattr(os, "getpgid"): - try: - pgid = os.getpgid(self.process.pid) # type: ignore - except OSError: - pass - - self.pid = self.process.pid - self.pgid = pgid - return self, self.connection_info - - @staticmethod - def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: - """Remove any keyword arguments that Popen does not tolerate.""" - keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id', 'kernel_manager'] - scrubbed_kwargs = kwargs.copy() - for kw in keywords_to_scrub: - scrubbed_kwargs.pop(kw, None) - return scrubbed_kwargs - - async def get_provisioner_info(self) -> Dict: - """Captures the base information necessary for persistence relative to this instance.""" - provisioner_info = await super().get_provisioner_info() - provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) - return provisioner_info - - async def load_provisioner_info(self, provisioner_info: Dict) -> None: - """Loads the base information necessary for persistence relative to this instance.""" - await super().load_provisioner_info(provisioner_info) - self.pid = provisioner_info['pid'] - self.pgid = provisioner_info['pgid'] - self.ip = provisioner_info['ip'] - - -class KernelProvisionerFactory(SingletonConfigurable): - """KernelProvisionerFactory is responsible for creating provisioner instances.""" - - GROUP_NAME = 'jupyter_client.kernel_provisioners' - provisioners: Dict[str, EntryPoint] = {} - - default_provisioner_name_env = "JUPYTER_DEFAULT_PROVISIONER_NAME" - default_provisioner_name = Unicode( - config=True, - help="""Indicates the name of the provisioner to use when no kernel_provisioner - entry is present in the kernelspec.""", - ) - - @default('default_provisioner_name') - def default_provisioner_name_default(self): - return os.getenv(self.default_provisioner_name_env, "local-provisioner") - - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - - for ep in KernelProvisionerFactory._get_all_provisioners(): - self.provisioners[ep.name] = ep - - def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: - """ - Reads the associated kernel_spec to determine the provisioner and returns whether it - exists as an entry_point (True) or not (False). If the referenced provisioner is not - in the current set of provisioners, attempt to retrieve its entrypoint. If found, add - to the list, else catch exception and return false. - """ - is_available: bool = True - provisioner_cfg = self._get_provisioner_config(kernel_spec) - provisioner_name = str(provisioner_cfg.get('provisioner_name')) - if provisioner_name not in self.provisioners: - try: - ep = KernelProvisionerFactory._get_provisioner(provisioner_name) - self.provisioners[provisioner_name] = ep # Update cache - except NoSuchEntryPoint: - is_available = False - self.log.warning( - f"Kernel '{kernel_name}' is referencing a kernel provisioner " - f"('{provisioner_name}') that is not available. Ensure the " - f"appropriate package has been installed and retry." - ) - return is_available - - def create_provisioner_instance( - self, kernel_id: str, kernel_spec: Any - ) -> KernelProvisionerBase: - """ - Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. - If one exists, it instantiates an instance. If a kernel provisioner is not - specified in the kernelspec, a DEFAULT_PROVISIONER stanza is fabricated and instantiated. - The instantiated instance is returned. - - If the provisioner is found to not exist (not registered via entry_points), - ModuleNotFoundError is raised. - """ - provisioner_cfg = self._get_provisioner_config(kernel_spec) - provisioner_name = provisioner_cfg.get('provisioner_name') - if provisioner_name not in self.provisioners: - raise ModuleNotFoundError( - f"Kernel provisioner '{provisioner_name}' has not been registered." - ) - - self.log.debug( - f"Instantiating kernel '{kernel_spec.display_name}' with " - f"kernel provisioner: {provisioner_name}" - ) - provisioner_class = self.provisioners[provisioner_name].load() - provisioner_config = provisioner_cfg.get('config') - return provisioner_class( - kernel_id=kernel_id, kernel_spec=kernel_spec, parent=self.parent, **provisioner_config - ) - - def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: - """ - Return the kernel_provisioner stanza from the kernel_spec. - - Checks the kernel_spec's metadata dictionary for a kernel_provisioner entry. - If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER - and returned. - - Parameters - ---------- - kernel_spec : Any - this is a KernelSpec type but listed as Any to avoid circular import - The kernel specification object from which the provisioner dictionary is derived. - - Returns - ------- - dict - The provisioner portion of the kernel_spec. If one does not exist, it will contain - the default information. If no `config` sub-dictionary exists, an empty `config` - dictionary will be added. - """ - env_provisioner = kernel_spec.metadata.get('kernel_provisioner', {}) - if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default - if ( - 'config' not in env_provisioner - ): # if provisioner_name, but no config stanza, add one - env_provisioner.update({"config": {}}) - return env_provisioner # Return what we found (plus config stanza if necessary) - return {"provisioner_name": self.default_provisioner_name, "config": {}} - - @staticmethod - def _get_all_provisioners() -> List[EntryPoint]: - """Wrapper around entrypoints.get_group_all() - primarily to facilitate testing.""" - return get_group_all(KernelProvisionerFactory.GROUP_NAME) - - @staticmethod - def _get_provisioner(name: str) -> EntryPoint: - """Wrapper around entrypoints.get_single() - primarily to facilitate testing.""" - return get_single(KernelProvisionerFactory.GROUP_NAME, name) diff --git a/jupyter_client/provisioning/__init__.py b/jupyter_client/provisioning/__init__.py new file mode 100644 index 000000000..2d6c47aee --- /dev/null +++ b/jupyter_client/provisioning/__init__.py @@ -0,0 +1,3 @@ +from .factory import KernelProvisionerFactory # noqa +from .local_provisioner import LocalProvisioner # noqa +from .provisioner_base import KernelProvisionerBase # noqa diff --git a/jupyter_client/provisioning/factory.py b/jupyter_client/provisioning/factory.py new file mode 100644 index 000000000..fb0656ebc --- /dev/null +++ b/jupyter_client/provisioning/factory.py @@ -0,0 +1,132 @@ +"""Kernel Provisioner Classes""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +from typing import Any +from typing import Dict +from typing import List + +from entrypoints import EntryPoint # type: ignore +from entrypoints import get_group_all +from entrypoints import get_single +from entrypoints import NoSuchEntryPoint +from traitlets.config import default # type: ignore +from traitlets.config import SingletonConfigurable +from traitlets.config import Unicode + +from .provisioner_base import KernelProvisionerBase + + +class KernelProvisionerFactory(SingletonConfigurable): + """KernelProvisionerFactory is responsible for creating provisioner instances.""" + + GROUP_NAME = 'jupyter_client.kernel_provisioners' + provisioners: Dict[str, EntryPoint] = {} + + default_provisioner_name_env = "JUPYTER_DEFAULT_PROVISIONER_NAME" + default_provisioner_name = Unicode( + config=True, + help="""Indicates the name of the provisioner to use when no kernel_provisioner + entry is present in the kernelspec.""", + ) + + @default('default_provisioner_name') + def default_provisioner_name_default(self): + return os.getenv(self.default_provisioner_name_env, "local-provisioner") + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + for ep in KernelProvisionerFactory._get_all_provisioners(): + self.provisioners[ep.name] = ep + + def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: + """ + Reads the associated kernel_spec to determine the provisioner and returns whether it + exists as an entry_point (True) or not (False). If the referenced provisioner is not + in the current set of provisioners, attempt to retrieve its entrypoint. If found, add + to the list, else catch exception and return false. + """ + is_available: bool = True + provisioner_cfg = self._get_provisioner_config(kernel_spec) + provisioner_name = str(provisioner_cfg.get('provisioner_name')) + if provisioner_name not in self.provisioners: + try: + ep = KernelProvisionerFactory._get_provisioner(provisioner_name) + self.provisioners[provisioner_name] = ep # Update cache + except NoSuchEntryPoint: + is_available = False + self.log.warning( + f"Kernel '{kernel_name}' is referencing a kernel provisioner " + f"('{provisioner_name}') that is not available. Ensure the " + f"appropriate package has been installed and retry." + ) + return is_available + + def create_provisioner_instance( + self, kernel_id: str, kernel_spec: Any + ) -> KernelProvisionerBase: + """ + Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. + If one exists, it instantiates an instance. If a kernel provisioner is not + specified in the kernelspec, a DEFAULT_PROVISIONER stanza is fabricated and instantiated. + The instantiated instance is returned. + + If the provisioner is found to not exist (not registered via entry_points), + ModuleNotFoundError is raised. + """ + provisioner_cfg = self._get_provisioner_config(kernel_spec) + provisioner_name = provisioner_cfg.get('provisioner_name') + if provisioner_name not in self.provisioners: + raise ModuleNotFoundError( + f"Kernel provisioner '{provisioner_name}' has not been registered." + ) + + self.log.debug( + f"Instantiating kernel '{kernel_spec.display_name}' with " + f"kernel provisioner: {provisioner_name}" + ) + provisioner_class = self.provisioners[provisioner_name].load() + provisioner_config = provisioner_cfg.get('config') + return provisioner_class( + kernel_id=kernel_id, kernel_spec=kernel_spec, parent=self.parent, **provisioner_config + ) + + def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: + """ + Return the kernel_provisioner stanza from the kernel_spec. + + Checks the kernel_spec's metadata dictionary for a kernel_provisioner entry. + If found, it is returned, else one is created relative to the DEFAULT_PROVISIONER + and returned. + + Parameters + ---------- + kernel_spec : Any - this is a KernelSpec type but listed as Any to avoid circular import + The kernel specification object from which the provisioner dictionary is derived. + + Returns + ------- + dict + The provisioner portion of the kernel_spec. If one does not exist, it will contain + the default information. If no `config` sub-dictionary exists, an empty `config` + dictionary will be added. + """ + env_provisioner = kernel_spec.metadata.get('kernel_provisioner', {}) + if 'provisioner_name' in env_provisioner: # If no provisioner_name, return default + if ( + 'config' not in env_provisioner + ): # if provisioner_name, but no config stanza, add one + env_provisioner.update({"config": {}}) + return env_provisioner # Return what we found (plus config stanza if necessary) + return {"provisioner_name": self.default_provisioner_name, "config": {}} + + @staticmethod + def _get_all_provisioners() -> List[EntryPoint]: + """Wrapper around entrypoints.get_group_all() - primarily to facilitate testing.""" + return get_group_all(KernelProvisionerFactory.GROUP_NAME) + + @staticmethod + def _get_provisioner(name: str) -> EntryPoint: + """Wrapper around entrypoints.get_single() - primarily to facilitate testing.""" + return get_single(KernelProvisionerFactory.GROUP_NAME, name) diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py new file mode 100644 index 000000000..bf3e5922b --- /dev/null +++ b/jupyter_client/provisioning/local_provisioner.py @@ -0,0 +1,188 @@ +"""Kernel Provisioner Classes""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import asyncio +import os +import signal +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +from ..launcher import async_launch_kernel +from ..localinterfaces import is_local_ip +from ..localinterfaces import local_ips +from ..utils import ensure_async +from .provisioner_base import KernelProvisionerBase + + +class LocalProvisioner(KernelProvisionerBase): + + process = None + async_subprocess = None + _exit_future = None + pid = None + pgid = None + ip = None + + async def poll(self) -> Optional[int]: + ret = None + if self.process: + if self.async_subprocess: + if not self._exit_future.done(): # adhere to process returncode + ret = None + else: + ret = self._exit_future.result() + else: + ret = self.process.poll() + return ret + + async def wait(self) -> Optional[int]: + ret = None + if self.process: + if self.async_subprocess: + ret = await self.process.wait() + else: + # Use busy loop at 100ms intervals, polling until the process is + # not alive. If we find the process is no longer alive, complete + # its cleanup via the blocking wait(). Callers are responsible for + # issuing calls to wait() using a timeout (see kill()). + while await self.poll() is None: + await asyncio.sleep(0.1) + + # Process is no longer alive, wait and clear + ret = self.process.wait() + self.process = None + return ret + + async def send_signal(self, signum: int) -> None: + """Sends a signal to the process group of the kernel (this + usually includes the kernel and any subprocesses spawned by + the kernel). + + Note that since only SIGTERM is supported on Windows, we will + check if the desired signal is for interrupt and apply the + applicable code on Windows in that case. + """ + if self.process: + if signum == signal.SIGINT and sys.platform == 'win32': + from ..win_interrupt import send_interrupt + + send_interrupt(self.process.win32_interrupt_event) + return + + # Prefer process-group over process + if self.pgid and hasattr(os, "killpg"): + try: + os.killpg(self.pgid, signum) + return + except OSError: + pass + try: + self.process.send_signal(signum) + except OSError: + pass + return + + async def kill(self, restart: bool = False) -> None: + if self.process: + try: + self.process.kill() + except OSError as e: + # In Windows, we will get an Access Denied error if the process + # has already terminated. Ignore it. + if sys.platform == 'win32': + if e.winerror != 5: + raise + # On Unix, we may get an ESRCH error if the process has already + # terminated. Ignore it. + else: + from errno import ESRCH + + if e.errno != ESRCH: + raise + + async def terminate(self, restart: bool = False) -> None: + if self.process: + return self.process.terminate() + + async def cleanup(self, restart: bool = False) -> None: + pass + + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + """Perform any steps in preparation for kernel process launch. + + This includes applying additional substitutions to the kernel launch command and env. + It also includes preparation of launch parameters. + + Returns the updated kwargs. + """ + + # If we have a kernel_manager pop it out of the args and use it to retain b/c. + # This should be considered temporary until a better division of labor can be defined. + km = kwargs.pop('kernel_manager', None) + if km: + if km.transport == 'tcp' and not is_local_ip(km.ip): + raise RuntimeError( + "Can only launch a kernel on a local interface. " + "This one is not: %s." + "Make sure that the '*_address' attributes are " + "configured properly. " + "Currently valid addresses are: %s" % (km.ip, local_ips()) + ) + # build the Popen cmd + extra_arguments = kwargs.pop('extra_arguments', []) + + # write connection file / get default ports + km.write_connection_file() # TODO - change when handshake pattern is adopted + self.connection_info = km.get_connection_info() + + kernel_cmd = km.format_kernel_cmd( + extra_arguments=extra_arguments + ) # This needs to remain here for b/c + else: + extra_arguments = kwargs.pop('extra_arguments', []) + kernel_cmd = self.kernel_spec.argv + extra_arguments + + return await super().pre_launch(cmd=kernel_cmd, **kwargs) + + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: + scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) + self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) + self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) + if self.async_subprocess: + self._exit_future = asyncio.ensure_future(ensure_async(self.process.wait())) + pgid = None + if hasattr(os, "getpgid"): + try: + pgid = os.getpgid(self.process.pid) # type: ignore + except OSError: + pass + + self.pid = self.process.pid + self.pgid = pgid + return self, self.connection_info + + @staticmethod + def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Remove any keyword arguments that Popen does not tolerate.""" + keywords_to_scrub: List[str] = ['extra_arguments', 'kernel_id', 'kernel_manager'] + scrubbed_kwargs = kwargs.copy() + for kw in keywords_to_scrub: + scrubbed_kwargs.pop(kw, None) + return scrubbed_kwargs + + async def get_provisioner_info(self) -> Dict: + """Captures the base information necessary for persistence relative to this instance.""" + provisioner_info = await super().get_provisioner_info() + provisioner_info.update({'pid': self.pid, 'pgid': self.pgid, 'ip': self.ip}) + return provisioner_info + + async def load_provisioner_info(self, provisioner_info: Dict) -> None: + """Loads the base information necessary for persistence relative to this instance.""" + await super().load_provisioner_info(provisioner_info) + self.pid = provisioner_info['pid'] + self.pgid = provisioner_info['pgid'] + self.ip = provisioner_info['ip'] diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py new file mode 100644 index 000000000..6288967db --- /dev/null +++ b/jupyter_client/provisioning/provisioner_base.py @@ -0,0 +1,167 @@ +"""Kernel Provisioner Classes""" +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +from abc import ABC +from abc import ABCMeta +from abc import abstractmethod +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +from traitlets.config import Instance # type: ignore +from traitlets.config import LoggingConfigurable +from traitlets.config import Unicode + + +class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): # type: ignore + pass + + +class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisionerMeta): + """Base class defining methods for KernelProvisioner classes. + + Theses methods model those of the Subprocess Popen class: + https://docs.python.org/3/library/subprocess.html#popen-objects + """ + + # The kernel specification associated with this provisioner + kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) + kernel_id: str = Unicode(None, allow_none=True) + connection_info: dict = {} + + @abstractmethod + async def poll(self) -> Optional[int]: + """Checks if kernel process is still running. + + If running, None is returned, otherwise the process's integer-valued exit code is returned. + """ + pass + + @abstractmethod + async def wait(self) -> Optional[int]: + """Waits for kernel process to terminate.""" + pass + + @abstractmethod + async def send_signal(self, signum: int) -> None: + """Sends signal identified by signum to the kernel process.""" + pass + + @abstractmethod + async def kill(self, restart: bool = False) -> None: + """Kills the kernel process. This is typically accomplished via a SIGKILL signal, + which cannot be caught. + + restart is True if this operation precedes a start launch_kernel request. + """ + pass + + @abstractmethod + async def terminate(self, restart: bool = False) -> None: + """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, + which can be caught, allowing the kernel provisioner to perform possible cleanup + of resources. + + restart is True if this operation precedes a start launch_kernel request. + """ + pass + + @abstractmethod + async def launch_kernel( + self, cmd: List[str], **kwargs: Any + ) -> Tuple['KernelProvisionerBase', Dict]: + """Launch the kernel process returning the class instance and connection info.""" + pass + + @abstractmethod + async def cleanup(self, restart: bool = False) -> None: + """Cleanup any resources allocated on behalf of the kernel provisioner. + + restart is True if this operation precedes a start launch_kernel request. + """ + pass + + async def shutdown_requested(self, restart: bool = False) -> None: + """Called after KernelManager sends a `shutdown_request` message to kernel. + + This method is optional and is primarily used in scenarios where the provisioner + communicates with a sibling (nanny) process to the kernel. + """ + pass + + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + """Perform any steps in preparation for kernel process launch. + + This includes applying additional substitutions to the kernel launch command and env. + It also includes preparation of launch parameters. + + Returns potentially updated kwargs. + """ + env = kwargs.pop('env', os.environ).copy() + env.update(self.__apply_env_substitutions(env)) + self._validate_parameters(env, **kwargs) + self._finalize_env(env) # TODO: Should finalize be called first? + kwargs['env'] = env + + return kwargs + + async def post_launch(self, **kwargs: Any) -> None: + """Perform any steps following the kernel process launch.""" + pass + + async def get_provisioner_info(self) -> Dict[str, Any]: + """Captures the base information necessary for persistence relative to this instance. + + The superclass method must always be called first to ensure proper ordering. + """ + provisioner_info: Dict[str, Any] = {} + return provisioner_info + + async def load_provisioner_info(self, provisioner_info: Dict) -> None: + """Loads the base information necessary for persistence relative to this instance. + + The superclass method must always be called first to ensure proper ordering. + """ + pass + + def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: + """Returns the time allowed for a complete shutdown. This may vary by provisioner. + + The recommended value will typically be what is configured in the kernel manager. + """ + return recommended + + def _finalize_env(self, env: Dict[str, str]) -> None: + """Ensures env is appropriate prior to launch. + + Subclasses should be sure to call super()._finalize_env(env) + """ + if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): + # Don't allow PYTHONEXECUTABLE to be passed to kernel process. + # If set, it can bork all the things. + env.pop('PYTHONEXECUTABLE', None) + + def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: + """Future: Validates that launch parameters adhere to schema specified in kernel spec.""" + pass + + def __apply_env_substitutions(self, substitution_values: Dict[str, str]): + """Walks entries in the kernelspec's env stanza and applies substitutions from current env. + + Returns the substituted list of env entries. + Note: This method is private and is not intended to be overridden by provisioners. + """ + substituted_env = {} + if self.kernel_spec: + from string import Template + + # For each templated env entry, fill any templated references + # matching names of env variables with those values and build + # new dict with substitutions. + templated_env = self.kernel_spec.env + for k, v in templated_env.items(): + substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) + return substituted_env From 2c034a93119540c7145982714eb7f9bb5e62f203 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 29 Apr 2021 14:18:13 -0700 Subject: [PATCH 20/37] Restore launcher, don't offer async launch_kernel --- jupyter_client/launcher.py | 171 +++--------------- .../provisioning/local_provisioner.py | 40 ++-- 2 files changed, 38 insertions(+), 173 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 02e89b6a4..a4adfe636 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -1,18 +1,13 @@ """Utilities for launching kernels""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import asyncio import os import sys from subprocess import PIPE from subprocess import Popen -from typing import Any from typing import Dict from typing import List -from typing import NoReturn from typing import Optional -from typing import Tuple -from typing import Union from traitlets.log import get_logger # type: ignore @@ -22,155 +17,32 @@ def launch_kernel( stdin: Optional[int] = None, stdout: Optional[int] = None, stderr: Optional[int] = None, - env: Optional[Dict[str, Any]] = None, + env: Optional[Dict[str, str]] = None, independent: bool = False, cwd: Optional[str] = None, **kw, ) -> Popen: """Launches a localhost kernel, binding to the specified ports. - Parameters ---------- cmd : Popen list, A string of Python code that imports and executes a kernel entry point. - stdin, stdout, stderr : optional (default None) Standards streams, as defined in subprocess.Popen. - env: dict, optional Environment variables passed to the kernel - independent : bool, optional (default False) If set, the kernel process is guaranteed to survive if this process dies. If not set, an effort is made to ensure that the kernel is killed when this process dies. Note that in this case it is still good practice to kill kernels manually before exiting. - cwd : path, optional The working dir of the kernel process (default: cwd of this process). - **kw: optional Additional arguments for Popen - Returns ------- - - subprocess.Popen instance for the kernel subprocess - """ - proc = None - - kwargs, interrupt_event = _prepare_process_args( - stdin=stdin, stdout=stdout, stderr=stderr, env=env, independent=independent, cwd=cwd, **kw - ) - - try: - # Allow to use ~/ in the command or its arguments - cmd = [os.path.expanduser(s) for s in cmd] - - proc = Popen(cmd, **kwargs) - except Exception as exc: - _handle_subprocess_exception(exc, cmd, env, **kwargs) - - _finish_process_launch(proc, stdin, interrupt_event) - - return proc - - -async def async_launch_kernel( - cmd: List[str], - stdin: Optional[int] = None, - stdout: Optional[int] = None, - stderr: Optional[int] = None, - env: Optional[Dict[str, Any]] = None, - independent: Optional[bool] = False, - cwd: Optional[str] = None, - **kw, -) -> Union[Popen, asyncio.subprocess.Process]: - """Launches a localhost kernel, binding to the specified ports using async subprocess. - - Parameters - ---------- - cmd : list, - A string of Python code that imports and executes a kernel entry point. - - stdin, stdout, stderr : optional (default None) - Standards streams, as defined in subprocess.Popen. - - env: dict, optional - Environment variables passed to the kernel - - independent : bool, optional (default False) - If set, the kernel process is guaranteed to survive if this process - dies. If not set, an effort is made to ensure that the kernel is killed - when this process dies. Note that in this case it is still good practice - to kill kernels manually before exiting. - - cwd : path, optional - The working dir of the kernel process (default: cwd of this process). - - **kw: optional - Additional arguments for Popen - - Returns - ------- - - asyncio.subprocess.Process instance for the kernel subprocess - """ - proc = None - - kwargs, interrupt_event = _prepare_process_args( - stdin=stdin, stdout=stdout, stderr=stderr, env=env, independent=independent, cwd=cwd, **kw - ) - - try: - # Allow to use ~/ in the command or its arguments - cmd = [os.path.expanduser(s) for s in cmd] - - if sys.platform == 'win32': - # Windows is still not ready for async subprocs. When the SelectorEventLoop is - # used (3.6, 3.7), a NotImplementedError is raised for _make_subprocess_transport(). - # When the ProactorEventLoop is used (3.8, 3.9), a NotImplementedError is raised - # for _add_reader(). - proc = Popen(cmd, **kwargs) - else: - proc = await asyncio.create_subprocess_exec(*cmd, **kwargs) - except Exception as exc: - _handle_subprocess_exception(exc, cmd, env, **kwargs) - - _finish_process_launch(proc, stdin, interrupt_event) - - return proc - - -def _handle_subprocess_exception( - exc: Exception, cmd: List[str], env: Optional[Dict[str, Any]], **kwargs -) -> NoReturn: - """ Log error message consisting of runtime arguments prior to raising the exception. """ - try: - msg = "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" - # exclude environment variables, - # which may contain access tokens and the like. - without_env = {key: value for key, value in kwargs.items() if key != 'env'} - assert env is not None - msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) - get_logger().error(msg) - finally: - raise exc - - -def _prepare_process_args( - stdin: Optional[int] = None, - stdout: Optional[int] = None, - stderr: Optional[int] = None, - env: Optional[Dict[str, str]] = None, - independent: Optional[bool] = False, - cwd: Optional[str] = None, - **kw, -) -> Tuple[Dict[str, Any], Any]: - """Prepares for process launch. - - This consists of getting arguments into a form Popen/Process understands and creating the - necessary Windows structures to support interrupts. + Popen instance for the kernel subprocess """ # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr @@ -181,7 +53,7 @@ def _prepare_process_args( # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. - + redirect_in = True _stdin = PIPE if stdin is None else stdin # If this process in running on pythonw, we know that stdin, stdout, and @@ -262,7 +134,6 @@ def _prepare_process_args( # (we always capture stdin, so this is already False by default on <3.7) kwargs["close_fds"] = False else: - interrupt_event = None # N/A # Create a new session. # This makes it easier to interrupt the kernel, # because we want to interrupt the whole process group. @@ -272,24 +143,32 @@ def _prepare_process_args( if not independent: env["JPY_PARENT_PID"] = str(os.getpid()) - return kwargs, interrupt_event - - -def _finish_process_launch( - subprocess: Union[Popen, asyncio.subprocess.Process], stdin: Optional[int], interrupt_event: Any -) -> None: - """Finishes the process launch by patching the interrupt event (windows) and closing stdin. """ - - assert subprocess is not None + try: + # Allow to use ~/ in the command or its arguments + cmd = [os.path.expanduser(s) for s in cmd] + proc = Popen(cmd, **kwargs) + except Exception: + msg = "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" + # exclude environment variables, + # which may contain access tokens and the like. + without_env = {key: value for key, value in kwargs.items() if key != "env"} + msg = msg.format(cmd, env.get("PATH", os.defpath), without_env) + get_logger().error(msg) + raise if sys.platform == "win32": # Attach the interrupt event to the Popen objet so it can be used later. - subprocess.win32_interrupt_event = interrupt_event # type: ignore + proc.win32_interrupt_event = interrupt_event # type: ignore # Clean up pipes created to work around Popen bug. - if stdin is None: - assert subprocess.stdin is not None - subprocess.stdin.close() + if redirect_in: + if stdin is None: + assert proc.stdin is not None + proc.stdin.close() + + return proc -__all__ = ["launch_kernel", "async_launch_kernel"] +__all__ = [ + "launch_kernel", +] diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index bf3e5922b..675daefc1 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -11,17 +11,15 @@ from typing import Optional from typing import Tuple -from ..launcher import async_launch_kernel +from ..launcher import launch_kernel from ..localinterfaces import is_local_ip from ..localinterfaces import local_ips -from ..utils import ensure_async from .provisioner_base import KernelProvisionerBase class LocalProvisioner(KernelProvisionerBase): process = None - async_subprocess = None _exit_future = None pid = None pgid = None @@ -30,31 +28,22 @@ class LocalProvisioner(KernelProvisionerBase): async def poll(self) -> Optional[int]: ret = None if self.process: - if self.async_subprocess: - if not self._exit_future.done(): # adhere to process returncode - ret = None - else: - ret = self._exit_future.result() - else: - ret = self.process.poll() + ret = self.process.poll() return ret async def wait(self) -> Optional[int]: ret = None if self.process: - if self.async_subprocess: - ret = await self.process.wait() - else: - # Use busy loop at 100ms intervals, polling until the process is - # not alive. If we find the process is no longer alive, complete - # its cleanup via the blocking wait(). Callers are responsible for - # issuing calls to wait() using a timeout (see kill()). - while await self.poll() is None: - await asyncio.sleep(0.1) - - # Process is no longer alive, wait and clear - ret = self.process.wait() - self.process = None + # Use busy loop at 100ms intervals, polling until the process is + # not alive. If we find the process is no longer alive, complete + # its cleanup via the blocking wait(). Callers are responsible for + # issuing calls to wait() using a timeout (see kill()). + while await self.poll() is None: + await asyncio.sleep(0.1) + + # Process is no longer alive, wait and clear + ret = self.process.wait() + self.process = None return ret async def send_signal(self, signum: int) -> None: @@ -150,10 +139,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) - self.process = await async_launch_kernel(cmd, **scrubbed_kwargs) - self.async_subprocess = isinstance(self.process, asyncio.subprocess.Process) - if self.async_subprocess: - self._exit_future = asyncio.ensure_future(ensure_async(self.process.wait())) + self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None if hasattr(os, "getpgid"): try: From a7cb40801bbc929c4a3a8152737f22063bc41071 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 30 Apr 2021 11:44:04 -0700 Subject: [PATCH 21/37] Restore whitespace to launch_kernel docstring --- jupyter_client/launcher.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index a4adfe636..eec1354d5 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -23,25 +23,33 @@ def launch_kernel( **kw, ) -> Popen: """Launches a localhost kernel, binding to the specified ports. + Parameters ---------- cmd : Popen list, A string of Python code that imports and executes a kernel entry point. + stdin, stdout, stderr : optional (default None) Standards streams, as defined in subprocess.Popen. + env: dict, optional Environment variables passed to the kernel + independent : bool, optional (default False) If set, the kernel process is guaranteed to survive if this process dies. If not set, an effort is made to ensure that the kernel is killed when this process dies. Note that in this case it is still good practice to kill kernels manually before exiting. + cwd : path, optional The working dir of the kernel process (default: cwd of this process). + **kw: optional Additional arguments for Popen + Returns ------- + Popen instance for the kernel subprocess """ From 73fa0faba60c9e37b19abb1f26b5a3ca0389d488 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 7 May 2021 17:34:53 -0700 Subject: [PATCH 22/37] Remove dual self.kernel, self.provisioner --- jupyter_client/manager.py | 45 +++++++++---------- .../provisioning/local_provisioner.py | 15 ++++--- .../provisioning/provisioner_base.py | 34 ++++++++------ jupyter_client/tests/test_provisioning.py | 29 ++++++------ 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 2d34eff68..801419e71 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -252,16 +252,16 @@ def from_ns(match): return [pat.sub(from_ns, arg) for arg in cmd] - async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> KernelProvisionerBase: + async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> None: """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ assert self.provisioner is not None - kernel_proc, connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) + await self.provisioner.launch_kernel(kernel_cmd, **kw) + assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager and write file. - self._force_connection_info(connection_info) - return kernel_proc + self._force_connection_info(self.provisioner.connection_info) _launch_kernel = run_sync(_async_launch_kernel) @@ -331,7 +331,7 @@ async def _async_start_kernel(self, **kw): # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) - self.kernel = await ensure_async(self._launch_kernel(kernel_cmd, **kw)) + await ensure_async(self._launch_kernel(kernel_cmd, **kw)) await self.post_start_kernel(**kw) start_kernel = run_sync(_async_start_kernel) @@ -345,6 +345,7 @@ async def request_shutdown(self, restart: bool = False) -> None: self.session.send(self._control_socket, msg) assert self.provisioner is not None await self.provisioner.shutdown_requested(restart=restart) + self._shutdown_status = _ShutdownStatus.ShutdownRequest async def _async_finish_shutdown( self, @@ -361,7 +362,7 @@ async def _async_finish_shutdown( waittime = max(self.shutdown_wait_time, 0) if self.provisioner: # Allow provisioner to override waittime = self.provisioner.get_shutdown_wait_time(recommended=waittime) - self._shutdown_status = _ShutdownStatus.ShutdownRequest + try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 @@ -381,9 +382,8 @@ async def _async_finish_shutdown( await ensure_async(self._kill_kernel(restart=restart)) else: # Process is no longer alive, wait and clear - if self.kernel is not None: - await self.kernel.wait() - self.kernel = None + if self.has_kernel: + await self.provisioner.wait() finish_shutdown = run_sync(_async_finish_shutdown) @@ -401,7 +401,7 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: if self.provisioner: await self.provisioner.cleanup(restart=restart) - self.provisioner = self.kernel = None + self.provisioner = None cleanup_resources = run_sync(_async_cleanup_resources) @@ -484,14 +484,14 @@ async def _async_restart_kernel(self, now: bool = False, newports: bool = False, @property def has_kernel(self) -> bool: - """Has a kernel been started that we are managing.""" - return self.kernel is not None + """Has a kernel process been started that we are managing.""" + return self.provisioner is not None and self.provisioner.has_process async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: - assert self.kernel is not None - await self.kernel.terminate(restart=restart) + assert self.provisioner is not None + await self.provisioner.terminate(restart=restart) _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm) @@ -501,8 +501,8 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - assert self.kernel is not None - await self.kernel.kill(restart=restart) + assert self.provisioner is not None + await self.provisioner.kill(restart=restart) # Wait until the kernel terminates. try: @@ -513,9 +513,8 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: pass else: # Process is no longer alive, wait and clear - if self.kernel is not None: - await self.kernel.wait() - self.kernel = None + if self.has_kernel: + await self.provisioner.wait() _kill_kernel = run_sync(_async_kill_kernel) @@ -549,8 +548,8 @@ async def _async_signal_kernel(self, signum: int) -> None: only useful on Unix systems. """ if self.has_kernel: - assert self.kernel is not None - await self.kernel.send_signal(signum) + assert self.provisioner is not None + await self.provisioner.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @@ -559,8 +558,8 @@ async def _async_signal_kernel(self, signum: int) -> None: async def _async_is_alive(self) -> bool: """Is the kernel process still running?""" if self.has_kernel: - assert self.kernel is not None - ret = await self.kernel.poll() + assert self.provisioner is not None + ret = await self.provisioner.poll() if ret is None: return True return False diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 675daefc1..e81632457 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -9,7 +9,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple from ..launcher import launch_kernel from ..localinterfaces import is_local_ip @@ -25,14 +24,17 @@ class LocalProvisioner(KernelProvisionerBase): pgid = None ip = None + def has_process(self) -> bool: + return self.process is not None + async def poll(self) -> Optional[int]: - ret = None + ret = 0 if self.process: ret = self.process.poll() return ret async def wait(self) -> Optional[int]: - ret = None + ret = 0 if self.process: # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete @@ -43,7 +45,7 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() - self.process = None + self.process = None # allow has_process to now return False return ret async def send_signal(self, signum: int) -> None: @@ -126,7 +128,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # write connection file / get default ports km.write_connection_file() # TODO - change when handshake pattern is adopted - self.connection_info = km.get_connection_info() + self._connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( extra_arguments=extra_arguments @@ -137,7 +139,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -149,7 +151,6 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProv self.pid = self.process.pid self.pgid = pgid - return self, self.connection_info @staticmethod def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index 6288967db..7eba70894 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -9,7 +9,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple from traitlets.config import Instance # type: ignore from traitlets.config import LoggingConfigurable @@ -30,7 +29,18 @@ class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisione # The kernel specification associated with this provisioner kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) kernel_id: str = Unicode(None, allow_none=True) - connection_info: dict = {} + _connection_info: dict = {} + + @property + def connection_info(self) -> Dict[str, Any]: + """Returns the connection information relative to this provisioner's managed instance""" + return self._connection_info + + @property + @abstractmethod + def has_process(self) -> bool: + """Returns true if this provisioner is currently managing a process.""" + pass @abstractmethod async def poll(self) -> Optional[int]: @@ -70,10 +80,8 @@ async def terminate(self, restart: bool = False) -> None: pass @abstractmethod - async def launch_kernel( - self, cmd: List[str], **kwargs: Any - ) -> Tuple['KernelProvisionerBase', Dict]: - """Launch the kernel process returning the class instance and connection info.""" + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: + """Launch the kernel process. """ pass @abstractmethod @@ -102,8 +110,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """ env = kwargs.pop('env', os.environ).copy() env.update(self.__apply_env_substitutions(env)) - self._validate_parameters(env, **kwargs) - self._finalize_env(env) # TODO: Should finalize be called first? + self._finalize_env(env) kwargs['env'] = env return kwargs @@ -117,7 +124,9 @@ async def get_provisioner_info(self) -> Dict[str, Any]: The superclass method must always be called first to ensure proper ordering. """ - provisioner_info: Dict[str, Any] = {} + provisioner_info: Dict[str, Any] = dict() + provisioner_info['kernel_id'] = self.kernel_id + provisioner_info['connection_info'] = self.connection_info return provisioner_info async def load_provisioner_info(self, provisioner_info: Dict) -> None: @@ -125,7 +134,8 @@ async def load_provisioner_info(self, provisioner_info: Dict) -> None: The superclass method must always be called first to ensure proper ordering. """ - pass + self.kernel_id = provisioner_info['kernel_id'] + self._connection_info = provisioner_info['connection_info'] def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: """Returns the time allowed for a complete shutdown. This may vary by provisioner. @@ -144,10 +154,6 @@ def _finalize_env(self, env: Dict[str, str]) -> None: # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) - def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel spec.""" - pass - def __apply_env_substitutions(self, substitution_values: Dict[str, str]): """Walks entries in the kernelspec's env stanza and applies substitutions from current env. diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 24d108a8a..6a5041460 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -11,7 +11,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple import pytest from entrypoints import EntryPoint @@ -48,19 +47,25 @@ class CustomTestProvisioner(KernelProvisionerBase): config_var_1: int = Int(config=True) config_var_2: str = Unicode(config=True) - async def poll(self) -> [int, None]: + def has_process(self) -> bool: + return self.process is not None + + async def poll(self) -> Optional[int]: + ret = 0 if self.process: - return self.process.poll() - return False + ret = self.process.poll() + return ret - async def wait(self) -> [int, None]: + async def wait(self) -> Optional[int]: + ret = 0 if self.process: while await self.poll() is None: await asyncio.sleep(0.1) # Process is no longer alive, wait and clear - self.process.wait() + ret = self.process.wait() self.process = None + return ret async def send_signal(self, signum: int) -> None: if self.process: @@ -97,7 +102,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # write connection file / get default ports km.write_connection_file() - self.connection_info = km.get_connection_info() + self._connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( extra_arguments=extra_arguments @@ -105,9 +110,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel( - self, cmd: List[str], **kwargs: Any - ) -> Tuple['CustomTestProvisioner', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: scrubbed_kwargs = kwargs self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -119,7 +122,6 @@ async def launch_kernel( self.pid = self.process.pid self.pgid = pgid - return self, self.connection_info async def cleanup(self, restart=False) -> None: pass @@ -268,7 +270,6 @@ async def akm_test(self, kernel_mgr): TestRuntime.validate_provisioner(kernel_mgr) await kernel_mgr.shutdown_kernel() - assert kernel_mgr.kernel is None assert kernel_mgr.provisioner is None @pytest.mark.asyncio @@ -316,8 +317,8 @@ async def test_default_provisioner_config(self, kpf, all_provisioners): @staticmethod def validate_provisioner(akm: AsyncKernelManager): - # Ensure kernel attribute is the provisioner - assert akm.kernel is akm.provisioner + # Ensure the provisioner is managing a process at this point + assert akm.provisioner is not None and akm.provisioner.has_process # Validate provisioner config if akm.kernel_name in ['no_provisioner', 'default_provisioner']: From 516d9df270b2e4603ee0ecd986554cb5fe1c2940 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 7 May 2021 17:34:53 -0700 Subject: [PATCH 23/37] Stop mirroring self.provisioner as self.kernel --- jupyter_client/manager.py | 46 +++++++++---------- .../provisioning/local_provisioner.py | 15 +++--- .../provisioning/provisioner_base.py | 34 ++++++++------ jupyter_client/tests/test_provisioning.py | 29 ++++++------ 4 files changed, 66 insertions(+), 58 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 2d34eff68..baf3e5619 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -252,16 +252,16 @@ def from_ns(match): return [pat.sub(from_ns, arg) for arg in cmd] - async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> KernelProvisionerBase: + async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> None: """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ assert self.provisioner is not None - kernel_proc, connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) + await self.provisioner.launch_kernel(kernel_cmd, **kw) + assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager and write file. - self._force_connection_info(connection_info) - return kernel_proc + self._force_connection_info(self.provisioner.connection_info) _launch_kernel = run_sync(_async_launch_kernel) @@ -331,7 +331,7 @@ async def _async_start_kernel(self, **kw): # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) - self.kernel = await ensure_async(self._launch_kernel(kernel_cmd, **kw)) + await ensure_async(self._launch_kernel(kernel_cmd, **kw)) await self.post_start_kernel(**kw) start_kernel = run_sync(_async_start_kernel) @@ -345,6 +345,7 @@ async def request_shutdown(self, restart: bool = False) -> None: self.session.send(self._control_socket, msg) assert self.provisioner is not None await self.provisioner.shutdown_requested(restart=restart) + self._shutdown_status = _ShutdownStatus.ShutdownRequest async def _async_finish_shutdown( self, @@ -361,7 +362,7 @@ async def _async_finish_shutdown( waittime = max(self.shutdown_wait_time, 0) if self.provisioner: # Allow provisioner to override waittime = self.provisioner.get_shutdown_wait_time(recommended=waittime) - self._shutdown_status = _ShutdownStatus.ShutdownRequest + try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 @@ -381,9 +382,9 @@ async def _async_finish_shutdown( await ensure_async(self._kill_kernel(restart=restart)) else: # Process is no longer alive, wait and clear - if self.kernel is not None: - await self.kernel.wait() - self.kernel = None + if self.has_kernel: + assert self.provisioner is not None + await self.provisioner.wait() finish_shutdown = run_sync(_async_finish_shutdown) @@ -401,7 +402,7 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: if self.provisioner: await self.provisioner.cleanup(restart=restart) - self.provisioner = self.kernel = None + self.provisioner = None cleanup_resources = run_sync(_async_cleanup_resources) @@ -484,14 +485,14 @@ async def _async_restart_kernel(self, now: bool = False, newports: bool = False, @property def has_kernel(self) -> bool: - """Has a kernel been started that we are managing.""" - return self.kernel is not None + """Has a kernel process been started that we are managing.""" + return self.provisioner is not None and self.provisioner.has_process async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: - assert self.kernel is not None - await self.kernel.terminate(restart=restart) + assert self.provisioner is not None + await self.provisioner.terminate(restart=restart) _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm) @@ -501,8 +502,8 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: - assert self.kernel is not None - await self.kernel.kill(restart=restart) + assert self.provisioner is not None + await self.provisioner.kill(restart=restart) # Wait until the kernel terminates. try: @@ -513,9 +514,8 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: pass else: # Process is no longer alive, wait and clear - if self.kernel is not None: - await self.kernel.wait() - self.kernel = None + if self.has_kernel: + await self.provisioner.wait() _kill_kernel = run_sync(_async_kill_kernel) @@ -549,8 +549,8 @@ async def _async_signal_kernel(self, signum: int) -> None: only useful on Unix systems. """ if self.has_kernel: - assert self.kernel is not None - await self.kernel.send_signal(signum) + assert self.provisioner is not None + await self.provisioner.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @@ -559,8 +559,8 @@ async def _async_signal_kernel(self, signum: int) -> None: async def _async_is_alive(self) -> bool: """Is the kernel process still running?""" if self.has_kernel: - assert self.kernel is not None - ret = await self.kernel.poll() + assert self.provisioner is not None + ret = await self.provisioner.poll() if ret is None: return True return False diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 675daefc1..e81632457 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -9,7 +9,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple from ..launcher import launch_kernel from ..localinterfaces import is_local_ip @@ -25,14 +24,17 @@ class LocalProvisioner(KernelProvisionerBase): pgid = None ip = None + def has_process(self) -> bool: + return self.process is not None + async def poll(self) -> Optional[int]: - ret = None + ret = 0 if self.process: ret = self.process.poll() return ret async def wait(self) -> Optional[int]: - ret = None + ret = 0 if self.process: # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete @@ -43,7 +45,7 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() - self.process = None + self.process = None # allow has_process to now return False return ret async def send_signal(self, signum: int) -> None: @@ -126,7 +128,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # write connection file / get default ports km.write_connection_file() # TODO - change when handshake pattern is adopted - self.connection_info = km.get_connection_info() + self._connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( extra_arguments=extra_arguments @@ -137,7 +139,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProvisioner', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -149,7 +151,6 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> Tuple['LocalProv self.pid = self.process.pid self.pgid = pgid - return self, self.connection_info @staticmethod def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index 6288967db..7eba70894 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -9,7 +9,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple from traitlets.config import Instance # type: ignore from traitlets.config import LoggingConfigurable @@ -30,7 +29,18 @@ class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisione # The kernel specification associated with this provisioner kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) kernel_id: str = Unicode(None, allow_none=True) - connection_info: dict = {} + _connection_info: dict = {} + + @property + def connection_info(self) -> Dict[str, Any]: + """Returns the connection information relative to this provisioner's managed instance""" + return self._connection_info + + @property + @abstractmethod + def has_process(self) -> bool: + """Returns true if this provisioner is currently managing a process.""" + pass @abstractmethod async def poll(self) -> Optional[int]: @@ -70,10 +80,8 @@ async def terminate(self, restart: bool = False) -> None: pass @abstractmethod - async def launch_kernel( - self, cmd: List[str], **kwargs: Any - ) -> Tuple['KernelProvisionerBase', Dict]: - """Launch the kernel process returning the class instance and connection info.""" + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: + """Launch the kernel process. """ pass @abstractmethod @@ -102,8 +110,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """ env = kwargs.pop('env', os.environ).copy() env.update(self.__apply_env_substitutions(env)) - self._validate_parameters(env, **kwargs) - self._finalize_env(env) # TODO: Should finalize be called first? + self._finalize_env(env) kwargs['env'] = env return kwargs @@ -117,7 +124,9 @@ async def get_provisioner_info(self) -> Dict[str, Any]: The superclass method must always be called first to ensure proper ordering. """ - provisioner_info: Dict[str, Any] = {} + provisioner_info: Dict[str, Any] = dict() + provisioner_info['kernel_id'] = self.kernel_id + provisioner_info['connection_info'] = self.connection_info return provisioner_info async def load_provisioner_info(self, provisioner_info: Dict) -> None: @@ -125,7 +134,8 @@ async def load_provisioner_info(self, provisioner_info: Dict) -> None: The superclass method must always be called first to ensure proper ordering. """ - pass + self.kernel_id = provisioner_info['kernel_id'] + self._connection_info = provisioner_info['connection_info'] def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: """Returns the time allowed for a complete shutdown. This may vary by provisioner. @@ -144,10 +154,6 @@ def _finalize_env(self, env: Dict[str, str]) -> None: # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) - def _validate_parameters(self, env: Dict[str, str], **kwargs: Any) -> None: - """Future: Validates that launch parameters adhere to schema specified in kernel spec.""" - pass - def __apply_env_substitutions(self, substitution_values: Dict[str, str]): """Walks entries in the kernelspec's env stanza and applies substitutions from current env. diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 24d108a8a..6a5041460 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -11,7 +11,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Tuple import pytest from entrypoints import EntryPoint @@ -48,19 +47,25 @@ class CustomTestProvisioner(KernelProvisionerBase): config_var_1: int = Int(config=True) config_var_2: str = Unicode(config=True) - async def poll(self) -> [int, None]: + def has_process(self) -> bool: + return self.process is not None + + async def poll(self) -> Optional[int]: + ret = 0 if self.process: - return self.process.poll() - return False + ret = self.process.poll() + return ret - async def wait(self) -> [int, None]: + async def wait(self) -> Optional[int]: + ret = 0 if self.process: while await self.poll() is None: await asyncio.sleep(0.1) # Process is no longer alive, wait and clear - self.process.wait() + ret = self.process.wait() self.process = None + return ret async def send_signal(self, signum: int) -> None: if self.process: @@ -97,7 +102,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # write connection file / get default ports km.write_connection_file() - self.connection_info = km.get_connection_info() + self._connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( extra_arguments=extra_arguments @@ -105,9 +110,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel( - self, cmd: List[str], **kwargs: Any - ) -> Tuple['CustomTestProvisioner', Dict]: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: scrubbed_kwargs = kwargs self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -119,7 +122,6 @@ async def launch_kernel( self.pid = self.process.pid self.pgid = pgid - return self, self.connection_info async def cleanup(self, restart=False) -> None: pass @@ -268,7 +270,6 @@ async def akm_test(self, kernel_mgr): TestRuntime.validate_provisioner(kernel_mgr) await kernel_mgr.shutdown_kernel() - assert kernel_mgr.kernel is None assert kernel_mgr.provisioner is None @pytest.mark.asyncio @@ -316,8 +317,8 @@ async def test_default_provisioner_config(self, kpf, all_provisioners): @staticmethod def validate_provisioner(akm: AsyncKernelManager): - # Ensure kernel attribute is the provisioner - assert akm.kernel is akm.provisioner + # Ensure the provisioner is managing a process at this point + assert akm.provisioner is not None and akm.provisioner.has_process # Validate provisioner config if akm.kernel_name in ['no_provisioner', 'default_provisioner']: From 2df20c4fdae33b137961604de426862176826c01 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Tue, 18 May 2021 17:35:11 -0700 Subject: [PATCH 24/37] Move port caching to LocalProvisioner --- jupyter_client/connect.py | 47 +++++++++++++++-- jupyter_client/manager.py | 18 ++----- jupyter_client/multikernelmanager.py | 50 ++++--------------- .../provisioning/local_provisioner.py | 28 ++++++++++- jupyter_client/tests/test_provisioning.py | 3 +- 5 files changed, 84 insertions(+), 62 deletions(-) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 096b1b5bd..1bd6d2354 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -19,6 +19,7 @@ from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Tuple from typing import Union @@ -29,11 +30,13 @@ from traitlets import Bool # type: ignore from traitlets import CaselessStrEnum from traitlets import Instance +from traitlets import Int from traitlets import Integer from traitlets import observe from traitlets import Type from traitlets import Unicode from traitlets.config import LoggingConfigurable # type: ignore +from traitlets.config import SingletonConfigurable from .localinterfaces import localhost from .utils import _filefind @@ -44,11 +47,11 @@ def write_connection_file( fname: Optional[str] = None, - shell_port: int = 0, - iopub_port: int = 0, - stdin_port: int = 0, - hb_port: int = 0, - control_port: int = 0, + shell_port: Union[Integer, Int, int] = 0, + iopub_port: Union[Integer, Int, int] = 0, + stdin_port: Union[Integer, Int, int] = 0, + hb_port: Union[Integer, Int, int] = 0, + control_port: Union[Integer, Int, int] = 0, ip: str = "", key: bytes = b"", transport: str = "tcp", @@ -643,6 +646,40 @@ def connect_control(self, identity: Optional[bytes] = None) -> zmq.sugar.socket. return self._create_connected_socket("control", identity=identity) +class LocalPortCache(SingletonConfigurable): + """ + Used to keep track of local ports in order to prevent race conditions that + can occur between port acquisition and usage by the kernel. All locally- + provisioned kernels should use this mechanism to limit the possibility of + race conditions. Note that this does not preclude other applications from + acquiring a cached but unused port, thereby re-introducing the issue this + class is attempting to resolve (minimize). + See: https://github.com/jupyter/jupyter_client/issues/487 + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.currently_used_ports: Set[int] = set() + + def find_available_port(self, ip: str) -> int: + while True: + tmp_sock = socket.socket() + tmp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b"\0" * 8) + tmp_sock.bind((ip, 0)) + port = tmp_sock.getsockname()[1] + tmp_sock.close() + + # This is a workaround for https://github.com/jupyter/jupyter_client/issues/487 + # We prevent two kernels to have the same ports. + if port not in self.currently_used_ports: + self.currently_used_ports.add(port) + return port + + def return_port(self, port: int) -> None: + if port in self.currently_used_ports: # Tolerate uncached ports + self.currently_used_ports.remove(port) + + __all__ = [ "write_connection_file", "find_connection_file", diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index baf3e5619..8876a7123 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -88,12 +88,6 @@ def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: # The kernel provisioner with which the KernelManager is communicating. # This will generally be a LocalProvisioner instance unless the kernelspec # indicates otherwise. - # Note that we use two attributes, kernel and provisioner, both of which - # will point at the same provisioner instance. `kernel` will be non-None - # during the kernel's lifecycle, while `provisioner` will span that time, - # being set prior to launch and unset following the kernel's termination. - - kernel: t.Optional[KernelProvisionerBase] = None provisioner: t.Optional[KernelProvisionerBase] = None kernel_spec_manager: Instance = Instance(kernelspec.KernelSpecManager) @@ -291,13 +285,12 @@ async def pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str, t.Any and launching the kernel (e.g. Popen kwargs). """ self.kernel_id = self.kernel_id or kw.pop('kernel_id', str(uuid.uuid4())) - self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( - self.kernel_id, self.kernel_spec - ) - # save kwargs for use in restart self._launch_args = kw.copy() - assert self.provisioner is not None + if self.provisioner is None: # will not be None on restarts + self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( + self.kernel_id, self.kernel_spec + ) kw = await self.provisioner.pre_launch(kernel_manager=self, **kw) kernel_cmd = kw.pop('cmd') return kernel_cmd, kw @@ -402,7 +395,6 @@ async def _async_cleanup_resources(self, restart: bool = False) -> None: if self.provisioner: await self.provisioner.cleanup(restart=restart) - self.provisioner = None cleanup_resources = run_sync(_async_cleanup_resources) @@ -485,7 +477,7 @@ async def _async_restart_kernel(self, now: bool = False, newports: bool = False, @property def has_kernel(self) -> bool: - """Has a kernel process been started that we are managing.""" + """Has a kernel process been started that we are actively managing.""" return self.provisioner is not None and self.provisioner.has_process async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index aaf93c55d..f9e25d809 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -65,12 +65,6 @@ class MultiKernelManager(LoggingConfigurable): """, ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Cache all the currently used ports - self.currently_used_ports = set() - @observe("kernel_manager_class") def _kernel_manager_class_changed(self, change): self.kernel_manager_factory = self._create_kernel_manager_factory() @@ -91,33 +85,10 @@ def create_kernel_manager(*args, **kwargs) -> KernelManager: self.context = self._context_default() kwargs.setdefault("context", self.context) km = kernel_manager_ctor(*args, **kwargs) - - if km.cache_ports: - km.shell_port = self._find_available_port(km.ip) - km.iopub_port = self._find_available_port(km.ip) - km.stdin_port = self._find_available_port(km.ip) - km.hb_port = self._find_available_port(km.ip) - km.control_port = self._find_available_port(km.ip) - return km return create_kernel_manager - def _find_available_port(self, ip: str) -> int: - while True: - tmp_sock = socket.socket() - tmp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b"\0" * 8) - tmp_sock.bind((ip, 0)) - port = tmp_sock.getsockname()[1] - tmp_sock.close() - - # This is a workaround for https://github.com/jupyter/jupyter_client/issues/487 - # We prevent two kernels to have the same ports. - if port not in self.currently_used_ports: - self.currently_used_ports.add(port) - - return port - shared_context = Bool( True, config=True, @@ -242,21 +213,9 @@ async def _async_shutdown_kernel( km = self.get_kernel(kernel_id) - ports = ( - km.shell_port, - km.iopub_port, - km.stdin_port, - km.hb_port, - km.control_port, - ) - await ensure_async(km.shutdown_kernel(now, restart)) self.remove_kernel(kernel_id) - if km.cache_ports and not restart: - for port in ports: - self.currently_used_ports.remove(port) - shutdown_kernel = run_sync(_async_shutdown_kernel) @kernel_method @@ -325,6 +284,8 @@ def signal_kernel(self, kernel_id: str, signum: int) -> None: ========== kernel_id : uuid The id of the kernel to signal. + signum : int + Signal number to send kernel. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @@ -336,6 +297,13 @@ def restart_kernel(self, kernel_id: str, now: bool = False) -> None: ========== kernel_id : uuid The id of the kernel to interrupt. + now : bool, optional + If True, the kernel is forcefully restarted *immediately*, without + having a chance to do any cleanup action. Otherwise the kernel is + given 1s to clean up before a forceful restart is issued. + + In all cases the kernel is restarted, the only difference is whether + it is given a chance to perform a clean shutdown or not. """ self.log.info("Kernel restarted: %s" % kernel_id) diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index e81632457..9ed46a13d 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -10,6 +10,7 @@ from typing import List from typing import Optional +from ..connect import LocalPortCache from ..launcher import launch_kernel from ..localinterfaces import is_local_ip from ..localinterfaces import local_ips @@ -23,7 +24,9 @@ class LocalProvisioner(KernelProvisionerBase): pid = None pgid = None ip = None + ports_cached = False + @property def has_process(self) -> bool: return self.process is not None @@ -100,7 +103,18 @@ async def terminate(self, restart: bool = False) -> None: return self.process.terminate() async def cleanup(self, restart: bool = False) -> None: - pass + if self.ports_cached and not restart: + # provisioner is about to be destroyed, return cached ports + lpc = LocalPortCache.instance() + ports = ( + self._connection_info['shell_port'], + self._connection_info['iopub_port'], + self._connection_info['stdin_port'], + self._connection_info['hb_port'], + self._connection_info['control_port'], + ) + for port in ports: + lpc.return_port(port) async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: """Perform any steps in preparation for kernel process launch. @@ -127,7 +141,17 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: extra_arguments = kwargs.pop('extra_arguments', []) # write connection file / get default ports - km.write_connection_file() # TODO - change when handshake pattern is adopted + # TODO - change when handshake pattern is adopted + if km.cache_ports and not self.ports_cached: + lpc = LocalPortCache.instance() + km.shell_port = lpc.find_available_port(km.ip) + km.iopub_port = lpc.find_available_port(km.ip) + km.stdin_port = lpc.find_available_port(km.ip) + km.hb_port = lpc.find_available_port(km.ip) + km.control_port = lpc.find_available_port(km.ip) + self.ports_cached = True + + km.write_connection_file() self._connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 6a5041460..eed92775e 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -47,6 +47,7 @@ class CustomTestProvisioner(KernelProvisionerBase): config_var_1: int = Int(config=True) config_var_2: str = Unicode(config=True) + @property def has_process(self) -> bool: return self.process is not None @@ -270,7 +271,7 @@ async def akm_test(self, kernel_mgr): TestRuntime.validate_provisioner(kernel_mgr) await kernel_mgr.shutdown_kernel() - assert kernel_mgr.provisioner is None + assert kernel_mgr.provisioner.has_process is False @pytest.mark.asyncio async def test_existing(self, kpf, akm): From 314e94a610b240a466afa4f958cbbb312aa8df25 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 21 May 2021 16:15:04 -0700 Subject: [PATCH 25/37] KernelManager is provisioner parent --- jupyter_client/connect.py | 1 + jupyter_client/manager.py | 8 +++++--- jupyter_client/provisioning/factory.py | 7 ++++--- jupyter_client/provisioning/local_provisioner.py | 3 +-- jupyter_client/tests/test_provisioning.py | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 1bd6d2354..0ce120e80 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -684,4 +684,5 @@ def return_port(self, port: int) -> None: "write_connection_file", "find_connection_file", "tunnel_to_kernel", + "LocalPortCache", ] diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 8876a7123..1ce63a5b0 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -83,7 +83,7 @@ def _client_factory_default(self) -> Type: def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: self.client_factory = import_item(str(change["new"])) - kernel_id = None + kernel_id: str = Unicode(None, allow_none=True) # The kernel provisioner with which the KernelManager is communicating. # This will generally be a LocalProvisioner instance unless the kernelspec @@ -289,9 +289,11 @@ async def pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str, t.Any self._launch_args = kw.copy() if self.provisioner is None: # will not be None on restarts self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( - self.kernel_id, self.kernel_spec + self.kernel_id, + self.kernel_spec, + parent=self, ) - kw = await self.provisioner.pre_launch(kernel_manager=self, **kw) + kw = await self.provisioner.pre_launch(**kw) kernel_cmd = kw.pop('cmd') return kernel_cmd, kw diff --git a/jupyter_client/provisioning/factory.py b/jupyter_client/provisioning/factory.py index fb0656ebc..9e3f94549 100644 --- a/jupyter_client/provisioning/factory.py +++ b/jupyter_client/provisioning/factory.py @@ -64,7 +64,7 @@ def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: return is_available def create_provisioner_instance( - self, kernel_id: str, kernel_spec: Any + self, kernel_id: str, kernel_spec: Any, parent: Any ) -> KernelProvisionerBase: """ Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. @@ -88,9 +88,10 @@ def create_provisioner_instance( ) provisioner_class = self.provisioners[provisioner_name].load() provisioner_config = provisioner_cfg.get('config') - return provisioner_class( - kernel_id=kernel_id, kernel_spec=kernel_spec, parent=self.parent, **provisioner_config + provisioner: KernelProvisionerBase = provisioner_class( + kernel_id=kernel_id, kernel_spec=kernel_spec, parent=parent, **provisioner_config ) + return provisioner def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: """ diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 9ed46a13d..f22036321 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -125,9 +125,8 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: Returns the updated kwargs. """ - # If we have a kernel_manager pop it out of the args and use it to retain b/c. # This should be considered temporary until a better division of labor can be defined. - km = kwargs.pop('kernel_manager', None) + km = self.parent if km: if km.transport == 'tcp' and not is_local_ip(km.ip): raise RuntimeError( diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index eed92775e..0c745534f 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -94,7 +94,7 @@ async def terminate(self, restart=False) -> None: self.process.terminate() async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: - km = kwargs.pop('kernel_manager') # TODO - this is temporary + km = self.parent if km: # save kwargs for use in restart km._launch_args = kwargs.copy() From 57c70ffae919bafcce51f4cd71c91adb1fb89827 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 3 Jun 2021 14:44:14 -0700 Subject: [PATCH 26/37] Initial docs update --- docs/api/index.rst | 1 + docs/api/manager.rst | 7 +- docs/api/provisioners.rst | 71 ++++ docs/index.rst | 1 + docs/provisioning.rst | 340 ++++++++++++++++++ jupyter_client/connect.py | 1 + jupyter_client/kernelspec.py | 2 +- jupyter_client/manager.py | 8 +- jupyter_client/provisioning/factory.py | 70 ++-- .../provisioning/local_provisioner.py | 15 +- .../provisioning/provisioner_base.py | 158 ++++++-- jupyter_client/tests/test_provisioning.py | 6 +- 12 files changed, 609 insertions(+), 71 deletions(-) create mode 100644 docs/api/provisioners.rst create mode 100644 docs/provisioning.rst diff --git a/docs/api/index.rst b/docs/api/index.rst index 125646f1e..aace26093 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -10,3 +10,4 @@ jupyter_client API kernelspec manager client + provisioners diff --git a/docs/api/manager.rst b/docs/api/manager.rst index 659f95db0..66194fddb 100644 --- a/docs/api/manager.rst +++ b/docs/api/manager.rst @@ -9,12 +9,11 @@ manager - starting, stopping, signalling The name of the kernel to launch (see :ref:`kernelspecs`). - .. automethod:: start_kernel + .. autoattribute:: provisioner - .. attribute:: kernel + The kernel provisioner with which this :class:`KernelManager` is communicating. This will generally be a :class:`LocalProvisioner` instance unless the kernelspec indicates otherwise. - Once the kernel has been started, this is the :class:`subprocess.Popen` - class for the kernel process. + .. automethod:: start_kernel .. automethod:: is_alive diff --git a/docs/api/provisioners.rst b/docs/api/provisioners.rst new file mode 100644 index 000000000..d468960e9 --- /dev/null +++ b/docs/api/provisioners.rst @@ -0,0 +1,71 @@ +kernel provisioner apis +======================= + +.. seealso:: + + :doc:`/provisioning` + +.. module:: jupyter_client.provisioning.provisioner_base + +.. autoclass:: KernelProvisionerBase + + .. attribute:: kernel_spec + + The kernel specification associated with the provisioned kernel (see :ref:`kernelspecs`). + + .. attribute:: kernel_id + + The provisioned kernel's ID. + + .. attribute:: connection_info + + The provisioned kernel's connection information. + + + .. autoproperty:: has_process + + .. automethod:: poll + + .. automethod:: wait + + .. automethod:: send_signal + + .. automethod:: kill + + .. automethod:: terminate + + .. automethod:: launch_kernel + + .. automethod:: cleanup + + .. automethod:: shutdown_requested + + .. automethod:: pre_launch + + .. automethod:: post_launch + + .. automethod:: get_provisioner_info + + .. automethod:: load_provisioner_info + + .. automethod:: get_shutdown_wait_time + + .. automethod:: _finalize_env + + .. automethod:: __apply_env_substitutions + +.. module:: jupyter_client.provisioning.local_provisioner + +.. autoclass:: LocalProvisioner + +.. module:: jupyter_client.provisioning.factory + +.. autoclass:: KernelProvisionerFactory + + .. attribute:: default_provisioner_name + + Indicates the name of the provisioner to use when no kernel_provisioner entry is present in the kernel specification. This value can also be specified via the environment variable ``JUPYTER_DEFAULT_PROVISIONER_NAME``. + + .. automethod:: is_provisioner_available + + .. automethod:: create_provisioner_instance diff --git a/docs/index.rst b/docs/index.rst index a238aba40..33667e070 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ with Jupyter kernels. kernels wrapperkernels + provisioning .. toctree:: :maxdepth: 2 diff --git a/docs/provisioning.rst b/docs/provisioning.rst new file mode 100644 index 000000000..495f35a81 --- /dev/null +++ b/docs/provisioning.rst @@ -0,0 +1,340 @@ +.. _provisioning + +Customizing the kernel's runtime environment +============================================ + +Kernel Provisioning +~~~~~~~~~~~~~~~~~~~ + +Introduced in the 7.0 release, Kernel Provisioning enables the ability +for third parties to manage the lifecycle of a kernel's runtime +environment. By implementing and configuring a *kernel provisioner*, +third parties now have the ability to provision kernels for different +environments, typically managed by resource managers like Kubernetes, +Hadoop YARN, Slurm, etc. For example, a *Kubernetes Provisioner* would +be responsible for launching a kernel within its own Kubernetes pod, +communicating the kernel's connection information back to the +application (residing in a separate pod), and terminating the pod upon +the kernel's termination. In essence, a kernel provisioner is an +*abstraction layer* between the ``KernelManager`` and today's kernel +*process* (i.e., ``Popen``). + +The kernel manager and kernel provisioner relationship +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Prior to this enhancement, the only extension point for customizing a +kernel's behavior could occur by subclassing ``KernelManager``. This +proved to be a limitation because the Jupyter framework allows for a +single ``KernelManager`` class at any time. While applications could +introduce a ``KernelManager`` subclass of their own, that +``KernelManager`` was then tied directly to *that* application and +thereby not usable as a ``KernelManager`` in another application. As a +result, we consider the ``KernelManager`` class to be an +*application-owned entity* upon which application-specific behaviors can +be implemented. + +Kernel provisioners, on the other hand, are contained within the +``Kernel Manager`` (i.e., a *has-a* relationship) and applications are +agnostic as to what *kind* of provisioner is in use other than what is +conveyed via the kernel's specification (kernelspec). All kernel +interactions still occur via the ``KernelManager`` and ``KernelClient`` +classes within ``jupyter_client`` and potentially subclassed by the +application. + +Kernel provisioners are not related in any way to the ``KernelManager`` +instance that controls their lifecycle, nor do they have any affinity to +the application within which they are used. They merely provide a +vehicle by which authors can extend the landscape in which a kernel can +reside, while not side-effecting the application. That said, some kernel +provisioners may introduce requirements on the application. For example +(and completely hypothetically speaking), a ``SlurmProvisioner`` may +impose the constraint that the server (``jupyter_client``) resides on an +edge node of the Slurm cluster. These kinds of requirements can be +mitigated by leveraging applications like `Jupyter Kernel Gateway `_ or +`Jupyter Enterprise Gateway `_ +where the gateway server resides on the edge +node of (or within) the cluster, etc. + +Discovery +~~~~~~~~~ + +Kernel provisioning does not alter today's kernel discovery mechanism +that utilizes well-known directories of ``kernel.json`` files. Instead, +it optionally extends the current ``metadata`` stanza within the +``kernel.json`` to include the specification of the kernel provisioner +name, along with an optional ``config`` stanza, consisting of +provisioner-specific configuration items. For example, a container-based +provisioner will likely need to specify the image name in this section. +The important point is that the content of this section is +provisioner-specific. + +.. code:: JSON + + "metadata": { + "kernel_provisioner": { + "provisioner_name": "k8s-provisioner", + "config": { + "image_name": "my_docker_org/kernel:2.1.5", + "max_cpus": 4 + } + } + }, + +Kernel provisioner authors implement their provisioners by deriving from +:class:`KernelProvisionerBase` and expose their provisioner for consumption +via entry-points: + +.. code:: python + + 'jupyter_client.kernel_provisioners': [ + 'k8s-provisioner = my_package:K8sProvisioner', + ], + +Backwards Compatibility +~~~~~~~~~~~~~~~~~~~~~~~ + +Prior to this release, no ``kernel.json`` (kernelspec) will contain a +provisioner entry, yet the framework is now based on using provisioners. +As a result, when a ``kernel_provisioner`` stanza is **not** present in +a selected kernelspec, jupyter client will, by default, use the built-in +``LocalProvisioner`` implementation as its provisioner. This provisioner +retains today's local kernel functionality. It can also be subclassed +for those provisioner authors wanting to extend the functionality of +local kernels. The result of launching a kernel in this manner is +equivalent to the following stanza existing in the ``kernel.json`` file: + +.. code:: JSON + + "metadata": { + "kernel_provisioner": { + "provisioner_name": "local-provisioner", + "config": { + } + } + }, + +Should a given installation wish to use a *different* provisioner as +their "default provisioner" (including subclasses of +``LocalProvisioner``), they can do so by specifying a value for +``KernelProvisionerFactory.default_provisioner_name``. + +Implementing a custom provisioner +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The impact of Kernel Provisioning is that it enables the ability to +implement custom kernel provisioners to manage a kernel's lifecycle +within any runtime environment. There are currently two approaches by +which that can be accomplished, extending the ``KernelProvisionerBase`` +class or extending the built-in class - ``LocalProvisioner``. As more +provisioners are introduced, some may be implemented in an abstract +sense, from which specific implementations can be authored. + +Extending ``LocalProvisioner`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you're interested in running kernels locally and yet adjust their +behavior, there's a good chance you can simply extend +``LocalProvisioner`` via subclassing. This amounts to deriving from +``LocalProvisioner`` and overriding appropriate methods to provide your +custom functionality. + +In this example, RBACProvisioner will verify whether the current user is +in the role meant for this kernel. If not, an exception will be thrown. + +.. code:: python + + class RBACProvisioner(LocalProvisioner): + + role: str = Unicode(config=True) + + async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: + + if not self.user_in_role(self.role): + raise PermissionError(f"User is not in role {self.role} and " + f"cannot launch this kernel.") + + return super().pre_launch(**kwargs) + +It is important to note *when* it's necessary to call the superclass in +a given method - since the operations it performs may be critical to the +kernel's management. As a result, you'll likely need to become familiar +with how ``LocalProvisioner`` operates. + +Extending ``KernelProvisionerBase`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you'd like to launch your kernel in an environment other than the +local server, then you will need to consider subclassing :class:`KernelProvisionerBase` +directly. This will allow you to implement the various kernel process +controls relative to your target environment. For instance, if you +wanted to have your kernel hosted in a Hadoop YARN cluster, you will +need to implement process-control methods like :meth:`poll` and :meth:`wait` +to use the YARN REST API. Or, similarly, a Kubernetes-based provisioner +would need to implement the process-control methods using the Kubernetes client +API, etc. + +By modeling the :class:`KernelProvisionerBase` methods after :class:`subprocess.Popen` +a natural mapping between today's kernel lifecycle management takes place. This, +coupled with the ability to add configuration directly into the ``config:`` stanza +of the ``kernel_provisioner`` metadata, allows for things like endpoint address, +image names, namespaces, hosts lists, etc. to be specified relative to your +kernel provisioner implementation. + +The ``kernel_id`` corresponding to the launched kernel and used by the +kernel manager is now available *prior* to the kernel's launch. This +enables provisioners with a unique *key* they can use to discover and +control their kernel when launched into resource-managed clusters such +as Hadoop YARN or Kubernetes. + +.. tip:: + Use ``kernel_id`` as a discovery mechanism from your provisioner! + +Here's a prototyped implementation of a couple of the abstract methods +of :class:`KernelProvisionerBase` for use in an Hadoop YARN cluster to +help illustrate a provisioner's implementation. Note that the built-in +implementation of :class:`LocalProvisioner` can also be used as a reference. + +Notice the internal method `_get_application_id()`. This method is +what the provisioner uses to determine the YARN application (i.e., +the kernel) is still running within te cluster. Although the provisioner +doesn't dictate the application id, the application id is +discovered via the application *name* which is a function of ``kernel_id``. + +.. code:: python + + async def poll(self) -> Optional[int]: + """Submitting a new kernel/app to YARN will take a while to be ACCEPTED. + Thus application ID will probably not be available immediately for poll. + So will regard the application as RUNNING when application ID still in + ACCEPTED or SUBMITTED state. + + :return: None if the application's ID is available and state is + ACCEPTED/SUBMITTED/RUNNING. Otherwise 0. + """ + result = 0 + if self._get_application_id(): + state = self._query_app_state_by_id(self.application_id) + if state in YarnProvisioner.initial_states: + result = None + + return result + + + async def send_signal(self, signum): + """Currently only support 0 as poll and other as kill. + + :param signum + :return: + """ + if signum == 0: + return await self.poll() + elif signum == signal.SIGKILL: + return await self.kill() + else: + return await super().send_signal(signum) + +Notice how in some cases we can compose provisioner methods to implement others. For +example, since sending a signal number of 0 is tantamount to polling the process, we +go ahead and call :meth:`poll` to handle `signum` of 0 and :meth:`kill` to handle +`SIGKILL` requests. + +Here we see how ``_get_application_id`` uses the ``kernel_id`` to acquire the application +id - which is the *primary id* for controlling YARN application lifecycles. Since startup +in resource-managed clusters can tend to take much longer than local kernels, you'll typically +need a polling or notification mechanism within your provisioner. In addition, your +provisioner will be asked by the ``KernelManager`` what is an acceptable startup time. +This answer is implemented in the provisioner via the :meth:`get_shutdown_wait_time` method. + +.. code:: python + + def _get_application_id(self, ignore_final_states: bool = False) -> str: + + if not self.application_id: + app = self._query_app_by_name(self.kernel_id) + state_condition = True + if type(app) is dict: + state = app.get('state') + self.last_known_state = state + + if ignore_final_states: + state_condition = state not in YarnProvisioner.final_states + + if len(app.get('id', '')) > 0 and state_condition: + self.application_id = app['id'] + self.log.info(f"ApplicationID: '{app['id']}' assigned for " + f"KernelID: '{self.kernel_id}', state: {state}.") + if not self.application_id: + self.log.debug(f"ApplicationID not yet assigned for KernelID: " + f"'{self.kernel_id}' - retrying...") + return self.application_id + + + def get_shutdown_wait_time(self, recommended: Optional[float] = 5.0) -> float: + + if recommended < yarn_shutdown_wait_time: + recommended = yarn_shutdown_wait_time + self.log.debug(f"{type(self).__name__} shutdown wait time adjusted to " + f"{recommended} seconds.") + + return recommended + +Registering your custom provisioner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once your custom provisioner has been authored, it needs to be exposed +as an +`entry point `_. +To do this add the following to your ``setup.py`` (or equivalent) in its +``entry_points`` stanza using the group name +``jupyter_client.kernel_provisioners``: + +:: + + 'jupyter_client.kernel_provisioners': [ + 'rbac-provisioner = acme.rbac.provisioner:RBACProvisioner', + ], + +where: + +- ``rbac-provisioner`` is the *name* of your provisioner and what will + be referenced within the ``kernel.json`` file +- ``acme.rbac.provisioner`` identifies the location of the provisioner + class, and +- ``provisioner:RBACProvisioner`` is the name of the custom provisioner + implementation that (directly or indirectly) derives from + ``KernelProvisionerBase`` + +Deploying your custom provisioner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The final step in getting your custom provisioner deployed is to add a +``kernel_provisioner`` stanza to the appropriate ``kernel.json`` files. +This can be accomplished manually or programmatically (in which some +tooling is implemented to create the appropriate ``kernel.json`` file). +In either case, the end result is the same - a ``kernel.json`` file with +the appropriate stanza within ``metadata``. The *vision* is that kernel +provisioner packages will include an application that creates kernel +specifications (i.e., ``kernel.json`` et. al.) pertaining to that +provisioner. + +Following on the previous example of ``RBACProvisioner``, one would find +the following ``kernel.json`` file in directory +``/usr/local/share/jupyter/kernels/rbac_kernel``: + +.. code:: JSON + + { + "argv": ["python", "-m", "ipykernel_launcher", "-f", "{connection_file}"], + "env": {}, + "display_name": "RBAC Kernel", + "language": "python", + "interrupt_mode": "signal", + "metadata": { + "kernel_provisioner": { + "provisioner_name": "rbac-provisioner", + "config": { + "role": "data_scientist" + } + } + } + } diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 0ce120e80..29ce180e4 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -684,5 +684,6 @@ def return_port(self, port: int) -> None: "write_connection_file", "find_connection_file", "tunnel_to_kernel", + "KernelConnectionInfo", "LocalPortCache", ] diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 503d124df..c6fa36f26 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -219,7 +219,7 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): if not kspec: kspec = self.kernel_spec_class.from_resource_dir(resource_dir) - if not KPF.instance(parent=self.parent).is_provisioner_available(kernel_name, kspec): + if not KPF.instance(parent=self.parent).is_provisioner_available(kspec): raise NoSuchKernel(kernel_name) return kspec diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 1ce63a5b0..fe23d3acd 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -85,7 +85,7 @@ def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: kernel_id: str = Unicode(None, allow_none=True) - # The kernel provisioner with which the KernelManager is communicating. + # The kernel provisioner with which this KernelManager is communicating. # This will generally be a LocalProvisioner instance unless the kernelspec # indicates otherwise. provisioner: t.Optional[KernelProvisionerBase] = None @@ -250,12 +250,14 @@ async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> None: """actually launch the kernel override in a subclass to launch kernel subprocesses differently + Note that provisioners can now be used to customize kernel environments + and """ assert self.provisioner is not None - await self.provisioner.launch_kernel(kernel_cmd, **kw) + connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager and write file. - self._force_connection_info(self.provisioner.connection_info) + self._force_connection_info(connection_info) _launch_kernel = run_sync(_async_launch_kernel) diff --git a/jupyter_client/provisioning/factory.py b/jupyter_client/provisioning/factory.py index 9e3f94549..0e1ccde2d 100644 --- a/jupyter_client/provisioning/factory.py +++ b/jupyter_client/provisioning/factory.py @@ -18,7 +18,19 @@ class KernelProvisionerFactory(SingletonConfigurable): - """KernelProvisionerFactory is responsible for creating provisioner instances.""" + """ + :class:`KernelProvisionerFactory` is responsible for creating provisioner instances. + + A singleton instance, `KernelProvisionerFactory` is also used by the :class:`KernelSpecManager` + to validate `kernel_provisioner` references found in kernel specifications to confirm their + availability (in cases where the kernel specification references a kernel provisioner that has + not been installed into the current Python environment). + + It's `default_provisioner_name` attribute can be used to specify the default provisioner + to use when a kernel_spec is found to not reference a provisioner. It's value defaults to + `"local-provisioner"` which identifies the local provisioner implemented by + :class:`LocalProvisioner`. + """ GROUP_NAME = 'jupyter_client.kernel_provisioners' provisioners: Dict[str, EntryPoint] = {} @@ -40,44 +52,41 @@ def __init__(self, **kwargs) -> None: for ep in KernelProvisionerFactory._get_all_provisioners(): self.provisioners[ep.name] = ep - def is_provisioner_available(self, kernel_name: str, kernel_spec: Any) -> bool: + def is_provisioner_available(self, kernel_spec: Any) -> bool: """ - Reads the associated kernel_spec to determine the provisioner and returns whether it + Reads the associated ``kernel_spec`` to determine the provisioner and returns whether it exists as an entry_point (True) or not (False). If the referenced provisioner is not - in the current set of provisioners, attempt to retrieve its entrypoint. If found, add - to the list, else catch exception and return false. + in the current cache or cannot be loaded via entry_points, a warning message is issued + indicating it is not available. """ is_available: bool = True provisioner_cfg = self._get_provisioner_config(kernel_spec) provisioner_name = str(provisioner_cfg.get('provisioner_name')) - if provisioner_name not in self.provisioners: - try: - ep = KernelProvisionerFactory._get_provisioner(provisioner_name) - self.provisioners[provisioner_name] = ep # Update cache - except NoSuchEntryPoint: - is_available = False - self.log.warning( - f"Kernel '{kernel_name}' is referencing a kernel provisioner " - f"('{provisioner_name}') that is not available. Ensure the " - f"appropriate package has been installed and retry." - ) + if not self._check_availability(provisioner_name): + is_available = False + self.log.warning( + f"Kernel '{kernel_spec.display_name}' is referencing a kernel " + f"provisioner ('{provisioner_name}') that is not available. " + f"Ensure the appropriate package has been installed and retry." + ) return is_available def create_provisioner_instance( self, kernel_id: str, kernel_spec: Any, parent: Any ) -> KernelProvisionerBase: """ - Reads the associated kernel_spec and to see if has a kernel_provisioner stanza. + Reads the associated ``kernel_spec`` to see if it has a `kernel_provisioner` stanza. If one exists, it instantiates an instance. If a kernel provisioner is not - specified in the kernelspec, a DEFAULT_PROVISIONER stanza is fabricated and instantiated. + specified in the kernel specification, a default provisioner stanza is fabricated + and instantiated corresponding to the current value of `default_provisioner_name` trait. The instantiated instance is returned. If the provisioner is found to not exist (not registered via entry_points), - ModuleNotFoundError is raised. + `ModuleNotFoundError` is raised. """ provisioner_cfg = self._get_provisioner_config(kernel_spec) - provisioner_name = provisioner_cfg.get('provisioner_name') - if provisioner_name not in self.provisioners: + provisioner_name = str(provisioner_cfg.get('provisioner_name')) + if not self._check_availability(provisioner_name): raise ModuleNotFoundError( f"Kernel provisioner '{provisioner_name}' has not been registered." ) @@ -93,6 +102,25 @@ def create_provisioner_instance( ) return provisioner + def _check_availability(self, provisioner_name: str) -> bool: + """ + Checks that the given provisioner is available. + + If the given provisioner is not in the current set of loaded provisioners an attempt + is made to fetch the named entry point and, if successful, loads it into the cache. + + :param provisioner_name: + :return: + """ + is_available = True + if provisioner_name not in self.provisioners: + try: + ep = KernelProvisionerFactory._get_provisioner(provisioner_name) + self.provisioners[provisioner_name] = ep # Update cache + except NoSuchEntryPoint: + is_available = False + return is_available + def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: """ Return the kernel_provisioner stanza from the kernel_spec. diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index f22036321..224a8aadf 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -10,6 +10,7 @@ from typing import List from typing import Optional +from ..connect import KernelConnectionInfo from ..connect import LocalPortCache from ..launcher import launch_kernel from ..localinterfaces import is_local_ip @@ -18,6 +19,16 @@ class LocalProvisioner(KernelProvisionerBase): + """ + :class:`LocalProvisioner` is a concrete class of ABC :py:class:`KernelProvisionerBase` + and is the out-of-box default implementation used when no kernel provisioner is + specified in the kernel specification (``kernel.json``). It provides functional + parity to existing applications by launching the kernel locally and using + :class:`subprocess.Popen` to manage its lifecycle. + + This class is intended to be subclassed for customizing local kernel environments + and serve as a reference implementation for other custom provisioners. + """ process = None _exit_future = None @@ -31,6 +42,7 @@ def has_process(self) -> bool: return self.process is not None async def poll(self) -> Optional[int]: + ret = 0 if self.process: ret = self.process.poll() @@ -162,7 +174,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: scrubbed_kwargs = LocalProvisioner._scrub_kwargs(kwargs) self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -174,6 +186,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: self.pid = self.process.pid self.pgid = pgid + return self._connection_info @staticmethod def _scrub_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: diff --git a/jupyter_client/provisioning/provisioner_base.py b/jupyter_client/provisioning/provisioner_base.py index 7eba70894..2ff350419 100644 --- a/jupyter_client/provisioning/provisioner_base.py +++ b/jupyter_client/provisioning/provisioner_base.py @@ -14,99 +14,150 @@ from traitlets.config import LoggingConfigurable from traitlets.config import Unicode +from ..connect import KernelConnectionInfo + class KernelProvisionerMeta(ABCMeta, type(LoggingConfigurable)): # type: ignore pass class KernelProvisionerBase(ABC, LoggingConfigurable, metaclass=KernelProvisionerMeta): - """Base class defining methods for KernelProvisioner classes. + """ + Abstract base class defining methods for KernelProvisioner classes. + + A majority of methods are abstract (requiring implementations via a subclass) while + some are optional and others provide implementations common to all instances. + Subclasses should be aware of which methods require a call to the superclass. - Theses methods model those of the Subprocess Popen class: - https://docs.python.org/3/library/subprocess.html#popen-objects + Many of these methods model those of :class:`subprocess.Popen` for parity with + previous versions where the kernel process was managed directly. """ # The kernel specification associated with this provisioner kernel_spec: Any = Instance('jupyter_client.kernelspec.KernelSpec', allow_none=True) kernel_id: str = Unicode(None, allow_none=True) - _connection_info: dict = {} - - @property - def connection_info(self) -> Dict[str, Any]: - """Returns the connection information relative to this provisioner's managed instance""" - return self._connection_info + connection_info: KernelConnectionInfo = {} @property @abstractmethod def has_process(self) -> bool: - """Returns true if this provisioner is currently managing a process.""" + """ + Returns true if this provisioner is currently managing a process. + + This property is asserted to be True immediately following a call to + the provisioner's :meth:`launch_kernel` method. + """ pass @abstractmethod async def poll(self) -> Optional[int]: - """Checks if kernel process is still running. + """ + Checks if kernel process is still running. If running, None is returned, otherwise the process's integer-valued exit code is returned. + This method is called from :meth:`KernelManager.is_alive`. """ pass @abstractmethod async def wait(self) -> Optional[int]: - """Waits for kernel process to terminate.""" + """ + Waits for kernel process to terminate. + + This method is called from `KernelManager.finish_shutdown()` and + `KernelManager.kill_kernel()` when terminating a kernel gracefully or + immediately, respectively. + """ pass @abstractmethod async def send_signal(self, signum: int) -> None: - """Sends signal identified by signum to the kernel process.""" + """ + Sends signal identified by signum to the kernel process. + + This method is called from `KernelManager.signal_kernel()` to send the + kernel process a signal. + """ pass @abstractmethod async def kill(self, restart: bool = False) -> None: - """Kills the kernel process. This is typically accomplished via a SIGKILL signal, - which cannot be caught. + """ + Kill the kernel process. - restart is True if this operation precedes a start launch_kernel request. + This is typically accomplished via a SIGKILL signal, which cannot be caught. + This method is called from `KernelManager.kill_kernel()` when terminating + a kernel immediately. + + restart is True if this operation will precede a subsequent launch_kernel request. """ pass @abstractmethod async def terminate(self, restart: bool = False) -> None: - """Terminates the kernel process. This is typically accomplished via a SIGTERM signal, - which can be caught, allowing the kernel provisioner to perform possible cleanup - of resources. + """ + Terminates the kernel process. + + This is typically accomplished via a SIGTERM signal, which can be caught, allowing + the kernel provisioner to perform possible cleanup of resources. This method is + called indirectly from `KernelManager.finish_shutdown()` during a kernel's + graceful termination. restart is True if this operation precedes a start launch_kernel request. """ pass @abstractmethod - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: - """Launch the kernel process. """ + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: + """ + Launch the kernel process and return its connection information. + + This method is called from `KernelManager.launch_kernel()` during the + kernel manager's start kernel sequence. + """ pass @abstractmethod async def cleanup(self, restart: bool = False) -> None: - """Cleanup any resources allocated on behalf of the kernel provisioner. + """ + Cleanup any resources allocated on behalf of the kernel provisioner. + + This method is called from `KernelManager.cleanup_resources()` as part of + its shutdown kernel sequence. restart is True if this operation precedes a start launch_kernel request. """ pass async def shutdown_requested(self, restart: bool = False) -> None: - """Called after KernelManager sends a `shutdown_request` message to kernel. + """ + Allows the provisioner to determine if the kernel's shutdown has been requested. + + This method is called from `KernelManager.request_shutdown()` as part of + its shutdown sequence. This method is optional and is primarily used in scenarios where the provisioner - communicates with a sibling (nanny) process to the kernel. + may need to perform other operations in preparation for a kernel's shutdown. """ pass async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: - """Perform any steps in preparation for kernel process launch. + """ + Perform any steps in preparation for kernel process launch. - This includes applying additional substitutions to the kernel launch command and env. - It also includes preparation of launch parameters. + This includes applying additional substitutions to the kernel launch command + and environment. It also includes preparation of launch parameters. - Returns potentially updated kwargs. + NOTE: Subclass implementations are advised to call this method as it applies + environment variable substitutions from the local environment and calls the + provisioner's :meth:`_finalize_env()` method to allow each provisioner the + ability to cleanup the environment variables that will be used by the kernel. + + This method is called from `KernelManager.pre_start_kernel()` as part of its + start kernel sequence. + + Returns the (potentially updated) keyword arguments that are passed to + :meth:`launch_kernel()`. """ env = kwargs.pop('env', os.environ).copy() env.update(self.__apply_env_substitutions(env)) @@ -116,13 +167,23 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return kwargs async def post_launch(self, **kwargs: Any) -> None: - """Perform any steps following the kernel process launch.""" + """ + Perform any steps following the kernel process launch. + + This method is called from `KernelManager.post_start_kernel()` as part of its + start kernel sequence. + """ pass async def get_provisioner_info(self) -> Dict[str, Any]: - """Captures the base information necessary for persistence relative to this instance. + """ + Captures the base information necessary for persistence relative to this instance. + + This enables applications that subclass `KernelManager` to persist a kernel provisioner's + relevant information to accomplish functionality like disaster recovery or high availability + by calling this method via the kernel manager's `provisioner` attribute. - The superclass method must always be called first to ensure proper ordering. + NOTE: The superclass method must always be called first to ensure proper serialization. """ provisioner_info: Dict[str, Any] = dict() provisioner_info['kernel_id'] = self.kernel_id @@ -130,24 +191,38 @@ async def get_provisioner_info(self) -> Dict[str, Any]: return provisioner_info async def load_provisioner_info(self, provisioner_info: Dict) -> None: - """Loads the base information necessary for persistence relative to this instance. + """ + Loads the base information necessary for persistence relative to this instance. - The superclass method must always be called first to ensure proper ordering. + The inverse of `get_provisioner_info()`, this enables applications that subclass + `KernelManager` to re-establish communication with a provisioner that is managing + a (presumably) remote kernel from an entirely different process that the original + provisioner. + + NOTE: The superclass method must always be called first to ensure proper deserialization. """ self.kernel_id = provisioner_info['kernel_id'] - self._connection_info = provisioner_info['connection_info'] + self.connection_info = provisioner_info['connection_info'] def get_shutdown_wait_time(self, recommended: float = 5.0) -> float: - """Returns the time allowed for a complete shutdown. This may vary by provisioner. + """ + Returns the time allowed for a complete shutdown. This may vary by provisioner. + + This method is called from `KernelManager.finish_shutdown()` during the graceful + phase of its kernel shutdown sequence. The recommended value will typically be what is configured in the kernel manager. """ return recommended def _finalize_env(self, env: Dict[str, str]) -> None: - """Ensures env is appropriate prior to launch. + """ + Ensures env is appropriate prior to launch. + + This method is called from `KernelProvisionerBase.pre_launch()` during the kernel's + start sequence. - Subclasses should be sure to call super()._finalize_env(env) + NOTE: Subclasses should be sure to call super()._finalize_env(env) """ if self.kernel_spec.language and self.kernel_spec.language.lower().startswith("python"): # Don't allow PYTHONEXECUTABLE to be passed to kernel process. @@ -155,10 +230,15 @@ def _finalize_env(self, env: Dict[str, str]) -> None: env.pop('PYTHONEXECUTABLE', None) def __apply_env_substitutions(self, substitution_values: Dict[str, str]): - """Walks entries in the kernelspec's env stanza and applies substitutions from current env. + """ + Walks entries in the kernelspec's env stanza and applies substitutions from current env. + + This method is called from `KernelProvisionerBase.pre_launch()` during the kernel's + start sequence. Returns the substituted list of env entries. - Note: This method is private and is not intended to be overridden by provisioners. + + NOTE: This method is private and is not intended to be overridden by provisioners. """ substituted_env = {} if self.kernel_spec: diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 0c745534f..197c08f87 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -19,6 +19,7 @@ from traitlets import Int from traitlets import Unicode +from ..connect import KernelConnectionInfo from ..kernelspec import KernelSpecManager from ..kernelspec import NoSuchKernel from ..launcher import launch_kernel @@ -103,7 +104,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: # write connection file / get default ports km.write_connection_file() - self._connection_info = km.get_connection_info() + self.connection_info = km.get_connection_info() kernel_cmd = km.format_kernel_cmd( extra_arguments=extra_arguments @@ -111,7 +112,7 @@ async def pre_launch(self, **kwargs: Any) -> Dict[str, Any]: return await super().pre_launch(cmd=kernel_cmd, **kwargs) - async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: + async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> KernelConnectionInfo: scrubbed_kwargs = kwargs self.process = launch_kernel(cmd, **scrubbed_kwargs) pgid = None @@ -123,6 +124,7 @@ async def launch_kernel(self, cmd: List[str], **kwargs: Any) -> None: self.pid = self.process.pid self.pgid = pgid + return self.connection_info async def cleanup(self, restart=False) -> None: pass From 0efc1b4dec7e31a6edaa30fda22418adb8009bae Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 18 Jun 2021 14:52:24 -0700 Subject: [PATCH 27/37] Add ability to list available kernel provisioners --- docs/provisioning.rst | 23 +++++++++++++++++++---- jupyter_client/kernelspecapp.py | 18 ++++++++++++++++++ jupyter_client/provisioning/factory.py | 12 ++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index 495f35a81..0fd32abd6 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -298,10 +298,9 @@ where: - ``rbac-provisioner`` is the *name* of your provisioner and what will be referenced within the ``kernel.json`` file -- ``acme.rbac.provisioner`` identifies the location of the provisioner - class, and -- ``provisioner:RBACProvisioner`` is the name of the custom provisioner - implementation that (directly or indirectly) derives from +- ``acme.rbac.provisioner`` identifies the provisioner module name, and +- ``RBACProvisioner`` is custom provisioner object name + (implementation) that (directly or indirectly) derives from ``KernelProvisionerBase`` Deploying your custom provisioner @@ -338,3 +337,19 @@ the following ``kernel.json`` file in directory } } } + +Listing available kernel provisioners +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To confirm that your custom provisioner is available for use, +the ``jupyter kernelspec`` command has been extended to include +a `provisioners` sub-command. As a result, running ``jupyter kernelspec provisioners`` +will list the available provisioners by name followed by their module and object +names (colon-separated): + +.. code:: bash + + $ jupyter kernelspec provisioners + + Available kernel provisioners: + local-provisioner jupyter_client.provisioning:LocalProvisioner + rbac-provisioner acme.rbac.provisioner:RBACProvisioner diff --git a/jupyter_client/kernelspecapp.py b/jupyter_client/kernelspecapp.py index da310d3be..7ade0de2a 100644 --- a/jupyter_client/kernelspecapp.py +++ b/jupyter_client/kernelspecapp.py @@ -17,6 +17,7 @@ from . import __version__ from .kernelspec import KernelSpecManager +from .provisioning.factory import KernelProvisionerFactory class ListKernelSpecs(JupyterApp): @@ -270,6 +271,22 @@ def start(self): self.exit(e) +class ListProvisioners(JupyterApp): + version = __version__ + description = """List available provisioners for use in kernel specifications.""" + + def start(self): + kfp = KernelProvisionerFactory(parent=self).instance() + print("Available kernel provisioners:") + provisioners = kfp.get_provisioner_entries() + + # pad to width of longest kernel name + name_len = len(sorted(provisioners, key=lambda name: len(name))[-1]) + + for name in sorted(provisioners): + print(f" {name.ljust(name_len)} {provisioners[name]}") + + class KernelSpecApp(Application): version = __version__ name = "jupyter kernelspec" @@ -288,6 +305,7 @@ class KernelSpecApp(Application): InstallNativeKernelSpec, InstallNativeKernelSpec.description.splitlines()[0], ), + "provisioners": (ListProvisioners, ListProvisioners.description.splitlines()[0]), } ) diff --git a/jupyter_client/provisioning/factory.py b/jupyter_client/provisioning/factory.py index 0e1ccde2d..212e03625 100644 --- a/jupyter_client/provisioning/factory.py +++ b/jupyter_client/provisioning/factory.py @@ -150,6 +150,18 @@ def _get_provisioner_config(self, kernel_spec: Any) -> Dict[str, Any]: return env_provisioner # Return what we found (plus config stanza if necessary) return {"provisioner_name": self.default_provisioner_name, "config": {}} + def get_provisioner_entries(self) -> Dict[str, str]: + """ + Returns a dictionary of provisioner entries. + + The key is the provisioner name for its entry point. The value is the colon-separated + string of the entry point's module name and object name. + """ + entries = {} + for name, ep in self.provisioners.items(): + entries[name] = f"{ep.module_name}:{ep.object_name}" + return entries + @staticmethod def _get_all_provisioners() -> List[EntryPoint]: """Wrapper around entrypoints.get_group_all() - primarily to facilitate testing.""" From 797360bb104dcd9c62ce95e0ef8cd48b042593e4 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Thu, 24 Jun 2021 13:23:22 -0700 Subject: [PATCH 28/37] Tolerate exception within excepton handler --- jupyter_client/launcher.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index eec1354d5..818b4c81a 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -155,14 +155,18 @@ def launch_kernel( # Allow to use ~/ in the command or its arguments cmd = [os.path.expanduser(s) for s in cmd] proc = Popen(cmd, **kwargs) - except Exception: - msg = "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" - # exclude environment variables, - # which may contain access tokens and the like. - without_env = {key: value for key, value in kwargs.items() if key != "env"} - msg = msg.format(cmd, env.get("PATH", os.defpath), without_env) - get_logger().error(msg) - raise + except Exception as ex: + try: + msg = "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" + # exclude environment variables, + # which may contain access tokens and the like. + without_env = {key: value for key, value in kwargs.items() if key != "env"} + msg = msg.format(cmd, env.get("PATH", os.defpath), without_env) + get_logger().error(msg) + except Exception as ex2: # Don't let a formatting/logger issue lead to the wrong exception + print(f"Failed to run command: '{cmd}' due to exception: {ex}") + print(f"The following exception occurred handling the previous failure: {ex2}") + raise ex if sys.platform == "win32": # Attach the interrupt event to the Popen objet so it can be used later. From d383cdbbdf4e4fc4e34b2c381dacf396ac5688d2 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Wed, 30 Jun 2021 11:38:49 -0700 Subject: [PATCH 29/37] Cleanup messaging when NoSuchKernel is raised --- jupyter_client/kernelspec.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index c6fa36f26..99b35447d 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -248,13 +248,12 @@ def get_kernel_spec(self, kernel_name): """ if not _is_valid_kernel_name(kernel_name): self.log.warning( - "Kernelspec name %r is invalid: %s", - kernel_name, - _kernel_name_description, + f"Kernelspec name {kernel_name} is invalid: {_kernel_name_description}" ) resource_dir = self._find_spec_directory(kernel_name.lower()) if resource_dir is None: + self.log.warning(f"Kernelspec name {kernel_name} cannot be found!") raise NoSuchKernel(kernel_name) return self._get_kernel_spec_by_name(kernel_name, resource_dir) @@ -285,6 +284,8 @@ def get_all_specs(self): spec = self.get_kernel_spec(kname) res[kname] = {"resource_dir": resource_dir, "spec": spec.to_dict()} + except NoSuchKernel: + pass # The appropriate warning has already been logged except Exception: self.log.warning("Error loading kernelspec %r", kname, exc_info=True) return res From 489b6b2c444ff504b27d393671191f0976d8dd01 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 9 Jul 2021 04:03:21 -0500 Subject: [PATCH 30/37] Update docs/provisioning.rst Co-authored-by: David Brochart --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index 0fd32abd6..3be93666c 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -194,7 +194,7 @@ of :class:`KernelProvisionerBase` for use in an Hadoop YARN cluster to help illustrate a provisioner's implementation. Note that the built-in implementation of :class:`LocalProvisioner` can also be used as a reference. -Notice the internal method `_get_application_id()`. This method is +Notice the internal method ``_get_application_id()``. This method is what the provisioner uses to determine the YARN application (i.e., the kernel) is still running within te cluster. Although the provisioner doesn't dictate the application id, the application id is From 31dcb8818cea5c145a7d7d1f0ebbe1322f833551 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 9 Jul 2021 04:03:30 -0500 Subject: [PATCH 31/37] Update docs/provisioning.rst Co-authored-by: David Brochart --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index 3be93666c..dc96fcae1 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -195,7 +195,7 @@ help illustrate a provisioner's implementation. Note that the built-in implementation of :class:`LocalProvisioner` can also be used as a reference. Notice the internal method ``_get_application_id()``. This method is -what the provisioner uses to determine the YARN application (i.e., +what the provisioner uses to determine if the YARN application (i.e., the kernel) is still running within te cluster. Although the provisioner doesn't dictate the application id, the application id is discovered via the application *name* which is a function of ``kernel_id``. From cc2130fa3c8236bc153fbc430c1d9535883f907e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 9 Jul 2021 04:03:41 -0500 Subject: [PATCH 32/37] Update docs/provisioning.rst Co-authored-by: David Brochart --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index dc96fcae1..c375ad01d 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -34,7 +34,7 @@ result, we consider the ``KernelManager`` class to be an be implemented. Kernel provisioners, on the other hand, are contained within the -``Kernel Manager`` (i.e., a *has-a* relationship) and applications are +``KernelManager`` (i.e., a *has-a* relationship) and applications are agnostic as to what *kind* of provisioner is in use other than what is conveyed via the kernel's specification (kernelspec). All kernel interactions still occur via the ``KernelManager`` and ``KernelClient`` From fe75742ca63dd3c8f2d338c265d05b285acda091 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 9 Jul 2021 06:45:16 -0700 Subject: [PATCH 33/37] Update jupyter_client/tests/test_provisioning.py Fix typo. Co-authored-by: David Brochart --- jupyter_client/tests/test_provisioning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index 197c08f87..3aa3db0e2 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -1,4 +1,4 @@ -"""Test Provisionering""" +"""Test Provisioning""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import asyncio From 3c40b5628dce2357e321a06742bc0066c1f4ee9f Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 23 Jul 2021 16:33:18 -0700 Subject: [PATCH 34/37] Properly instantiate KPFactory in kernelspecapp --- jupyter_client/kernelspecapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/kernelspecapp.py b/jupyter_client/kernelspecapp.py index 7ade0de2a..0690840e2 100644 --- a/jupyter_client/kernelspecapp.py +++ b/jupyter_client/kernelspecapp.py @@ -276,7 +276,7 @@ class ListProvisioners(JupyterApp): description = """List available provisioners for use in kernel specifications.""" def start(self): - kfp = KernelProvisionerFactory(parent=self).instance() + kfp = KernelProvisionerFactory.instance(parent=self) print("Available kernel provisioners:") provisioners = kfp.get_provisioner_entries() From d697fb4cd4229c1a5dbdf7dd28da9dd23f6dc898 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 23 Jul 2021 16:37:06 -0700 Subject: [PATCH 35/37] Add entrypoints into new requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index aa3a832e4..9219c73b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +entrypoints jupyter_core>=4.6.0 nest-asyncio>=1.5 python-dateutil>=2.1 From d3e01ea1b5cc872c42533fb3bd7f8b90506e3773 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 23 Jul 2021 16:41:52 -0700 Subject: [PATCH 36/37] Update mypy to install missing types --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c9c5ece2..d3c8d30ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -112,7 +112,7 @@ jobs: pip freeze - name: Check types - run: mypy jupyter_client --exclude '\/tests|kernelspecapp|ioloop|runapp' + run: mypy jupyter_client --exclude '\/tests|kernelspecapp|ioloop|runapp' --install-types --non-interactive - name: Run the tests run: pytest --cov jupyter_client -v jupyter_client From fc3227ee71ea7db8f54614d5eb58cec99bce98b5 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sat, 24 Jul 2021 14:50:57 -0700 Subject: [PATCH 37/37] Clarify example that method is part of custom provisioner impl --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index c375ad01d..aba9fbe81 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -139,7 +139,7 @@ behavior, there's a good chance you can simply extend custom functionality. In this example, RBACProvisioner will verify whether the current user is -in the role meant for this kernel. If not, an exception will be thrown. +in the role meant for this kernel by calling a method implemented within *this* provisioner. If the user is not in the role, an exception will be thrown. .. code:: python