diff --git a/.gitignore b/.gitignore index 7dd734fcf..86083f990 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,11 @@ htmlcov .cache docs/source/*.tpl docs/source/config_options.rst + # Eclipse pollutes the filesystem .project .pydevproject .settings + +# VSCode +.vscode \ No newline at end of file diff --git a/nbconvert/preprocessors/tests/files/Parallel Execute.ipynb b/nbconvert/preprocessors/tests/files/Parallel Execute A.ipynb similarity index 75% rename from nbconvert/preprocessors/tests/files/Parallel Execute.ipynb rename to nbconvert/preprocessors/tests/files/Parallel Execute A.ipynb index d40545c30..699abbcca 100644 --- a/nbconvert/preprocessors/tests/files/Parallel Execute.ipynb +++ b/nbconvert/preprocessors/tests/files/Parallel Execute A.ipynb @@ -9,7 +9,7 @@ "This notebook uses a file system based \"lock\" to assert that two instances of the notebook kernel will run in parallel. Each instance writes to a file in a temporary directory, and then tries to read the other file from\n", "the temporary directory, so that running them in sequence will fail, but running them in parallel will succed.\n", "\n", - "Two notebooks are launched, each with an injected cell which sets the `this_notebook` variable. One notebook is set to `this_notebook = 'A'` and the other `this_notebook = 'B'`." + "Two notebooks are launched, each which sets the `this_notebook` variable. One notebook is set to `this_notebook = 'A'` and the other `this_notebook = 'B'`." ] }, { @@ -31,7 +31,8 @@ "outputs": [], "source": [ "# the variable this_notebook is injectected in a cell above by the test framework.\n", - "other_notebook = {'A':'B', 'B':'A'}[this_notebook]\n", + "this_notebook = 'A'\n", + "other_notebook = 'B'\n", "directory = os.environ['NBEXECUTE_TEST_PARALLEL_TMPDIR']\n", "with open(os.path.join(directory, 'test_file_{}.txt'.format(this_notebook)), 'w') as f:\n", " f.write('Hello from {}'.format(this_notebook))" @@ -59,7 +60,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, "nbformat": 4, "nbformat_minor": 2 } diff --git a/nbconvert/preprocessors/tests/files/Parallel Execute B.ipynb b/nbconvert/preprocessors/tests/files/Parallel Execute B.ipynb new file mode 100644 index 000000000..54bd6ab91 --- /dev/null +++ b/nbconvert/preprocessors/tests/files/Parallel Execute B.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ensure notebooks can execute in parallel\n", + "\n", + "This notebook uses a file system based \"lock\" to assert that two instances of the notebook kernel will run in parallel. Each instance writes to a file in a temporary directory, and then tries to read the other file from\n", + "the temporary directory, so that running them in sequence will fail, but running them in parallel will succed.\n", + "\n", + "Two notebooks are launched, each which sets the `this_notebook` variable. One notebook is set to `this_notebook = 'A'` and the other `this_notebook = 'B'`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import os.path\n", + "import tempfile\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# the variable this_notebook is injectected in a cell above by the test framework.\n", + "this_notebook = 'B'\n", + "other_notebook = 'A'\n", + "directory = os.environ['NBEXECUTE_TEST_PARALLEL_TMPDIR']\n", + "with open(os.path.join(directory, 'test_file_{}.txt'.format(this_notebook)), 'w') as f:\n", + " f.write('Hello from {}'.format(this_notebook))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start = time.time()\n", + "timeout = 5\n", + "end = start + timeout\n", + "target_file = os.path.join(directory, 'test_file_{}.txt'.format(other_notebook))\n", + "while time.time() < end:\n", + " time.sleep(0.1)\n", + " if os.path.exists(target_file):\n", + " with open(target_file, 'r') as f:\n", + " text = f.read()\n", + " if text == 'Hello from {}'.format(other_notebook):\n", + " break\n", + "else:\n", + " assert False, \"Timed out – didn't get a message from {}\".format(other_notebook)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbconvert/preprocessors/tests/files/Sleep One.ipynb b/nbconvert/preprocessors/tests/files/Sleep One.ipynb new file mode 100644 index 000000000..d161b6e13 --- /dev/null +++ b/nbconvert/preprocessors/tests/files/Sleep One.ipynb @@ -0,0 +1,50 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "time.sleep(0.01)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbconvert/preprocessors/tests/test_execute.py b/nbconvert/preprocessors/tests/test_execute.py index 1ca6ac3ed..ce7a95f46 100644 --- a/nbconvert/preprocessors/tests/test_execute.py +++ b/nbconvert/preprocessors/tests/test_execute.py @@ -32,6 +32,7 @@ from nbconvert.filters import strip_ansi from testpath import modified_env from ipython_genutils.py3compat import string_types +from pebble import ProcessPool try: from queue import Empty # Py 3 @@ -46,6 +47,11 @@ except ImportError: from mock import MagicMock, patch # Py 2 + +PY3 = False +if sys.version_info[0] >= 3: + PY3 = True + addr_pat = re.compile(r'0x[0-9a-f]{7,9}') ipython_input_pat = re.compile(r'') current_dir = os.path.dirname(__file__) @@ -74,7 +80,7 @@ def build_preprocessor(opts): return preprocessor -def run_notebook(filename, opts, resources, preprocess_notebook=None): +def run_notebook(filename, opts, resources): """Loads and runs a notebook, returning both the version prior to running it and the version after running it. @@ -82,9 +88,6 @@ def run_notebook(filename, opts, resources, preprocess_notebook=None): with io.open(filename) as f: input_nb = nbformat.read(f, 4) - if preprocess_notebook: - input_nb = preprocess_notebook(input_nb) - preprocessor = build_preprocessor(opts) cleaned_input_nb = copy.deepcopy(input_nb) for cell in cleaned_input_nb.cells: @@ -264,24 +267,14 @@ def test_run_all_notebooks(input_name, opts): assert_notebooks_equal(input_nb, output_nb) -def label_parallel_notebook(nb, label): - """Insert a cell in a notebook which sets the variable `this_notebook` to the string `label`. - - Used for parallel testing to label two notebooks which are run simultaneously. - """ - label_cell = nbformat.v4.new_code_cell(source="this_notebook = '{}'".format(label)) - nb.cells.insert(1, label_cell) - return nb - - def test_parallel_notebooks(capfd, tmpdir): """Two notebooks should be able to be run simultaneously without problems. - + The two notebooks spawned here use the filesystem to check that the other notebook wrote to the filesystem.""" opts = dict(kernel_name="python") - input_name = "Parallel Execute.ipynb" + input_name = "Parallel Execute {label}.ipynb" input_file = os.path.join(current_dir, "files", input_name) res = notebook_resources() @@ -290,10 +283,9 @@ def test_parallel_notebooks(capfd, tmpdir): threading.Thread( target=run_notebook, args=( - input_file, + input_file.format(label=label), opts, res, - functools.partial(label_parallel_notebook, label=label), ), ) for label in ("A", "B") @@ -304,6 +296,34 @@ def test_parallel_notebooks(capfd, tmpdir): captured = capfd.readouterr() assert captured.err == "" +@pytest.mark.skipif(not PY3, + reason = "Not tested for Python 2") +def test_many_parallel_notebooks(capfd): + """Ensure that when many IPython kernels are run in parallel, nothing awful happens. + + Specifically, many IPython kernels when run simultaneously would enocunter errors + due to using the same SQLite history database. + """ + opts = dict(kernel_name="python", timeout=5) + input_name = "HelloWorld.ipynb" + input_file = os.path.join(current_dir, "files", input_name) + res = PreprocessorTestsBase().build_resources() + res["metadata"]["path"] = os.path.join(current_dir, "files") + + # run once, to trigger creating the original context + run_notebook(input_file, opts, res) + + with ProcessPool(max_workers=4) as pool: + futures = [ + # Travis needs a lot more time even though 10s is enough on most dev machines + pool.schedule(run_notebook, args=(input_file, opts, res), timeout=30) + for i in range(0, 8) + ] + for index, future in enumerate(futures): + future.result() + + captured = capfd.readouterr() + assert captured.err == "" class TestExecute(PreprocessorTestsBase): """Contains test functions for execute.py""" diff --git a/setup.py b/setup.py index dd7c62864..580838320 100644 --- a/setup.py +++ b/setup.py @@ -209,10 +209,10 @@ def run(self): 'testpath', 'defusedxml', ] -jupyter_client_req = 'jupyter_client>=4.2' +jupyter_client_req = 'jupyter_client>=4.3' extra_requirements = { - 'test': ['pytest', 'pytest-cov', 'mock; python_version < "3.4"', 'ipykernel', jupyter_client_req, 'ipywidgets>=7'], + 'test': ['pytest', 'pytest-cov', 'mock; python_version < "3.4"', 'ipykernel', jupyter_client_req, 'ipywidgets>=7', 'pebble'], 'serve': ['tornado>=4.0'], 'execute': [jupyter_client_req], 'docs': ['sphinx>=1.5.1',