diff --git a/.appveyor.yml b/.appveyor.yml index 4c10fb3c..34fb8b4d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -27,7 +27,7 @@ install: - conda update -q conda - conda info -a # Host dependencies - - conda install xeus=1.0.0 nlohmann_json cppzmq xtl jedi pybind11=2.6.0 pybind11_json=0.2.6 pygments gtest=1.8.0 debugpy ipython=7.14.0 -c conda-forge + - conda install xeus=1.0.0 nlohmann_json cppzmq xtl pybind11=2.6.0 pybind11_json=0.2.8 gtest=1.8.0 debugpy ipython=7.14.0 -c conda-forge # Build dependencies - conda install cmake -c conda-forge # Build and install xeus-python @@ -38,10 +38,9 @@ install: - nmake install # Install test dependencies - conda install mamba -c conda-forge - - mamba install jupyter_kernel_test -c conda-forge - - pip install example_magic + - mamba install pytest nbval ipympl -c conda-forge build_script: - cd test - test_xeus_python - - cd.. && cd.. && cd test && python test_xeus_python_kernel.py -vvv + - cd.. && cd.. && cd test && pytest . -vvv diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index 23f31447..f95c73d5 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -20,7 +20,6 @@ steps: - script: | source activate xeus-python - pip install example_magic mkdir build cd build if [[ $(xpyt_used_shared_xeus_python) == '0' ]]; then @@ -49,6 +48,6 @@ steps: - script: | source activate xeus-python - python test_xeus_python_kernel.py -vvv + pytest . -vvv displayName: Test xeus-python (Python) workingDirectory: $(Build.sourcesDirectory)/test diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b290b54..265db131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,7 @@ OPTION(XPYT_DOWNLOAD_GTEST "build gtest from downloaded sources" OFF) set(xtl_REQUIRED_VERSION 0.6.23) set(xeus_REQUIRED_VERSION 0.25.0) set(pybind11_REQUIRED_VERSION 2.6.0) -set(pybind11_json_REQUIRED_VERSION 0.2.2) +set(pybind11_json_REQUIRED_VERSION 0.2.8) if (NOT TARGET xtl) find_package(xtl ${xtl_REQUIRED_VERSION} REQUIRED) @@ -120,6 +120,10 @@ endif () # ============ set(XEUS_PYTHON_SRC + src/xcomm.cpp + src/xcomm.hpp + src/xcompiler.cpp + src/xcompiler.hpp src/xdebugger.cpp src/xdebugpy_client.hpp src/xdebugpy_client.cpp @@ -127,22 +131,10 @@ set(XEUS_PYTHON_SRC src/xdisplay.hpp src/xinput.cpp src/xinput.hpp - src/xinspect.cpp - src/xinspect.hpp - src/xinteractiveshell.cpp - src/xinteractiveshell.hpp src/xinternal_utils.cpp src/xinternal_utils.hpp src/xinterpreter.cpp - src/xis_complete.cpp - src/xis_complete.hpp - src/xlinecache.cpp - src/xlinecache.hpp - src/xnullcontext.cpp - src/xnullcontext.hpp src/xpaths.cpp - src/xpython_kernel.cpp - src/xpython_kernel.hpp src/xstream.cpp src/xstream.hpp src/xtraceback.cpp diff --git a/README.md b/README.md index 93e89ba3..867e3e6b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The ongoing effort to package xeus-python for pip takes place in the [xeus-pytho Or you can install it from the sources, you will first need to install dependencies ```bash -conda install cmake xeus nlohmann_json cppzmq xtl pybind11 pybind11_json jedi pygments notebook -c conda-forge +conda install cmake xeus nlohmann_json cppzmq xtl pybind11 pybind11_json ipython debugpy jupyterlab -c conda-forge ``` Then you can compile the sources @@ -115,10 +115,9 @@ http://xeus-python.readthedocs.io Check-out this blog post for the answer: https://blog.jupyter.org/a-new-python-kernel-for-jupyter-fcdf211e30a8. Long story short: -xeus-python does not cover 100% of the features of ipykernel. For example, only some magics are supported for now (Matplotlib magics are not supported, but nothing prevents from using the Matplotlib API directly). However: -- xeus-python is a lot lighter than ipykernel and IPython combined, which makes it a lot easier to implement new features on top of it. -- as an example, xeus-python already works with the **Jupyter Lab debugger**: https://github.com/jupyterlab/debugger +- xeus-python is a lot lighter than ipykernel, which makes it a lot easier to implement new features on top of it. +- xeus-python already works with the **Jupyter Lab debugger**: https://github.com/jupyterlab/debugger - xeus-based kernels are more versatile in that one can overload e.g. the concurrency model. This is something that Kitware’s SlicerJupyter project takes advantage of to integrate with the Qt event loop of their Qt-based desktop application. ## Dependencies @@ -132,25 +131,25 @@ xeus-python does not cover 100% of the features of ipykernel. For example, only - [nlohmann_json](https://github.com/nlohmann/json) -| `xeus-python`| `xeus` | `xtl` | `cppzmq` | `nlohmann_json` | `pybind11` | `pybind11_json` | `jedi` | `pygments` | `ptvsd` | `debugpy` | -|--------------|------------------|-----------------|----------|-----------------|----------------|-------------------|-------------------|-------------------|---------|-----------| -| master | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.10.2 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.10.1 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.10.0 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.5 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.4 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.18 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.3 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.2 | >=0.25.3,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.1 | >=0.25.0,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.9.0 | >=0.25.0,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.8.7 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | -| 0.8.6 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | -| 0.8.5 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | -| 0.8.4 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | -| 0.8.3 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | -| 0.8.2 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | -| 0.8.1 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | +| `xeus-python`| `xeus` | `xtl` | `cppzmq` | `nlohmann_json` | `pybind11` | `pybind11_json` | `jedi` | `pygments` | `ptvsd` | `debugpy` | `IPython` | +|--------------|------------------|-----------------|----------|-----------------|----------------|-------------------|-------------------|-------------------|---------|-----------|-----------| +| master | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.8,<0.3 | | | | >=1.1.0 | >=7.20,<8 | +| 0.10.2 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.10.1 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.10.0 | >=1.0.0,<0.26 | >=0.7.0,<0.8 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.5 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.18.0,<0.19 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.4 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.18 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.3 | >=0.25.3,<0.26 | >=0.6.23,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.2 | >=0.25.3,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.6.0,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.1 | >=0.25.0,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.9.0 | >=0.25.0,<0.26 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.8.7 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1 | >=2.3.1,<3.0.0 | | >=1.1.0 | | +| 0.8.6 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | +| 0.8.5 | >=0.24.2,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | +| 0.8.4 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | +| 0.8.3 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | +| 0.8.2 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | +| 0.8.1 | >=0.24.1,<0.25 | >=0.6.8,<0.7 | ~4.4.1 | >=3.6.1,<4.0 | >=2.2.4,<3.0 | >=0.2.6,<0.3 | >=0.15.1,<0.16.0 | >=2.3.1,<3.0.0 | >=4.3.2 | | | ## Contributing diff --git a/environment-dev.yml b/environment-dev.yml index 7ad99969..f7104d28 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,18 +2,19 @@ name: xeus-python channels: - conda-forge dependencies: - # Builde dependencies + # Build dependencies - cmake # Host dependencies - xeus=1.0.0 - nlohmann_json - cppzmq - xtl=0.7.0 - - jedi>=0.15.1,<0.19 + - jedi>=0.15.1,<0.18 - pybind11=2.6.0 - - pybind11_json=0.2.6 - - pygments - - ipython=7.14.0 + - pybind11_json>=0.2.6,<0.3 + - ipython>=7.14.0,<8 - debugpy # Test dependencies - - jupyter_kernel_test + - pytest + - nbval + - ipympl diff --git a/environment.yml b/environment.yml index 44ac7105..fef39144 100644 --- a/environment.yml +++ b/environment.yml @@ -7,3 +7,5 @@ dependencies: - ipywidgets>=7.6 - jupyterlab=3 - itkwidgets + - matplotlib + - ipympl diff --git a/include/xeus-python/xinterpreter.hpp b/include/xeus-python/xinterpreter.hpp index 5c9463ee..9d60dda2 100644 --- a/include/xeus-python/xinterpreter.hpp +++ b/include/xeus-python/xinterpreter.hpp @@ -73,9 +73,9 @@ namespace xpyt nl::json internal_request_impl(const nl::json& content) override; void redirect_output(); - void redirect_display(bool install_hook=true); - void load_extensions(); + py::object m_ipython_shell; + py::dict m_user_ns; py::object m_displayhook; // The interpreter has the same scope as a `gil_scoped_release` instance @@ -92,7 +92,7 @@ namespace xpyt bool m_release_gil_at_startup = true; gil_scoped_release_ptr m_release_gil = nullptr; - bool m_has_ipython; + bool m_redirect_display_enabled; }; } diff --git a/include/xeus-python/xtraceback.hpp b/include/xeus-python/xtraceback.hpp index 07132152..215aee63 100644 --- a/include/xeus-python/xtraceback.hpp +++ b/include/xeus-python/xtraceback.hpp @@ -29,10 +29,15 @@ namespace xpyt std::vector m_traceback; }; + XEUS_PYTHON_API py::module get_traceback_module(); + XEUS_PYTHON_API void register_filename_mapping(const std::string& filename, int execution_count); - + XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT xerror extract_error(py::error_already_set& error); + + XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT + xerror extract_error(const py::object& type, const py::object& value, const py::object& traceback); } #endif diff --git a/include/xeus-python/xutils.hpp b/include/xeus-python/xutils.hpp index a14a2d10..6d83aff3 100644 --- a/include/xeus-python/xutils.hpp +++ b/include/xeus-python/xutils.hpp @@ -46,7 +46,7 @@ namespace xpyt XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT void exec(const py::object& code, const py::object& scope = py::globals()); - + XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT py::object eval(const py::object& code, const py::object& scope = py::globals()); } diff --git a/notebooks/xeus-magics.ipynb b/notebooks/xeus-magics.ipynb index edab395d..d7c0e424 100644 --- a/notebooks/xeus-magics.ipynb +++ b/notebooks/xeus-magics.ipynb @@ -486,7 +486,7 @@ ], "metadata": { "kernelspec": { - "display_name": "xpython", + "display_name": "Python 3.9 (XPython)", "language": "python", "name": "xpython" }, @@ -494,7 +494,7 @@ "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "version": "3.8.2" + "version": "3.9.1" } }, "nbformat": 4, diff --git a/notebooks/xeus-python.ipynb b/notebooks/xeus-python.ipynb index 7aac1b15..12a0b75b 100644 --- a/notebooks/xeus-python.ipynb +++ b/notebooks/xeus-python.ipynb @@ -50,23 +50,19 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "# Redirected streams" + "print" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "import time\n", - "\n", - "for x in range(3):\n", - " print(x)\n", - " time.sleep(0.5)" + "# Redirected streams" ] }, { @@ -117,34 +113,6 @@ "# Code completion" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "my_very_long_variable_name = 1234\n", - "\n", - "def my_very_long_function_name():\n", - " pass" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### press `tab` for code completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "my_ver" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -259,7 +227,7 @@ "outputs": [], "source": [ "james = Person(\"James Smith\", \"Boston\")\n", - "james" + "display(james)" ] }, { @@ -269,7 +237,74 @@ "outputs": [], "source": [ "marie = Person(\"Marie Curie\", \"Poland\", \"./marie.png\")\n", - "marie" + "display(marie)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "plt.plot(np.sin(np.linspace(0, 20, 100)));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib widget" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure()\n", + "plt.plot(np.sin(np.linspace(0, 20, 100)));" ] }, { @@ -425,7 +460,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Monkey-patched IPython.display module" + "## IPython.display module" ] }, { @@ -618,7 +653,7 @@ ], "metadata": { "kernelspec": { - "display_name": "xpython", + "display_name": "Python 3.9 (XPython)", "language": "python", "name": "xpython" }, @@ -626,7 +661,7 @@ "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "version": "3.8.2" + "version": "3.9.1" } }, "nbformat": 4, diff --git a/src/xcomm.cpp b/src/xcomm.cpp new file mode 100644 index 00000000..ddbd8f17 --- /dev/null +++ b/src/xcomm.cpp @@ -0,0 +1,203 @@ +/*************************************************************************** +* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * +* Wolf Vollprecht * +* Copyright (c) 2018, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include +#include + +#include "nlohmann/json.hpp" + +#include "xeus/xcomm.hpp" +#include "xeus/xinterpreter.hpp" + +#include "pybind11_json/pybind11_json.hpp" + +#include "pybind11/pybind11.h" +#include "pybind11/functional.h" +#include "pybind11/eval.h" + +#include "xeus-python/xutils.hpp" + +#include "xcomm.hpp" +#include "xinternal_utils.hpp" + +namespace py = pybind11; +namespace nl = nlohmann; + +namespace xpyt +{ + /********************* + * xcomm declaration * + ********************/ + + class xcomm + { + public: + + using python_callback_type = std::function; + using cpp_callback_type = std::function; + using zmq_buffers_type = std::vector; + + xcomm(const py::args& args, const py::kwargs& kwargs); + xcomm(xeus::xcomm&& comm); + xcomm(xcomm&& comm) = default; + virtual ~xcomm(); + + std::string comm_id() const; + bool kernel() const; + + void close(const py::args& args, const py::kwargs& kwargs); + void send(const py::args& args, const py::kwargs& kwargs); + void on_msg(const python_callback_type& callback); + void on_close(const python_callback_type& callback); + + private: + + xeus::xtarget* target(const py::kwargs& kwargs) const; + xeus::xguid id(const py::kwargs& kwargs) const; + cpp_callback_type cpp_callback(const python_callback_type& callback) const; + + xeus::xcomm m_comm; + }; + + struct xcomm_manager + { + xcomm_manager() = default; + + void register_target(const py::str& target_name, const py::object& callback); + }; + + /************************ + * xcomm implementation * + ************************/ + + xcomm::xcomm(const py::args& /*args*/, const py::kwargs& kwargs) + : m_comm(target(kwargs), id(kwargs)) + { + m_comm.open( + kwargs.attr("get")("metadata", py::dict()), + kwargs.attr("get")("data", py::dict()), + pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) + ); + } + + xcomm::xcomm(xeus::xcomm&& comm) + : m_comm(std::move(comm)) + { + } + + xcomm::~xcomm() + { + } + + std::string xcomm::comm_id() const + { + return m_comm.id(); + } + + bool xcomm::kernel() const + { + return true; + } + + void xcomm::close(const py::args& /*args*/, const py::kwargs& kwargs) + { + m_comm.close( + kwargs.attr("get")("metadata", py::dict()), + kwargs.attr("get")("data", py::dict()), + pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) + ); + } + + void xcomm::send(const py::args& /*args*/, const py::kwargs& kwargs) + { + m_comm.send( + kwargs.attr("get")("metadata", py::dict()), + kwargs.attr("get")("data", py::dict()), + pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) + ); + } + + void xcomm::on_msg(const python_callback_type& callback) + { + m_comm.on_message(cpp_callback(callback)); + } + + void xcomm::on_close(const python_callback_type& callback) + { + m_comm.on_close(cpp_callback(callback)); + } + + xeus::xtarget* xcomm::target(const py::kwargs& kwargs) const + { + std::string target_name = kwargs["target_name"].cast(); + return xeus::get_interpreter().comm_manager().target(target_name); + } + + xeus::xguid xcomm::id(const py::kwargs& kwargs) const + { + if (py::hasattr(kwargs, "comm_id")) + { + // TODO: prevent copy + return xeus::xguid(kwargs["comm_id"].cast()); + } + else + { + return xeus::new_xguid(); + } + } + + auto xcomm::cpp_callback(const python_callback_type& py_callback) const -> cpp_callback_type + { + return [this, py_callback](const xeus::xmessage& msg) { + XPYT_HOLDING_GIL(py_callback(cppmessage_to_pymessage(msg))) + }; + } + + void xcomm_manager::register_target(const py::str& target_name, const py::object& callback) + { + auto target_callback = [&callback] (xeus::xcomm&& comm, const xeus::xmessage& msg) { + XPYT_HOLDING_GIL(callback(xcomm(std::move(comm)), cppmessage_to_pymessage(msg))); + }; + + xeus::get_interpreter().comm_manager().register_comm_target( + static_cast(target_name), target_callback + ); + } + + /*************** + * comm module * + ***************/ + + py::module get_comm_module_impl() + { + py::module comm_module = create_module("comm"); + + py::class_(comm_module, "Comm") + .def(py::init()) + .def("close", &xcomm::close) + .def("send", &xcomm::send) + .def("on_msg", &xcomm::on_msg) + .def("on_close", &xcomm::on_close) + .def_property_readonly("comm_id", &xcomm::comm_id) + .def_property_readonly("kernel", &xcomm::kernel); + + py::class_(comm_module, "CommManager") + .def(py::init<>()) + .def("register_target", &xcomm_manager::register_target); + + return comm_module; + } + + py::module get_comm_module() + { + static py::module comm_module = get_comm_module_impl(); + return comm_module; + } +} diff --git a/src/xis_complete.hpp b/src/xcomm.hpp similarity index 89% rename from src/xis_complete.hpp rename to src/xcomm.hpp index ace97f0b..d82149cc 100644 --- a/src/xis_complete.hpp +++ b/src/xcomm.hpp @@ -8,8 +8,8 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XPYT_COMPLETION_HPP -#define XPYT_COMPLETION_HPP +#ifndef XPYT_COMM_HPP +#define XPYT_COMM_HPP #include "pybind11/pybind11.h" @@ -17,7 +17,7 @@ namespace py = pybind11; namespace xpyt { - py::module get_completion_module(); + py::module get_comm_module(); } #endif diff --git a/src/xcompiler.cpp b/src/xcompiler.cpp new file mode 100644 index 00000000..2af5bc61 --- /dev/null +++ b/src/xcompiler.cpp @@ -0,0 +1,145 @@ +/*************************************************************************** +* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * +* Wolf Vollprecht * +* Copyright (c) 2018, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include +#include + +#include "nlohmann/json.hpp" + +#include "xeus/xinterpreter.hpp" + +#include "pybind11_json/pybind11_json.hpp" + +#include "pybind11/pybind11.h" +#include "pybind11/functional.h" + +#include "xeus-python/xutils.hpp" + +#include "xcompiler.hpp" +#include "xinternal_utils.hpp" + +namespace py = pybind11; +namespace nl = nlohmann; + +namespace xpyt +{ + + py::str get_filename(const py::str& raw_code) + { + return get_cell_tmp_file(raw_code); + } + + /******************* + * compiler module * + *******************/ + + py::module get_compiler_module_impl() + { + py::module compiler_module = create_module("compiler"); + + compiler_module.def("get_filename", get_filename); + + ::xpyt::exec(py::str(R"( +## TODO Uncomment the following when IPython is released with https://github.com/ipython/ipython/pull/12809 and remove the rest +# from IPython.core.compilerop import CachingCompiler + +# class XCachingCompiler(CachingCompiler): +# def __init__(self, *args, **kwargs): +# super(XCachingCompiler, self).__init__(*args, **kwargs) +# +# self.filename_mapper = None +# +# def get_code_name(self, raw_code, code, number): +# filename = get_filename(raw_code) +# +# if self.filename_mapper is not None: +# self.filename_mapper(filename, number) +# +# return filename + +import __future__ +from ast import PyCF_ONLY_AST +import codeop +import functools +import hashlib +import linecache +import operator +import time +from contextlib import contextmanager + +PyCF_MASK = functools.reduce(operator.or_, + (getattr(__future__, fname).compiler_flag + for fname in __future__.all_feature_names)) + + +class CachingCompiler(codeop.Compile): + def __init__(self): + codeop.Compile.__init__(self) + + if not hasattr(linecache, '_ipython_cache'): + linecache._ipython_cache = {} + if not hasattr(linecache, '_checkcache_ori'): + linecache._checkcache_ori = linecache.checkcache + + linecache.checkcache = check_linecache_ipython + + self.filename_mapper = None + + + def ast_parse(self, source, filename='', symbol='exec'): + return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) + + def reset_compiler_flags(self): + self.flags = codeop.PyCF_DONT_IMPLY_DEDENT + + @property + def compiler_flags(self): + return self.flags + + def cache(self, code, number=0, raw_code=None): + if raw_code is None: + raw_code = code + + name = get_filename(raw_code) + entry = (len(code), time.time(), + [line+'\n' for line in code.splitlines()], name) + linecache.cache[name] = entry + linecache._ipython_cache[name] = entry + + if self.filename_mapper is not None: + self.filename_mapper(name, number) + + return name + + @contextmanager + def extra_flags(self, flags): + turn_on_bits = ~self.flags & flags + + self.flags = self.flags | flags + try: + yield + finally: + self.flags &= ~turn_on_bits + + +def check_linecache_ipython(*args): + linecache._checkcache_ori(*args) + linecache.cache.update(linecache._ipython_cache) + )"), compiler_module.attr("__dict__")); + + return compiler_module; + } + + py::module get_compiler_module() + { + static py::module compiler_module = get_compiler_module_impl(); + return compiler_module; + } +} diff --git a/src/xpython_kernel.hpp b/src/xcompiler.hpp similarity index 86% rename from src/xpython_kernel.hpp rename to src/xcompiler.hpp index f73949ea..abedc3c4 100644 --- a/src/xpython_kernel.hpp +++ b/src/xcompiler.hpp @@ -8,17 +8,16 @@ * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -#ifndef XPYT_KERNEL_HPP -#define XPYT_KERNEL_HPP +#ifndef XPYT_COMPILER_HPP +#define XPYT_COMPILER_HPP #include "pybind11/pybind11.h" -#include "xeus/xhistory_manager.hpp" namespace py = pybind11; namespace xpyt { - py::module get_kernel_module(); + py::module get_compiler_module(); } #endif diff --git a/src/xdisplay.cpp b/src/xdisplay.cpp index c9a02b44..ada90e0f 100644 --- a/src/xdisplay.cpp +++ b/src/xdisplay.cpp @@ -16,7 +16,6 @@ #include "nlohmann/json.hpp" #include "xeus/xinterpreter.hpp" -#include "xeus/xguid.hpp" #include "pybind11_json/pybind11_json.hpp" @@ -29,251 +28,45 @@ #include "xdisplay.hpp" #include "xinternal_utils.hpp" -#ifdef __GNUC__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wattributes" -#endif - namespace py = pybind11; namespace nl = nlohmann; using namespace pybind11::literals; namespace xpyt { + /**************************************** + * xpublish_display_data implementation * + ****************************************/ - bool should_include(const std::string& mimetype, const std::vector& include) - { - return include.size() == 0 || std::find(include.cbegin(), include.cend(), mimetype) != include.end(); - } - - bool should_exclude(const std::string& mimetype, const std::vector& exclude) - { - return exclude.size() != 0 && std::find(exclude.cbegin(), exclude.cend(), mimetype) != exclude.end(); - } - - void compute_repr( - const py::object& obj, const std::string& repr_method, const std::string& mimetype, - const std::vector& include, const std::vector& exclude, - py::dict& pub_data, py::dict& pub_metadata) + void xpublish_display_data(const py::object& data, const py::object& metadata, const py::object& transient, bool update) { - if (hasattr(obj, repr_method.c_str()) && should_include(mimetype, include) && !should_exclude(mimetype, exclude)) - { - const py::object& repr = obj.attr(repr_method.c_str())(); - - if (!repr.is_none()) - { - if (py::isinstance(repr)) - { - py::tuple repr_tuple = repr; - - pub_data[mimetype.c_str()] = repr_tuple[0]; - pub_metadata[mimetype.c_str()] = repr_tuple[1]; - } - else - { - pub_data[mimetype.c_str()] = repr; - } - } - } - } - - py::tuple mime_bundle_repr(const py::object& obj, const std::vector& include = {}, const std::vector& exclude = {}) - { - py::module py_json = py::module::import("json"); - py::module builtins = py::module::import("builtins"); - py::dict pub_data; - py::dict pub_metadata; + auto& interp = xeus::get_interpreter(); - if (hasattr(obj, "_repr_mimebundle_")) + if (update) { - auto bundle = obj.attr("_repr_mimebundle_")(include, exclude); - - if (py::isinstance(bundle) || py::isinstance(bundle)) - { - py::list data_metadata = bundle; - pub_data = data_metadata[0]; - pub_metadata = data_metadata[1]; - } - else - { - pub_data = bundle; - } + interp.update_display_data(data, metadata, transient); } else { - compute_repr(obj, "_repr_html_", "text/html", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_markdown_", "text/markdown", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_svg_", "image/svg+xml", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_png_", "image/png", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_jpeg_", "image/jpeg", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_latex_", "text/latex", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_json_", "application/json", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_javascript_", "application/javascript", include, exclude, pub_data, pub_metadata); - compute_repr(obj, "_repr_pdf_", "application/pdf", include, exclude, pub_data, pub_metadata); + interp.display_data(data, metadata, transient); } - - pub_data["text/plain"] = py::str(builtins.attr("repr")(obj)); - - return py::make_tuple(pub_data, pub_metadata); } - /**************************** - * xdisplayhook declaration * - ****************************/ - - class xdisplayhook - { - public: - - xdisplayhook(); - virtual ~xdisplayhook(); - - void set_execution_count(int execution_count); - void operator()(const py::object& obj, bool raw) const; - - private: - - int m_execution_count; - }; - - /******************************* - * xdisplayhook implementation * - *******************************/ + /******************************************** + * xpublish_execution_result implementation * + ********************************************/ - xdisplayhook::xdisplayhook() - : m_execution_count(0) - { - } - - xdisplayhook::~xdisplayhook() - { - } - - void xdisplayhook::set_execution_count(int execution_count) - { - m_execution_count = execution_count; - } - - void xdisplayhook::operator()(const py::object& obj, bool raw = false) const - { - auto& interp = xeus::get_interpreter(); - - if (!obj.is_none()) - { - if (hasattr(obj, "_ipython_display_")) - { - obj.attr("_ipython_display_")(); - return; - } - - py::object pub_data; - py::object pub_metadata; - if (raw) - { - pub_data = obj; - } - else - { - const py::tuple& repr = mime_bundle_repr(obj); - pub_data = repr[0]; - pub_metadata = repr[1]; - } - - interp.publish_execution_result(m_execution_count, pub_data, pub_metadata); - } - } - - /*************************** - * xdisplay implementation * - ***************************/ - - void xdisplay_impl(const py::args objs, - const std::vector& include, - const std::vector& exclude, - const py::dict& metadata, - const py::object& transient, - const py::object& display_id, - bool update, - bool raw) + void xpublish_execution_result(const py::int_& execution_count, const py::object& data, const py::object& metadata) { auto& interp = xeus::get_interpreter(); - for (std::size_t i = 0; i < objs.size(); ++i) + nl::json cpp_data = data; + if (cpp_data.size() != 0) { - py::object obj = objs[i]; - if (!obj.is_none()) - { - if (hasattr(obj, "_ipython_display_")) - { - obj.attr("_ipython_display_")(); - return; - } - - py::object pub_data; - py::object pub_metadata = py::dict(); - if (raw) - { - pub_data = obj; - } - else - { - const py::tuple& repr = mime_bundle_repr(obj, include, exclude); - pub_data = repr[0]; - pub_metadata = repr[1]; - } - pub_metadata.attr("update")(metadata); - - nl::json cpp_transient = transient.is_none() ? nl::json::object() : nl::json(transient); - - if (!display_id.is_none()) - { - cpp_transient["display_id"] = display_id; - } - - if (update) - { - interp.update_display_data(pub_data, pub_metadata, std::move(cpp_transient)); - } - else - { - interp.display_data(pub_data, pub_metadata, std::move(cpp_transient)); - } - } + interp.publish_execution_result(execution_count, std::move(cpp_data), metadata); } } - void xdisplay(py::args objs, py::kwargs kw) - { - auto raw = kw.contains("raw") ? kw["raw"].cast() : false; - auto include = kw.contains("include") ? kw["include"].cast>() : std::vector(); - auto exclude = kw.contains("exclude") ? kw["exclude"].cast>() : std::vector({}); - auto metadata = kw.contains("metadata") ? py::object(kw["metadata"]) : py::dict(); - auto transient = kw.contains("transient") ? py::object(kw["transient"]) : py::none(); - auto display_id = kw.contains("display_id") ? py::object(kw["display_id"]) : py::none(); - auto update = kw.contains("update") ? py::bool_(kw["update"]).cast() : false; - - xdisplay_impl(objs, include, exclude, metadata, transient, display_id, update, raw); - } - - void xupdate_display(py::args objs, py::kwargs kw) - { - auto raw = kw.contains("raw") ? kw["raw"].cast() : false; - auto include = kw.contains("include") ? kw["include"].cast>() : std::vector(); - auto exclude = kw.contains("exclude") ? kw["exclude"].cast>() : std::vector({}); - auto metadata = kw.contains("metadata") ? py::object(kw["metadata"]) : py::dict(); - auto transient = kw.contains("transient") ? py::object(kw["transient"]) : py::none(); - auto display_id = kw.contains("display_id") ? py::object(kw["display_id"]) : py::none(); - - xdisplay_impl(objs, include, exclude, metadata, transient, display_id, true, raw); - } - - void xpublish_display_data(const py::object& data, const py::object& metadata, const py::str& /*source*/, const py::object& transient) - { - auto& interp = xeus::get_interpreter(); - - interp.display_data(data, metadata, transient); - } - /************************* * xclear implementation * *************************/ @@ -281,6 +74,7 @@ namespace xpyt void xclear(bool wait = false) { auto& interp = xeus::get_interpreter(); + interp.clear_output(wait); } @@ -292,1895 +86,61 @@ namespace xpyt { py::module display_module = create_module("display"); - py::class_(display_module, "DisplayHook") - .def(py::init<>()) - .def("set_execution_count", &xdisplayhook::set_execution_count) - .def("__call__", &xdisplayhook::operator(), py::arg("obj"), py::arg("raw") = false); - - display_module.def("display", xdisplay); - - display_module.def("update_display", xupdate_display); - display_module.def("publish_display_data", xpublish_display_data, py::arg("data"), - py::arg("metadata") = py::dict(), - py::arg("source") = py::str(), - py::arg("transient") = py::dict()); + py::arg("metadata"), + py::arg("transient"), + py::arg("update") + ); + + display_module.def("publish_execution_result", + xpublish_execution_result, + py::arg("execution_count"), + py::arg("data"), + py::arg("metadata") + ); display_module.def("clear_output", - xclear, - py::arg("wait") = false); + xclear, + py::arg("wait") = false + ); - // This is a temporary copy of IPython.display module - // When IPython 8 is out, we can monkey patch only the module containing the core display functions - // See https://github.com/jupyter-xeus/xeus-python/pull/266 exec(py::str(R"( -# -*- coding: utf-8 -*- -"""Top-level display functions for displaying object in different formats.""" - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - - -from binascii import b2a_hex, b2a_base64, hexlify -import json -import mimetypes -import os -import struct import sys -import warnings -from copy import deepcopy -from os.path import splitext, exists, isfile, abspath, join, isdir -from os import walk, sep, fsdecode -from pathlib import Path, PurePath -from html import escape as html_escape - - -#----------------------------------------------------------------------------- -# utility functions -#----------------------------------------------------------------------------- - -def decode(s, encoding=None): - encoding = encoding or sys.getdefaultencoding() - return s.decode(encoding, "replace") - -def cast_unicode(s, encoding=None): - if isinstance(s, bytes): - return decode(s, encoding) - return s - -def _safe_exists(path): - """Check path, but don't let exceptions raise""" - try: - return os.path.exists(path) - except Exception: - return False - -def _merge(d1, d2): - """Like update, but merges sub-dicts instead of clobbering at the top level. - - Updates d1 in-place - """ - - if not isinstance(d2, dict) or not isinstance(d1, dict): - return d2 - for key, value in d2.items(): - d1[key] = _merge(d1.get(key), value) - return d1 - -def _display_mimetype(mimetype, objs, raw=False, metadata=None): - """internal implementation of all display_foo methods - - Parameters - ---------- - mimetype : str - The mimetype to be published (e.g. 'image/png') - objs : tuple of objects - The Python objects to display, or if raw=True raw text data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - if metadata: - metadata = {mimetype: metadata} - if raw: - # turn list of pngdata into list of { 'image/png': pngdata } - objs = [ {mimetype: obj} for obj in objs ] - display(*objs, raw=raw, metadata=metadata, include=[mimetype]) - -#----------------------------------------------------------------------------- -# Main functions -#----------------------------------------------------------------------------- - - -def _new_id(): - """Generate a new random text id with urandom""" - return b2a_hex(os.urandom(16)).decode('ascii') - - -class DisplayHandle(object): - """A handle on an updatable display - - Call `.update(obj)` to display a new object. - - Call `.display(obj`) to add a new instance of this display, - and update existing instances. - - See Also - -------- - - :func:`display`, :func:`update_display` - - """ - - def __init__(self, display_id=None): - if display_id is None: - display_id = _new_id() - self.display_id = display_id - - def __repr__(self): - return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) - - def display(self, obj, **kwargs): - """Make a new display with my id, updating existing instances. - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to display - """ - display(obj, display_id=self.display_id, **kwargs) - - def update(self, obj, **kwargs): - """Update existing displays with my id - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to update_display - """ - update_display(obj, display_id=self.display_id, **kwargs) - -def display_pretty(*objs, **kwargs): - """Display the pretty (default) representation of an object. +from IPython.core.displaypub import DisplayPublisher +from IPython.core.displayhook import DisplayHook - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw text data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/plain', objs, **kwargs) +class XDisplayPublisher(DisplayPublisher): + def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None: + publish_display_data(data, metadata, transient, update) -def display_html(*objs, **kwargs): - """Display the HTML representation of an object. + def clear_output(self, wait=False): + clear_output(wait) - Note: If raw=False and the object does not have a HTML - representation, no HTML will be shown. - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw HTML data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/html', objs, **kwargs) +class XDisplayHook(DisplayHook): + def start_displayhook(self): + self.data = {} + self.metadata = {} - -def display_markdown(*objs, **kwargs): - """Displays the Markdown representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw markdown data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - - _display_mimetype('text/markdown', objs, **kwargs) - - -def display_svg(*objs, **kwargs): - """Display the SVG representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw svg data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/svg+xml', objs, **kwargs) - - -def display_png(*objs, **kwargs): - """Display the PNG representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw png data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/png', objs, **kwargs) - - -def display_jpeg(*objs, **kwargs): - """Display the JPEG representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw JPEG data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('image/jpeg', objs, **kwargs) - - -def display_latex(*objs, **kwargs): - """Display the LaTeX representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw latex data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('text/latex', objs, **kwargs) - - -def display_json(*objs, **kwargs): - """Display the JSON representation of an object. - - Note that not many frontends support displaying JSON. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw json data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/json', objs, **kwargs) - - -def display_javascript(*objs, **kwargs): - """Display the Javascript representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw javascript data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/javascript', objs, **kwargs) - - -def display_pdf(*objs, **kwargs): - """Display the PDF representation of an object. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display, or if raw=True raw javascript data to - display. - raw : bool - Are the data objects raw data or Python objects that need to be - formatted before display? [default: False] - metadata : dict (optional) - Metadata to be associated with the specific mimetype output. - """ - _display_mimetype('application/pdf', objs, **kwargs) - - - )"), display_module.attr("__dict__")); - exec(py::str(R"( - -#----------------------------------------------------------------------------- -# Smart classes -#----------------------------------------------------------------------------- - - -class DisplayObject(object): - """An object that wraps data to be displayed.""" - - _read_flags = 'r' - _show_mem_addr = False - metadata = None - - def __init__(self, data=None, url=None, filename=None, metadata=None): - """Create a display object given raw data. - - When this object is returned by an expression or passed to the - display function, it will result in the data being displayed - in the frontend. The MIME type of the data should match the - subclasses used, so the Png subclass should be used for 'image/png' - data. If the data is a URL, the data will first be downloaded - and then displayed. If - - Parameters - ---------- - data : unicode, str or bytes - The raw data or a URL or file to load the data from - url : unicode - A URL to download the data from. - filename : unicode - Path to a local file to load the data from. - metadata : dict - Dict of metadata associated to be the object when displayed - """ - if isinstance(data, (Path, PurePath)): - data = str(data) - - if data is not None and isinstance(data, str): - if data.startswith('http') and url is None: - url = data - filename = None - data = None - elif _safe_exists(data) and filename is None: - url = None - filename = data - data = None - - self.url = url - self.filename = filename - # because of @data.setter methods in - # subclasses ensure url and filename are set - # before assigning to self.data - self.data = data - - if metadata is not None: - self.metadata = metadata - elif self.metadata is None: - self.metadata = {} - - self.reload() - self._check_data() - - def __repr__(self): - if not self._show_mem_addr: - cls = self.__class__ - r = "<%s.%s object>" % (cls.__module__, cls.__name__) - else: - r = super(DisplayObject, self).__repr__() - return r - - def _check_data(self): - """Override in subclasses if there's something to check.""" + def write_output_prompt(self): pass - def _data_and_metadata(self): - """shortcut for returning metadata with shape information, if defined""" - if self.metadata: - return self.data, deepcopy(self.metadata) - else: - return self.data - - def reload(self): - """Reload the raw data from file or URL.""" - if self.filename is not None: - with open(self.filename, self._read_flags) as f: - self.data = f.read() - elif self.url is not None: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - data = response.read() - # extract encoding from header, if there is one: - encoding = None - if 'content-type' in response.headers: - for sub in response.headers['content-type'].split(';'): - sub = sub.strip() - if sub.startswith('charset'): - encoding = sub.split('=')[-1].strip() - break - if 'content-encoding' in response.headers: - # TODO: do deflate? - if 'gzip' in response.headers['content-encoding']: - import gzip - from io import BytesIO - with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: - encoding = None - data = fp.read() - - # decode data, if an encoding was specified - # We only touch self.data once since - # subclasses such as SVG have @data.setter methods - # that transform self.data into ... well svg. - if encoding: - self.data = data.decode(encoding, 'replace') - else: - self.data = data - - -class TextDisplayObject(DisplayObject): - """Validate that display data is text""" - def _check_data(self): - if self.data is not None and not isinstance(self.data, str): - raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data)) - -class Pretty(TextDisplayObject): - - def _repr_pretty_(self, pp, cycle): - return pp.text(self.data) - - -class HTML(TextDisplayObject): - - def __init__(self, data=None, url=None, filename=None, metadata=None): - def warn(): - if not data: - return False - - # - # Avoid calling lower() on the entire data, because it could be a - # long string and we're only interested in its beginning and end. - # - prefix = data[:10].lower() - suffix = data[-10:].lower() - return prefix.startswith(" - """ - - def __init__(self, src, width, height, **kwargs): - self.src = src - self.width = width - self.height = height - self.params = kwargs - - def _repr_html_(self): - """return the embed iframe""" - if self.params: - try: - from urllib.parse import urlencode # Py 3 - except ImportError: - from urllib import urlencode - params = "?" + urlencode(self.params) - else: - params = "" - return self.iframe.format(src=self.src, - width=self.width, - height=self.height, - params=params) - -class YouTubeVideo(IFrame): - """Class for embedding a YouTube Video in an IPython session, based on its video id. - - e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would - do:: - - vid = YouTubeVideo("foo") - display(vid) - - To start from 30 seconds:: - - vid = YouTubeVideo("abc", start=30) - display(vid) - - To calculate seconds from time as hours, minutes, seconds use - :class:`datetime.timedelta`:: - - start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds()) - - Other parameters can be provided as documented at - https://developers.google.com/youtube/player_parameters#Parameters - - When converting the notebook using nbconvert, a jpeg representation of the video - will be inserted in the document. - """ - - def __init__(self, id, width=400, height=300, **kwargs): - self.id=id - src = "https://www.youtube.com/embed/{0}".format(id) - super(YouTubeVideo, self).__init__(src, width, height, **kwargs) - - def _repr_jpeg_(self): - # Deferred import - from urllib.request import urlopen - - try: - return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read() - except IOError: - return None - -class VimeoVideo(IFrame): - """ - Class for embedding a Vimeo video in an IPython session, based on its video id. - """ - - def __init__(self, id, width=400, height=300, **kwargs): - src="https://player.vimeo.com/video/{0}".format(id) - super(VimeoVideo, self).__init__(src, width, height, **kwargs) - -class ScribdDocument(IFrame): - """ - Class for embedding a Scribd document in an IPython session - - Use the start_page params to specify a starting point in the document - Use the view_mode params to specify display type one off scroll | slideshow | book - - e.g to Display Wes' foundational paper about PANDAS in book mode from page 3 - - ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book") - """ - - def __init__(self, id, width=400, height=300, **kwargs): - src="https://www.scribd.com/embeds/{0}/content".format(id) - super(ScribdDocument, self).__init__(src, width, height, **kwargs) - - )"), display_module.attr("__dict__")); - exec(py::str(R"( - -class FileLink(object): - """Class for embedding a local file link in an IPython session, based on path - - e.g. to embed a link that was generated in the IPython notebook as my/data.txt - - you would do:: - - local_file = FileLink("my/data.txt") - display(local_file) - - or in the HTML notebook, just:: - - FileLink("my/data.txt") - """ - - html_link_str = "%s" - - def __init__(self, - path, - url_prefix='', - result_html_prefix='', - result_html_suffix='
'): - """ - Parameters - ---------- - path : str - path to the file or directory that should be formatted - url_prefix : str - prefix to be prepended to all files to form a working link [default: - ''] - result_html_prefix : str - text to append to beginning to link [default: ''] - result_html_suffix : str - text to append at the end of link [default: '
'] - """ - if isdir(path): - raise ValueError("Cannot display a directory using FileLink. " - "Use FileLinks to display '%s'." % path) - self.path = fsdecode(path) - self.url_prefix = url_prefix - self.result_html_prefix = result_html_prefix - self.result_html_suffix = result_html_suffix - - def _format_path(self): - fp = ''.join([self.url_prefix, html_escape(self.path)]) - return ''.join([self.result_html_prefix, - self.html_link_str % \ - (fp, html_escape(self.path, quote=False)), - self.result_html_suffix]) - - def _repr_html_(self): - """return html link to file - """ - if not exists(self.path): - return ("Path (%s) doesn't exist. " - "It may still be in the process of " - "being generated, or you may have the " - "incorrect path." % self.path) - - return self._format_path() - - def __repr__(self): - """return absolute path to file - """ - return abspath(self.path) - -class FileLinks(FileLink): - """Class for embedding local file links in an IPython session, based on path - - e.g. to embed links to files that were generated in the IPython notebook - under ``my/data``, you would do:: - - local_files = FileLinks("my/data") - display(local_files) - - or in the HTML notebook, just:: - - FileLinks("my/data") - """ - def __init__(self, - path, - url_prefix='', - included_suffixes=None, - result_html_prefix='', - result_html_suffix='
', - notebook_display_formatter=None, - terminal_display_formatter=None, - recursive=True): - """ - See :class:`FileLink` for the ``path``, ``url_prefix``, - ``result_html_prefix`` and ``result_html_suffix`` parameters. - - included_suffixes : list - Filename suffixes to include when formatting output [default: include - all files] - - notebook_display_formatter : function - Used to format links for display in the notebook. See discussion of - formatter functions below. - - terminal_display_formatter : function - Used to format links for display in the terminal. See discussion of - formatter functions below. - - Formatter functions must be of the form:: - - f(dirname, fnames, included_suffixes) - - dirname : str - The name of a directory - fnames : list - The files in that directory - included_suffixes : list - The file suffixes that should be included in the output (passing None - meansto include all suffixes in the output in the built-in formatters) - recursive : boolean - Whether to recurse into subdirectories. Default is True. - - The function should return a list of lines that will be printed in the - notebook (if passing notebook_display_formatter) or the terminal (if - passing terminal_display_formatter). This function is iterated over for - each directory in self.path. Default formatters are in place, can be - passed here to support alternative formatting. - - """ - if isfile(path): - raise ValueError("Cannot display a file using FileLinks. " - "Use FileLink to display '%s'." % path) - self.included_suffixes = included_suffixes - # remove trailing slashes for more consistent output formatting - path = path.rstrip('/') - - self.path = path - self.url_prefix = url_prefix - self.result_html_prefix = result_html_prefix - self.result_html_suffix = result_html_suffix - - self.notebook_display_formatter = \ - notebook_display_formatter or self._get_notebook_display_formatter() - self.terminal_display_formatter = \ - terminal_display_formatter or self._get_terminal_display_formatter() - - self.recursive = recursive - - def _get_display_formatter(self, - dirname_output_format, - fname_output_format, - fp_format, - fp_cleaner=None): - """ generate built-in formatter function - - this is used to define both the notebook and terminal built-in - formatters as they only differ by some wrapper text for each entry - - dirname_output_format: string to use for formatting directory - names, dirname will be substituted for a single "%s" which - must appear in this string - fname_output_format: string to use for formatting file names, - if a single "%s" appears in the string, fname will be substituted - if two "%s" appear in the string, the path to fname will be - substituted for the first and fname will be substituted for the - second - fp_format: string to use for formatting filepaths, must contain - exactly two "%s" and the dirname will be substituted for the first - and fname will be substituted for the second - """ - def f(dirname, fnames, included_suffixes=None): - result = [] - # begin by figuring out which filenames, if any, - # are going to be displayed - display_fnames = [] - for fname in fnames: - if (isfile(join(dirname,fname)) and - (included_suffixes is None or - splitext(fname)[1] in included_suffixes)): - display_fnames.append(fname) - - if len(display_fnames) == 0: - # if there are no filenames to display, don't print anything - # (not even the directory name) - pass - else: - # otherwise print the formatted directory name followed by - # the formatted filenames - dirname_output_line = dirname_output_format % dirname - result.append(dirname_output_line) - for fname in display_fnames: - fp = fp_format % (dirname,fname) - if fp_cleaner is not None: - fp = fp_cleaner(fp) - try: - # output can include both a filepath and a filename... - fname_output_line = fname_output_format % (fp, fname) - except TypeError: - # ... or just a single filepath - fname_output_line = fname_output_format % fname - result.append(fname_output_line) - return result - return f - - def _get_notebook_display_formatter(self, - spacer="  "): - """ generate function to use for notebook formatting - """ - dirname_output_format = \ - self.result_html_prefix + "%s/" + self.result_html_suffix - fname_output_format = \ - self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix - fp_format = self.url_prefix + '%s/%s' - if sep == "\\": - # Working on a platform where the path separator is "\", so - # must convert these to "/" for generating a URI - def fp_cleaner(fp): - # Replace all occurrences of backslash ("\") with a forward - # slash ("/") - this is necessary on windows when a path is - # provided as input, but we must link to a URI - return fp.replace('\\','/') - else: - fp_cleaner = None - - return self._get_display_formatter(dirname_output_format, - fname_output_format, - fp_format, - fp_cleaner) - - def _get_terminal_display_formatter(self, - spacer=" "): - """ generate function to use for terminal formatting - """ - dirname_output_format = "%s/" - fname_output_format = spacer + "%s" - fp_format = '%s/%s' - - return self._get_display_formatter(dirname_output_format, - fname_output_format, - fp_format) - - def _format_path(self): - result_lines = [] - if self.recursive: - walked_dir = list(walk(self.path)) - else: - walked_dir = [next(walk(self.path))] - walked_dir.sort() - for dirname, subdirs, fnames in walked_dir: - result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes) - return '\n'.join(result_lines) - - def __repr__(self): - """return newline-separated absolute paths - """ - result_lines = [] - if self.recursive: - walked_dir = list(walk(self.path)) - else: - walked_dir = [next(walk(self.path))] - walked_dir.sort() - for dirname, subdirs, fnames in walked_dir: - result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes) - return '\n'.join(result_lines) - - )"), display_module.attr("__dict__")); - exec(py::str(R"( - -class Code(TextDisplayObject): - """Display syntax-highlighted source code. - - This uses Pygments to highlight the code for HTML and Latex output. - - Parameters - ---------- - data : str - The code as a string - url : str - A URL to fetch the code from - filename : str - A local filename to load the code from - language : str - The short name of a Pygments lexer to use for highlighting. - If not specified, it will guess the lexer based on the filename - or the code. Available lexers: http://pygments.org/docs/lexers/ - """ - def __init__(self, data=None, url=None, filename=None, language=None): - self.language = language - super().__init__(data=data, url=url, filename=filename) - - def _get_lexer(self): - if self.language: - from pygments.lexers import get_lexer_by_name - return get_lexer_by_name(self.language) - elif self.filename: - from pygments.lexers import get_lexer_for_filename - return get_lexer_for_filename(self.filename) - else: - from pygments.lexers import guess_lexer - return guess_lexer(self.data) - - def __repr__(self): - return self.data - - def _repr_html_(self): - from pygments import highlight - from pygments.formatters import HtmlFormatter - fmt = HtmlFormatter() - style = ''.format(fmt.get_style_defs('.output_html')) - return style + highlight(self.data, self._get_lexer(), fmt) - - def _repr_latex_(self): - from pygments import highlight - from pygments.formatters import LatexFormatter - return highlight(self.data, self._get_lexer(), LatexFormatter()) + publish_execution_result(self.prompt_count, self.data, self.metadata) + self.data = {} + self.metadata = {} )"), display_module.attr("__dict__")); return display_module; @@ -2192,7 +152,3 @@ class Code(TextDisplayObject): return display_module; } } - -#ifdef __GNUC__ - #pragma GCC diagnostic pop -#endif diff --git a/src/xinspect.cpp b/src/xinspect.cpp deleted file mode 100644 index 4ebf2e6e..00000000 --- a/src/xinspect.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include -#include - -#include "xeus-python/xutils.hpp" -#include "xinspect.hpp" - -#include "pybind11/pybind11.h" - -#include "xinternal_utils.hpp" - -using namespace pybind11::literals; - -namespace xpyt -{ - int jedi_minor_version() - { - py::list jedi_version = py::module::import("jedi").attr("__version__").attr("split")("."); - - static int minor_version = py::int_(jedi_version[1]).cast(); - - return minor_version; - } - - py::object static_inspect(const std::string& code) - { - py::module jedi = py::module::import("jedi"); - return jedi.attr("Interpreter")(code, py::make_tuple(py::globals())); - } - - py::object static_inspect(const std::string& code, int cursor_pos) - { - if (jedi_minor_version() < 18) - { - py::module jedi = py::module::import("jedi"); - - py::str py_code = code.substr(0, cursor_pos); - - py::int_ line = 1; - py::int_ column = 0; - if (py::len(py_code) != 0) - { - py::list lines = py_code.attr("splitlines")(); - line = py::len(lines); - column = py::len(lines[py::len(lines) - 1]); - } - - return jedi.attr("Interpreter")(py_code, py::make_tuple(py::globals()), "line"_a = line, "column"_a = column); - } - else - { - std::string sub_code = code.substr(0, cursor_pos); - return static_inspect(sub_code); - } - } - - py::list get_completions(const std::string& code, int cursor_pos) - { - if (jedi_minor_version() < 18) - { - return static_inspect(code, cursor_pos).attr("completions")(); - } - else - { - return static_inspect(code, cursor_pos).attr("complete")(); - } - } - - py::list get_completions(const std::string& code) - { - if (jedi_minor_version() < 18) - { - return static_inspect(code).attr("completions")(); - } - else - { - return static_inspect(code).attr("complete")(); - } - } - - std::string formatted_docstring_impl(py::object inter) - { - py::object definition = py::none(); - - py::list call_sig; - py::list definitions; - if (jedi_minor_version() < 18) - { - call_sig = inter.attr("call_signatures")(); - definitions = inter.attr("goto_definitions")(); - } - else - { - call_sig = inter.attr("get_signatures")(); - definitions = inter.attr("infer")(); - } - - // If it's a function call - if (py::len(call_sig) != 0) - { - definition = call_sig[0]; - } - else if (py::len(definitions) != 0) - { - definition = definitions[0]; - } - else - { - return ""; - } - - auto name = definition.attr("name").cast(); - auto docstring = definition.attr("docstring")().cast(); - auto type = definition.attr("type").cast(); - - std::string result; - - // Retrieving the argument names and default values for keyword arguments if it's a function - py::list signatures = definition.attr("get_signatures")(); - if (py::len(signatures) != 0) - { - py::list py_params = signatures[0].attr("params"); - result.append(red_text("Signature: ") + name + blue_text("(")); - - for (py::handle param : py_params) - { - py::list param_description = param.attr("to_string")().attr("split")("="); - std::string param_name = param_description[0].cast(); - - // The argument is not a kwarg (no default value) - if (py::len(param_description) == 1) - { - result.append(param_name); - } - else - { - std::string default_value = param_description[1].cast(); - result.append(param_name + blue_text("=") + green_text(default_value)); - } - - // If it's not the last element, add a comma. - if (!param.is(py_params[py::len(py_params) - 1])) - { - result.append(blue_text(", ")); - } - } - - result.append(blue_text(")")); - - // Remove signature from the docstring - py::list splitted_docstring = definition.attr("docstring")().attr("split")("\n\n", 1); - if (py::len(splitted_docstring) == 2) - { - docstring = splitted_docstring[1].cast(); - } - } - else - { - result.append(red_text("Name: ") + name); - } - - result.append(red_text("\nType: ") + type + red_text("\nDocstring: ") + docstring); - - return result; - } - - std::string formatted_docstring(const std::string& code, int cursor_pos) - { - py::object inter = static_inspect(code, cursor_pos); - return formatted_docstring_impl(inter); - } - - std::string formatted_docstring(const std::string& code) - { - py::object inter = static_inspect(code); - return formatted_docstring_impl(inter); - } -} diff --git a/src/xinspect.hpp b/src/xinspect.hpp deleted file mode 100644 index edd235da..00000000 --- a/src/xinspect.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XPYT_INSPECT_HPP -#define XPYT_INSPECT_HPP - -#include - -#include "pybind11/pybind11.h" - -namespace py = pybind11; - -namespace xpyt -{ - py::list get_completions(const std::string& code, int cursor_pos); - py::list get_completions(const std::string& code); - - std::string formatted_docstring(const std::string& code, int cursor_pos); - std::string formatted_docstring(const std::string& code); -} - -#endif diff --git a/src/xinteractiveshell.cpp b/src/xinteractiveshell.cpp deleted file mode 100644 index 1fdbba12..00000000 --- a/src/xinteractiveshell.cpp +++ /dev/null @@ -1,269 +0,0 @@ -#include "xinteractiveshell.hpp" - -#include "xeus/xinterpreter.hpp" -#include "xeus/xhistory_manager.hpp" - -#include "pybind11/eval.h" - -#include "xdisplay.hpp" -#include "xeus-python/xutils.hpp" -#include "xinspect.hpp" -#include "xnullcontext.hpp" - -using namespace pybind11::literals; -namespace py = pybind11; -namespace nl = nlohmann; - -namespace xpyt -{ - void xinteractive_shell::init_magics() - { - m_magic_core = py::module::import("IPython.core.magic"); - m_magics_module = py::module::import("IPython.core.magics"); - m_extension_module = py::module::import("IPython.core.extensions"); - - m_magics_manager = m_magic_core.attr("MagicsManager")("shell"_a=this); - m_extension_manager = m_extension_module.attr("ExtensionManager")("shell"_a=this); - - // Shell features required by extension manager - m_builtin_trap = get_nullcontext_module().attr("nullcontext")(); - - m_ipython_dir = ""; - - py::object osm_magics = m_magics_module.attr("OSMagics"); - py::object basic_magics = m_magics_module.attr("BasicMagics"); - py::object user_magics = m_magics_module.attr("UserMagics"); - py::object extension_magics = m_magics_module.attr("ExtensionMagics"); - py::object history_magics = m_magics_module.attr("HistoryMagics"); - py::object ns_magics = m_magics_module.attr("NamespaceMagics"); - py::object execution_magics = m_magics_module.attr("ExecutionMagics"); - m_magics_manager.attr("register")(osm_magics); - m_magics_manager.attr("register")(basic_magics); - m_magics_manager.attr("register")(user_magics); - m_magics_manager.attr("register")(extension_magics); - m_magics_manager.attr("register")(history_magics); - m_magics_manager.attr("register")(ns_magics); - m_magics_manager.attr("register")(execution_magics); - m_magics_manager.attr("user_magics") = user_magics("shell"_a=this); - - //select magics supported by xeus-python - auto line_magics = m_magics_manager.attr("magics")["line"]; - auto cell_magics = m_magics_manager.attr("magics")["cell"]; - line_magics = py::dict( - "cd"_a=line_magics["cd"], - "env"_a=line_magics["env"], - "set_env"_a=line_magics["set_env"], - "pwd"_a=line_magics["pwd"], - "magic"_a=line_magics["magic"], - "load_ext"_a=line_magics["load_ext"], - "pushd"_a=line_magics["pushd"], - "popd"_a=line_magics["popd"], - "dirs"_a=line_magics["dirs"], - "dhist"_a=line_magics["dhist"], - "sx"_a=line_magics["sx"], - "system"_a=line_magics["system"], - "bookmark"_a=line_magics["bookmark"], - //history magics - "history"_a=line_magics["history"], - "recall"_a=line_magics["recall"], - "rerun"_a=line_magics["rerun"], - //namespace magics - "pinfo"_a=line_magics["pinfo"], - //execution magics - "timeit"_a=line_magics["timeit"] - ); - cell_magics = py::dict( - "writefile"_a=cell_magics["writefile"], - "sx"_a=cell_magics["sx"], - "system"_a=cell_magics["system"], - //execution magics - "timeit"_a=cell_magics["timeit"] - ); - - m_magics_manager.attr("magics") = py::dict( - "line"_a=line_magics, - "cell"_a=cell_magics); - } - - - xinteractive_shell::xinteractive_shell() - { - p_history_manager = &xeus::get_interpreter().get_history_manager(); - m_hooks = hooks_object(); - m_ipy_process = py::module::import("IPython.utils.process"); - py::module os_module = py::module::import("os"); - m_db = py::dict(); - m_user_ns = py::dict("_dh"_a=py::list()); - m_dir_stack = py::list(); - m_home_dir = os_module.attr("path").attr("expanduser")("~"); - init_magics(); - } - - py::object xinteractive_shell::system(py::str cmd) - { - return m_ipy_process.attr("system")(cmd); - } - - py::object xinteractive_shell::getoutput(py::str cmd) - { - auto stream = m_ipy_process.attr("getoutput")(cmd); - return stream.attr("splitlines")(); - } - - py::object xinteractive_shell::run_line_magic(std::string name, std::string arg) - { - - py::object magic_method = m_magics_manager - .attr("magics")["line"] - .attr("get")(name); - - if (magic_method.is_none()) - { - PyErr_SetString(PyExc_ValueError, "magics not found"); - throw py::error_already_set(); - } - - // required by timeit magics (which uses user_ns as globals dict) - m_user_ns.attr("update")(py::globals()); - - return magic_method(arg); - - } - - py::object xinteractive_shell::run_cell_magic(std::string name, std::string arg, std::string body) - { - py::object magic_method = m_magics_manager.attr("magics")["cell"].attr("get")(name); - - if (magic_method.is_none()) - { - PyErr_SetString(PyExc_ValueError, "cell magics not found"); - throw py::error_already_set(); - } - - // required by timeit magics (which uses user_ns as globals dict) - m_user_ns.attr("update")(py::globals()); - - return magic_method(arg, body); - } - - void xinteractive_shell::register_magic_function(py::object func, std::string magic_kind, py::object magic_name) - { - m_magics_manager.attr("register_function")( - func, "magic_kind"_a=magic_kind, "magic_name"_a=magic_name); - } - - void xinteractive_shell::register_magics(py::args args) - { - m_magics_manager.attr("register")(*args); - } - - // manage payloads - // payloads are required by recall magic - void xinteractive_shell::set_next_input(std::string s, bool replace) - { - nl::json data = nl::json::object({ - {"text", s}, - {"source", "set_next_input"}, - {"replace", replace} - }); - - m_payloads.push_back(std::move(data)); - } - - void xinteractive_shell::inspect(std::string, std::string oname, py::kwargs) - { - auto result = formatted_docstring(oname); - nl::json data = nl::json::object({ - {"data", { - {"text/plain", result} - }}, - {"source", "page"}, - {"start", 0} - }); - - m_payloads.push_back(std::move(data)); - } - - void xinteractive_shell::clear_payloads() - { - m_payloads.clear(); - } - - const xinteractive_shell::payload_type & xinteractive_shell::get_payloads() - { - return m_payloads; - } - - // run_cell required my %rerun magic - void xinteractive_shell::run_cell(py::str code, bool) - { - // this is a placeholder for a real implementation - // it does not handle multiple statements - // nor magics parsing - py::module builtins = py::module::import("builtins"); - std::string filename = "debug_this_thread"; - auto compiled_code = builtins.attr("compile")(code, filename, "single"); - - // we need to pass user_ns because in a nested interpreter - // we loose global namespace (results of previous evals) - exec(compiled_code, m_user_ns); - } - - void xinteractive_shell::ex(py::str cmd) - { - exec(cmd, m_user_ns); - } - - py::object xinteractive_shell::ev(py::str expr) - { - return eval(expr, m_user_ns); - } - - // define getters - py::object xinteractive_shell::get_magics_manager() const - { - return m_magics_manager; - } - - - py::object xinteractive_shell::get_extension_manager() const - { - return m_extension_manager; - } - - py::dict xinteractive_shell::get_db() const - { - return m_db; - } - - py::dict xinteractive_shell::get_user_ns() const - { - return m_user_ns; - } - - py::dict xinteractive_shell::get_user_global_ns() const - { - return py::globals(); - } - - py::object xinteractive_shell::get_builtin_trap() const - { - return m_builtin_trap; - } - - py::str xinteractive_shell::get_ipython_dir() const - { - return m_ipython_dir; - } - - hooks_object xinteractive_shell::get_hooks() const - { - return m_hooks; - } - - const xeus::xhistory_manager& xinteractive_shell::get_history_manager() - { - return *p_history_manager; - } - -} diff --git a/src/xinteractiveshell.hpp b/src/xinteractiveshell.hpp deleted file mode 100644 index 880b3960..00000000 --- a/src/xinteractiveshell.hpp +++ /dev/null @@ -1,115 +0,0 @@ -#include - -#include "nlohmann/json.hpp" -#include "pybind11/pybind11.h" -#include "xdisplay.hpp" - -#include "xeus/xhistory_manager.hpp" - -#ifdef __GNUC__ - #pragma GCC diagnostic ignored "-Wattributes" -#endif - -namespace nl = nlohmann; -namespace py = pybind11; -using namespace pybind11::literals; - -namespace xpyt -{ - struct hooks_object - { - static inline void show_in_pager(py::str data, py::kwargs) - { - xdisplay(py::make_tuple(py::dict("text/plain"_a = data)), - py::dict(py::arg("raw") = true)); - } - }; - - class xinteractive_shell - { - public: - - // default constructor - xinteractive_shell(); - - // mock required methods - void register_post_execute(py::args, py::kwargs) {}; - void enable_gui(py::args, py::kwargs) {}; - void observe(py::args, py::kwargs) {}; - void showtraceback(py::args, py::kwargs) {}; - - // run system commands - py::object system(py::str cmd); - py::object getoutput(py::str cmd); - - // run magics - py::object run_line_magic(std::string name, std::string arg); - py::object run_cell_magic(std::string name, std::string arg, std::string body); - - // register magics - void register_magic_function(py::object func, std::string magic_kind, py::object magic_name); - void register_magics(py::args args); - - // required by history magics - void set_next_input(std::string s, bool replace); - void run_cell(py::str code, bool store_history); - void ex(py::str cmd); - py::object ev(py::str expr); - - // required by pinfo - void inspect(std::string, std::string oname, py::kwargs); - - // public getters - py::object get_magics_manager() const; - py::object get_extension_manager() const; - py::dict get_db() const; - py::dict get_user_ns() const; - py::dict get_user_global_ns() const; - py::object get_builtin_trap() const; - py::str get_ipython_dir() const; - hooks_object get_hooks() const; - - inline py::list get_dir_stack() const { return m_dir_stack; }; - inline py::str get_home_dir() const { return m_home_dir; }; - - const xeus::xhistory_manager& get_history_manager(); - - // payload - void clear_payloads(); - using payload_type = std::vector; - const payload_type& get_payloads(); //pure C++ not exposed to Python - - private: - - py::module m_ipy_process; - py::module m_magic_core; - py::module m_magics_module; - py::module m_extension_module; - - py::object m_magics_manager; - py::object m_extension_manager; // required by %load_ext - - // required by cd magic and others - py::dict m_db; - py::dict m_user_ns; - - // required by extension_manager - py::object m_builtin_trap; - py::str m_ipython_dir; - - // pager, required by %magics - hooks_object m_hooks; - - // required by pushd - py::list m_dir_stack; - py::str m_home_dir; - - void init_magics(); - - // history manager - const xeus::xhistory_manager* p_history_manager; - - // store jupyter message protocol payloads - payload_type m_payloads; - }; -}; diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 4de08dbe..193141a9 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -9,7 +9,6 @@ ****************************************************************************/ #include -#include #include #include #include @@ -23,19 +22,18 @@ #include "pybind11/functional.h" +#include "pybind11_json/pybind11_json.hpp" + #include "xeus-python/xinterpreter.hpp" #include "xeus-python/xeus_python_config.hpp" #include "xeus-python/xtraceback.hpp" #include "xeus-python/xutils.hpp" -#include "xpython_kernel.hpp" +#include "xcomm.hpp" +#include "xcompiler.hpp" #include "xdisplay.hpp" #include "xinput.hpp" -#include "xinspect.hpp" -#include "xinteractiveshell.hpp" #include "xinternal_utils.hpp" -#include "xis_complete.hpp" -#include "xlinecache.hpp" #include "xstream.hpp" namespace py = pybind11; @@ -46,17 +44,15 @@ namespace xpyt { interpreter::interpreter(bool redirect_output_enabled/*=true*/, bool redirect_display_enabled/*=true*/) + : m_redirect_display_enabled{redirect_display_enabled} { xeus::register_interpreter(this); if (redirect_output_enabled) { redirect_output(); } - redirect_display(redirect_display_enabled); - // Monkey patch the IPython modules later in the execution, in the configure_impl - // This is needed because the kernel needs to initialize the history_manager before - // we can expose it to Python. + m_user_ns = py::dict(); } interpreter::~interpreter() @@ -69,7 +65,6 @@ namespace xpyt { // The GIL is not held by default by the interpreter, so every time we need to execute Python code we // will need to acquire the GIL - // m_release_gil = gil_scoped_release_ptr(new py::gil_scoped_release()); } @@ -77,160 +72,256 @@ namespace xpyt py::module sys = py::module::import("sys"); - // Monkey patching "import linecache". This monkey patch does not work with Python2. - sys.attr("modules")["linecache"] = get_linecache_module(); - - py::module jedi = py::module::import("jedi"); - jedi.attr("api").attr("environment").attr("get_default_environment") = py::cpp_function([jedi] () { - jedi.attr("api").attr("environment").attr("SameEnvironment")(); - }); - // Monkey patching "from ipykernel.comm import Comm" - sys.attr("modules")["ipykernel.comm"] = get_kernel_module(); - - // Monkey patching "import IPython.display" and internal IPython.display imports - sys.attr("modules")["IPython.core.display"] = get_display_module(); - sys.attr("modules")["IPython.lib.display"] = get_display_module(); - sys.attr("modules")["IPython.display"] = get_display_module(); - - // Monkey patching "from IPython import get_ipython" - sys.attr("modules")["IPython.core.getipython"] = get_kernel_module(); + sys.attr("modules")["ipykernel.comm"] = get_comm_module(); - // Add get_ipython to global namespace - py::globals()["get_ipython"] = get_kernel_module().attr("get_ipython"); + // TODO Remove this when we use https://github.com/ipython/ipython/pull/12809 + // Monkey patching IPython.core.compilerop + sys.attr("modules")["IPython.core.compilerop"] = get_compiler_module(); - // Initializes get_ipython result - get_kernel_module().attr("get_ipython")(); - - m_has_ipython = get_kernel_module().attr("has_ipython").cast(); - - // Initialize cached inputs - py::globals()["_i"] = ""; - py::globals()["_ii"] = ""; - py::globals()["_iii"] = ""; + py::module display_module = get_display_module(); + py::module traceback_module = get_traceback_module(); + + py::dict scope; + scope["CommManager"] = get_comm_module().attr("CommManager"); + scope["set_last_error"] = traceback_module.attr("set_last_error"); + + exec(py::str(R"( +import sys + +# TODO Just import InteractiveShell when we use https://github.com/ipython/ipython/pull/12809 +# from IPython.core.interactiveshell import InteractiveShell +from IPython.core.interactiveshell import * + + +class XKernel(): + def __init__(self): + self.comm_manager = CommManager() + + +class XPythonShell(InteractiveShell): + def __init__(self, *args, **kwargs): + super(XPythonShell, self).__init__(*args, **kwargs) + self.kernel = XKernel() + + def enable_gui(self, gui=None): + """Not implemented yet.""" + pass + + # Workaround for preventing IPython to show error traceback + # We catch it and will display it later properly + def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None, + exception_only=False, running_compiled_code=False): + try: + etype, value, tb = self._get_exc_info(exc_tuple) + except ValueError: + print('No traceback available to show.', file=sys.stderr) + return + + set_last_error(etype, value, tb) + + # TODO Remove this method when we use https://github.com/ipython/ipython/pull/12809 + async def run_cell_async( + self, + raw_cell: str, + store_history=False, + silent=False, + shell_futures=True, + *, + transformed_cell=None, + preprocessing_exc_tuple=None + ): + info = ExecutionInfo( + raw_cell, store_history, silent, shell_futures) + result = ExecutionResult(info) + + if (not raw_cell) or raw_cell.isspace(): + self.last_execution_succeeded = True + self.last_execution_result = result + return result + + if silent: + store_history = False + + if store_history: + result.execution_count = self.execution_count + + def error_before_exec(value): + if store_history: + self.execution_count += 1 + result.error_before_exec = value + self.last_execution_succeeded = False + self.last_execution_result = result + return result + + self.events.trigger('pre_execute') + if not silent: + self.events.trigger('pre_run_cell', info) + + if transformed_cell is None: + warnings.warn( + "`run_cell_async` will not call `transform_cell`" + " automatically in the future. Please pass the result to" + " `transformed_cell` argument and any exception that happen" + " during the" + "transform in `preprocessing_exc_tuple` in" + " IPython 7.17 and above.", + DeprecationWarning, + stacklevel=2, + ) + # If any of our input transformation (input_transformer_manager or + # prefilter_manager) raises an exception, we store it in this variable + # so that we can display the error after logging the input and storing + # it in the history. + try: + cell = self.transform_cell(raw_cell) + except Exception: + preprocessing_exc_tuple = sys.exc_info() + cell = raw_cell # cell has to exist so it can be stored/logged + else: + preprocessing_exc_tuple = None + else: + if preprocessing_exc_tuple is None: + cell = transformed_cell + else: + cell = raw_cell + + # Store raw and processed history + if store_history: + self.history_manager.store_inputs(self.execution_count, + cell, raw_cell) + if not silent: + self.logger.log(cell, raw_cell) + + # Display the exception if input processing failed. + if preprocessing_exc_tuple is not None: + self.showtraceback(preprocessing_exc_tuple) + if store_history: + self.execution_count += 1 + return error_before_exec(preprocessing_exc_tuple[1]) + + # Our own compiler remembers the __future__ environment. If we want to + # run code with a separate __future__ environment, use the default + # compiler + compiler = self.compile if shell_futures else CachingCompiler() + + _run_async = False + + with self.builtin_trap: + cell_name = self.compile.cache(cell, self.execution_count, raw_cell) + + with self.display_trap: + # Compile to bytecode + try: + code_ast = compiler.ast_parse(cell, filename=cell_name) + except self.custom_exceptions as e: + etype, value, tb = sys.exc_info() + self.CustomTB(etype, value, tb) + return error_before_exec(e) + except IndentationError as e: + self.showindentationerror() + return error_before_exec(e) + except (OverflowError, SyntaxError, ValueError, TypeError, + MemoryError) as e: + self.showsyntaxerror() + return error_before_exec(e) + + # Apply AST transformations + try: + code_ast = self.transform_ast(code_ast) + except InputRejected as e: + self.showtraceback() + return error_before_exec(e) + + # Give the displayhook a reference to our ExecutionResult so it + # can fill in the output value. + self.displayhook.exec_result = result + + # Execute the user code + interactivity = "none" if silent else self.ast_node_interactivity + if _run_async: + interactivity = 'async' + + has_raised = await self.run_ast_nodes(code_ast.body, cell_name, + interactivity=interactivity, compiler=compiler, result=result) + + self.last_execution_succeeded = not has_raised + self.last_execution_result = result + + # Reset this so later displayed values do not modify the + # ExecutionResult + self.displayhook.exec_result = None + + if store_history: + # Write output to the database. Does nothing unless + # history output logging is enabled. + self.history_manager.store_output(self.execution_count) + # Each cell is a *single* input, regardless of how many lines it has + self.execution_count += 1 + + return result + )"), scope); + + if (m_redirect_display_enabled) + { + m_ipython_shell = scope["XPythonShell"].attr("instance")( + "display_pub_class"_a=display_module.attr("XDisplayPublisher"), + "displayhook_class"_a=display_module.attr("XDisplayHook"), + // TODO Uncomment this when we use https://github.com/ipython/ipython/pull/12809 + // "compiler_class"_a=get_compiler_module().attr("XCachingCompiler"), + "user_ns"_a=m_user_ns + ); + + m_displayhook = m_ipython_shell.attr("displayhook"); + } + else + { + m_ipython_shell = scope["XPythonShell"].attr("instance")( + "display_pub_class"_a=display_module.attr("XDisplayPublisher"), + // TODO Uncomment this when we use https://github.com/ipython/ipython/pull/12809 + // "compiler_class"_a=get_compiler_module().attr("XCachingCompiler"), + "user_ns"_a=m_user_ns + ); + + m_displayhook = display_module.attr("XDisplayHook")( + "parent"_a=m_ipython_shell, "shell"_a=m_ipython_shell, "cache_size"_a=m_ipython_shell.attr("cache_size") + ); + } - load_extensions(); + m_ipython_shell.attr("compile").attr("filename_mapper") = traceback_module.attr("register_filename_mapping"); } - nl::json interpreter::execute_request_impl(int execution_count, + nl::json interpreter::execute_request_impl(int /*execution_count*/, const std::string& code, bool silent, - bool /*store_history*/, - nl::json /*user_expressions*/, + bool store_history, + nl::json user_expressions, bool allow_stdin) { py::gil_scoped_acquire acquire; nl::json kernel_res; - py::str code_copy; - if(m_has_ipython) - { - py::module input_transformers = py::module::import("IPython.core.inputtransformer2"); - py::object transformer_manager = input_transformers.attr("TransformerManager")(); - code_copy = transformer_manager.attr("transform_cell")(code); - } - else - { - // Special handling of question mark whe IPython is not installed. Otherwise, this - // is already implemented in the IPython.core.inputtransformer2 module that we - // import. This is a temporary implementation until: - // - either we reimplement the parsing logic in xeus-python - // - or this logic is extracted from IPython into a dedicated package, that becomes - // a dependency of both xeus-python and IPython. - if (code.size() >= 2 && code[0] == '?') - { - std::string result = formatted_docstring(code); - if (result.empty()) - { - result = "Object " + code.substr(1) + " not found."; - } - - kernel_res["status"] = "ok"; - kernel_res["payload"] = nl::json::array(); - kernel_res["payload"][0] = nl::json::object({ - {"data", { - {"text/plain", result} - }}, - {"source", "page"}, - {"start", 0} - }); - kernel_res["user_expressions"] = nl::json::object(); - - return kernel_res; - } - code_copy = code; - } + py::module traceback = get_traceback_module(); // Scope guard performing the temporary monkey patching of input and // getpass with a function sending input_request messages. auto input_guard = input_redirection(allow_stdin); - try - { - // Import modules - py::module ast = py::module::import("ast"); - py::module builtins = py::module::import("builtins"); - - // Parse code to AST - py::object code_ast = ast.attr("parse")(code_copy, "", "exec"); - py::list expressions = code_ast.attr("body"); - - std::string filename = get_cell_tmp_file(code); - register_filename_mapping(filename, execution_count); - - // Caching the input code - py::module linecache = py::module::import("linecache"); - linecache.attr("xupdatecache")(code, filename); - - // If the last statement is an expression, we compile it separately - // in an interactive mode (This will trigger the display hook) - py::object last_stmt = expressions[py::len(expressions) - 1]; - if (py::isinstance(last_stmt, ast.attr("Expr"))) - { - code_ast.attr("body").attr("pop")(); + py::object ipython_res = m_ipython_shell.attr("run_cell")(code, "store_history"_a=store_history, "silent"_a=silent); - py::list interactive_nodes; - interactive_nodes.append(last_stmt); - - py::object interactive_ast = ast.attr("Interactive")(interactive_nodes); - - py::object compiled_code = builtins.attr("compile")(code_ast, filename, "exec"); - - py::object compiled_interactive_code = builtins.attr("compile")(interactive_ast, filename, "single"); - - if (m_displayhook.ptr() != nullptr) - { - m_displayhook.attr("set_execution_count")(execution_count); - } - - exec(compiled_code); - exec(compiled_interactive_code); - } - else - { - py::object compiled_code = builtins.attr("compile")(code_ast, filename, "exec"); - exec(compiled_code); - } + // Get payload + kernel_res["payload"] = m_ipython_shell.attr("payload_manager").attr("read_payload")(); + m_ipython_shell.attr("payload_manager").attr("clear_payload")(); + if (traceback.attr("get_last_error")().is_none()) + { kernel_res["status"] = "ok"; - kernel_res["user_expressions"] = nl::json::object(); - if (m_has_ipython) - { - xinteractive_shell* xshell = get_kernel_module() - .attr("get_ipython")() - .cast(); - auto payload = xshell->get_payloads(); - kernel_res["payload"] = payload; - xshell->clear_payloads(); - } - else - { - kernel_res["payload"] = nl::json::array(); - } + kernel_res["user_expressions"] = m_ipython_shell.attr("user_expressions")(user_expressions); } - catch (py::error_already_set& e) + else { - xerror error = extract_error(e); + py::list pyerror = traceback.attr("get_last_error")(); + xerror error = extract_error(pyerror[0], pyerror[1], pyerror[2]); if (!silent) { @@ -241,12 +332,9 @@ namespace xpyt kernel_res["ename"] = error.m_ename; kernel_res["evalue"] = error.m_evalue; kernel_res["traceback"] = error.m_traceback; - } - // Cache inputs - py::globals()["_iii"] = py::globals()["_ii"]; - py::globals()["_ii"] = py::globals()["_i"]; - py::globals()["_i"] = code; + traceback.attr("reset_last_error")(); + } return kernel_res; } @@ -257,46 +345,74 @@ namespace xpyt { py::gil_scoped_acquire acquire; nl::json kernel_res; - std::vector matches; - int cursor_start = cursor_pos; - py::list completions = get_completions(code, cursor_pos); - - if (py::len(completions) != 0) - { - cursor_start -= py::len(completions[0].attr("name_with_symbols")) - py::len(completions[0].attr("complete")); - for (py::handle completion : completions) - { - matches.push_back(completion.attr("name_with_symbols").cast()); - } - } - - kernel_res["cursor_start"] = cursor_start; - kernel_res["cursor_end"] = cursor_pos; - kernel_res["matches"] = matches; - kernel_res["status"] = "ok"; + py::module completer = py::module::import("IPython.core.completer"); + + py::dict scope; + scope["provisionalcompleter"] = completer.attr("provisionalcompleter"); + scope["rectify_completions"] = completer.attr("rectify_completions"); + scope["shell"] = m_ipython_shell; + scope["code"] = code; + scope["cursor_pos"] = cursor_pos; + exec(py::str(R"( +with provisionalcompleter(): + raw_completions = shell.Completer.completions(code, cursor_pos) + completions = list(rectify_completions(code, raw_completions)) + + comps = [] + for comp in completions: + comps.append(dict( + start=comp.start, + end=comp.end, + text=comp.text, + type=comp.type, + )) + +if completions: + cursor_start = completions[0].start + cursor_end = completions[0].end + matches = [c.text for c in completions] +else: + cursor_start = cursor_pos + cursor_end = cursor_pos + matches = [] + )"), scope); + + kernel_res["matches"] = scope["matches"]; + kernel_res["cursor_end"] = scope["cursor_end"]; + kernel_res["cursor_start"] = scope["cursor_start"]; kernel_res["metadata"] = nl::json::object(); + kernel_res["status"] = "ok"; + return kernel_res; } nl::json interpreter::inspect_request_impl(const std::string& code, int cursor_pos, - int /*detail_level*/) + int detail_level) { py::gil_scoped_acquire acquire; nl::json kernel_res; - nl::json pub_data; + nl::json data = nl::json::object(); + bool found = false; - std::string docstring = formatted_docstring(code, cursor_pos); + py::module tokenutil = py::module::import("IPython.utils.tokenutil"); + py::str name = tokenutil.attr("token_at_cursor")(code, cursor_pos); - bool found = false; - if (!docstring.empty()) + try { + data = m_ipython_shell.attr("object_inspect_mime")( + name, + "detail_level"_a=detail_level + ); found = true; - pub_data["text/plain"] = docstring; + } + catch (py::error_already_set& e) + { + // pass } - kernel_res["data"] = pub_data; + kernel_res["data"] = data; kernel_res["metadata"] = nl::json::object(); kernel_res["found"] = found; kernel_res["status"] = "ok"; @@ -308,9 +424,13 @@ namespace xpyt py::gil_scoped_acquire acquire; nl::json kernel_res; - py::module completion_module = get_completion_module(); - py::list result = completion_module.attr("check_complete")(code); + py::object transformer_manager = py::getattr(m_ipython_shell, "input_transformer_manager", py::none()); + if (transformer_manager.is_none()) + { + transformer_manager = m_ipython_shell.attr("input_splitter"); + } + py::list result = transformer_manager.attr("check_complete")(code); auto status = result[0].cast(); kernel_res["status"] = status; @@ -381,16 +501,7 @@ namespace xpyt nl::json reply; try { - // Import modules - py::module ast = py::module::import("ast"); - py::module builtins = py::module::import("builtins"); - - // Parse code to AST - py::object code_ast = ast.attr("parse")(code, "", "exec"); - - std::string filename = "debug_this_thread"; - py::object compiled_code = builtins.attr("compile")(code_ast, filename, "exec"); - exec(compiled_code); + exec(py::str(code)); reply["status"] = "ok"; } @@ -419,69 +530,4 @@ namespace xpyt sys.attr("stderr") = stream_module.attr("Stream")("stderr"); } - void interpreter::redirect_display(bool install_hook/*=true*/) - { - py::module display_module = get_display_module(); - m_displayhook = display_module.attr("DisplayHook")(); - if (install_hook) - { - py::module sys = py::module::import("sys"); - sys.attr("displayhook") = m_displayhook; - } - - // Expose display functions to Python - py::globals()["display"] = display_module.attr("display"); - py::globals()["update_display"] = display_module.attr("update_display"); - } - - void interpreter::load_extensions() - { - if (m_has_ipython) - { - py::module os = py::module::import("os"); - py::module path = py::module::import("os.path"); - py::module sys = py::module::import("sys"); - py::module fnmatch = py::module::import("fnmatch"); - - py::str extensions_path = path.attr("join")(sys.attr("exec_prefix"), "etc", "xeus-python", "extensions"); - - if (!is_pyobject_true(path.attr("exists")(extensions_path))) - { - return; - } - - py::list list_files = os.attr("listdir")(extensions_path); - - xinteractive_shell* xshell = get_kernel_module() - .attr("get_ipython")() - .cast(); - py::object extension_manager = xshell->get_extension_manager(); - - for (const py::handle& file : list_files) - { - if (!is_pyobject_true(fnmatch.attr("fnmatch")(file, "*.json"))) - { - continue; - } - - try - { - std::ifstream config_file(py::str(path.attr("join")(extensions_path, file)).cast()); - nl::json config; - config_file >> config; - - if (config["enabled"].get()) - { - extension_manager.attr("load_extension")(config["module"].get()); - } - } - catch (py::error_already_set& e) - { - xerror error = extract_error(e); - - std::cerr << "Warning: Failed loading extension with: " << error.m_ename << ": " << error.m_evalue << std::endl; - } - } - } - } } diff --git a/src/xis_complete.cpp b/src/xis_complete.cpp deleted file mode 100644 index 80760fa8..00000000 --- a/src/xis_complete.cpp +++ /dev/null @@ -1,237 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include "pybind11/pybind11.h" - -#include "xeus-python/xutils.hpp" - -#include "xinternal_utils.hpp" - -namespace py = pybind11; - -namespace xpyt -{ - - /********************* - * completion module * - *********************/ - - py::module get_completion_module_impl() - { - py::module completion_module = create_module("completion"); - - exec(py::str(R"( -# Implementation from https://github.com/ipython/ipython/blob/master/IPython/core/inputtransformer2.py -import sys -import re -import tokenize -import warnings -from codeop import compile_command - -_indent_re = re.compile(r'^[ \t]+') - -def find_last_indent(lines): - m = _indent_re.match(lines[-1]) - if not m: - return 0 - return len(m.group(0).replace('\t', ' '*4)) - -def leading_indent(lines): - if not lines: - return lines - m = _indent_re.match(lines[0]) - if not m: - return lines - space = m.group(0) - n = len(space) - return [l[n:] if l.startswith(space) else l - for l in lines] - -class PromptStripper: - def __init__(self, prompt_re, initial_re=None): - self.prompt_re = prompt_re - self.initial_re = initial_re or prompt_re - - def _strip(self, lines): - return [self.prompt_re.sub('', l, count=1) for l in lines] - - def __call__(self, lines): - if not lines: - return lines - if self.initial_re.match(lines[0]) or \ - (len(lines) > 1 and self.prompt_re.match(lines[1])): - return self._strip(lines) - return lines - -classic_prompt = PromptStripper( - prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'), - initial_re=re.compile(r'^>>>( |$)') -) - -interactive_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')) - -def _extract_token(token, tokens_by_line, parenlev): - tokens_by_line[-1].append(token) - if (token.type == tokenize.NEWLINE) \ - or ((token.type == tokenize.NL) and (parenlev <= 0)): - tokens_by_line.append([]) - elif token.string in {'(', '[', '{'}: - parenlev += 1 - elif token.string in {')', ']', '}'}: - if parenlev > 0: - parenlev -= 1 - -if sys.version_info.major == 3: - def _gen_tokens(lines, tokens_by_line, parenlev): - for token in tokenize.generate_tokens(iter(lines).__next__): - _extract_token(token, tokens_by_line, parenlev) -else: - class Token(): - def __init__(self, token_tuple): - self.type = token_tuple[0] - self.string = token_tuple[1] - self.start = token_tuple[2] - self.end = token_tuple[3] - self.line = token_tuple[4] - - def _gen_tokens(lines, tokens_by_line, parenlev): - for token_tuple in tokenize.generate_tokens(iter(lines).next): - token = Token(token_tuple) - _extract_token(token, tokens_by_line, parenlev) - -def make_tokens_by_line(lines): - tokens_by_line = [[]] - if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')): - warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified") - parenlev = 0 - try: - _gen_tokens(lines, tokens_by_line, parenlev) - except tokenize.TokenError: - # Input ended in a multiline string or expression. That's OK for us. - pass - - if not tokens_by_line[-1]: - tokens_by_line.pop() - - return tokens_by_line - -def check_complete(cell): - # Remember if the lines ends in a new line. - ends_with_newline = False - for character in reversed(cell): - if character == '\n': - ends_with_newline = True - break - elif character.strip(): - break - else: - continue - - if not ends_with_newline: - cell += '\n' - - lines = cell.splitlines(True) - - if not lines: - return 'complete', None - - if lines[-1].endswith('\\'): - # Explicit backslash continuation - return 'incomplete', find_last_indent(lines) - - cleanup_transforms = [ - leading_indent, - classic_prompt, - interactive_prompt, - ] - try: - for transform in cleanup_transforms: - lines = transform(lines) - except SyntaxError: - return 'invalid', None - - if lines[0].startswith('%%'): - # Special case for cell magics - completion marked by blank line - if lines[-1].strip(): - return 'incomplete', find_last_indent(lines) - else: - return 'complete', None - - tokens_by_line = make_tokens_by_line(lines) - - if not tokens_by_line: - return 'incomplete', find_last_indent(lines) - - if tokens_by_line[-1][-1].type != tokenize.ENDMARKER: - # We're in a multiline string or expression - return 'incomplete', find_last_indent(lines) - - newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} - - # Pop the last line which only contains DEDENTs and ENDMARKER - last_token_line = None - if {t.type for t in tokens_by_line[-1]} in [ - {tokenize.DEDENT, tokenize.ENDMARKER}, - {tokenize.ENDMARKER} - ] and len(tokens_by_line) > 1: - last_token_line = tokens_by_line.pop() - - while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types: - tokens_by_line[-1].pop() - - if len(tokens_by_line) == 1 and not tokens_by_line[-1]: - return 'incomplete', 0 - - if tokens_by_line[-1][-1].string == ':': - # The last line starts a block (e.g. 'if foo:') - ix = 0 - while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}: - ix += 1 - - indent = tokens_by_line[-1][ix].start[1] - return 'incomplete', indent + 4 - - if tokens_by_line[-1][0].line.endswith('\\'): - return 'incomplete', None - - # At this point, our checks think the code is complete (or invalid). - # We'll use codeop.compile_command to check this with the real parser - try: - with warnings.catch_warnings(): - warnings.simplefilter('error', SyntaxWarning) - res = compile_command(''.join(lines), symbol='exec') - except (SyntaxError, OverflowError, ValueError, TypeError, - MemoryError, SyntaxWarning): - return 'invalid', None - else: - if res is None: - return 'incomplete', find_last_indent(lines) - - if last_token_line and last_token_line[0].type == tokenize.DEDENT: - if ends_with_newline: - return 'complete', None - return 'incomplete', find_last_indent(lines) - - # If there's a blank line at the end, assume we're ready to execute - if not lines[-1].strip(): - return 'complete', None - - return 'complete', None - )"), completion_module.attr("__dict__")); - - return completion_module; - } - - py::module get_completion_module() - { - static py::module completion_module = get_completion_module_impl(); - return completion_module; - } -} diff --git a/src/xlinecache.cpp b/src/xlinecache.cpp deleted file mode 100644 index 8ee2a42b..00000000 --- a/src/xlinecache.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include "xeus/xinterpreter.hpp" - -#include "pybind11/pybind11.h" -#include "pybind11/functional.h" - -#include "xeus-python/xutils.hpp" - -#include "xlinecache.hpp" -#include "xinternal_utils.hpp" - -namespace py = pybind11; - -namespace xpyt -{ - /****************************** - * xcheckcache implementation * - ******************************/ - - void xcheckcache(const py::str& filename = py::none()) - { - py::module linecache = py::module::import("linecache"); - - linecache.attr("_checkcache_orig")(filename); - linecache.attr("cache").attr("update")(linecache.attr("xcache")); - } - - /******************************* - * xupdatecache implementation * - *******************************/ - - void xupdatecache(const py::str& code, const py::str& filename) - { - py::module time = py::module::import("time"); - py::module linecache = py::module::import("linecache"); - - py::tuple entry = py::make_tuple(py::len(code), time.attr("time")(), code.attr("splitlines")(true), filename); - - linecache.attr("cache")[filename] = entry; - linecache.attr("xcache")[filename] = entry; - } - - /******************** - * linecache module * - ********************/ - - py::module get_linecache_module_impl() - { - py::module linecache_module = create_module("linecache"); - - exec(py::str(R"( -from linecache import getline, getlines, updatecache, cache, clearcache, lazycache -from linecache import checkcache as _checkcache_orig - -xcache = {} - )"), linecache_module.attr("__dict__")); - - linecache_module.def("checkcache", - xcheckcache, - py::arg("filename") = py::none() - ); - - linecache_module.def("xupdatecache", - xupdatecache, - py::arg("code"), - py::arg("filename") - ); - - return linecache_module; - } - - py::module get_linecache_module() - { - static py::module linecache_module = get_linecache_module_impl(); - return linecache_module; - } -} diff --git a/src/xlinecache.hpp b/src/xlinecache.hpp deleted file mode 100644 index 12815271..00000000 --- a/src/xlinecache.hpp +++ /dev/null @@ -1,24 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XPYT_LINECACHE_HPP -#define XPYT_LINECACHE_HPP - -#include "pybind11/pybind11.h" -#include "pybind11/functional.h" - -namespace py = pybind11; - -namespace xpyt -{ - py::module get_linecache_module(); -} - -#endif diff --git a/src/xnullcontext.cpp b/src/xnullcontext.cpp deleted file mode 100644 index 3e64ae0b..00000000 --- a/src/xnullcontext.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include "xeus/xinterpreter.hpp" - -#include "pybind11/pybind11.h" -#include "pybind11/functional.h" - -#include "xeus-python/xutils.hpp" - -#include "xnullcontext.hpp" -#include "xinternal_utils.hpp" - -namespace py = pybind11; - -namespace xpyt -{ - /********************** - * nullcontext module * - **********************/ - - py::module get_nullcontext_module_impl() - { - py::module nullcontext_module = create_module("xeus_nullcontext"); - exec(py::str(R"( -from contextlib import AbstractContextManager -class nullcontext(AbstractContextManager): - """nullcontext for contextlib.nullcontext""" - def __init__(self, enter_result=None): - self.enter_result = enter_result - def __enter__(self): - return self.enter_result - def __exit__(self, *excinfo): - pass - )"), nullcontext_module.attr("__dict__")); - return nullcontext_module; - } - - py::module get_nullcontext_module() - { - static py::module nullcontext_module = get_nullcontext_module_impl(); - return nullcontext_module; - } -} diff --git a/src/xnullcontext.hpp b/src/xnullcontext.hpp deleted file mode 100644 index 873b15f7..00000000 --- a/src/xnullcontext.hpp +++ /dev/null @@ -1,25 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#ifndef XPYT_NULLCONTEXT_HPP -#define XPYT_NULLCONTEXT_HPP - -#include "pybind11/pybind11.h" -#include "pybind11/functional.h" - -namespace py = pybind11; - -namespace xpyt -{ - // Provides a polyfill for Python 3.7's contextlib.nullcontext - py::module get_nullcontext_module(); -} - -#endif diff --git a/src/xpython_kernel.cpp b/src/xpython_kernel.cpp deleted file mode 100644 index 8c700c9a..00000000 --- a/src/xpython_kernel.cpp +++ /dev/null @@ -1,444 +0,0 @@ -/*************************************************************************** -* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * -* Wolf Vollprecht * -* Copyright (c) 2018, QuantStack * -* * -* Distributed under the terms of the BSD 3-Clause License. * -* * -* The full license is in the file LICENSE, distributed with this software. * -****************************************************************************/ - -#include -#include - -#include "nlohmann/json.hpp" - -#include "xeus/xcomm.hpp" -#include "xeus/xinterpreter.hpp" - -#include "pybind11_json/pybind11_json.hpp" - -#include "pybind11/pybind11.h" -#include "pybind11/functional.h" -#include "pybind11/eval.h" - -#include "xeus-python/xutils.hpp" - -#include "xpython_kernel.hpp" -#include "xinteractiveshell.hpp" -#include "xinternal_utils.hpp" - -namespace py = pybind11; -namespace nl = nlohmann; - -namespace xpyt -{ - /********************* - * xcomm declaration * - ********************/ - - class xcomm - { - public: - - using python_callback_type = std::function; - using cpp_callback_type = std::function; - using zmq_buffers_type = std::vector; - - xcomm(const py::args& args, const py::kwargs& kwargs); - xcomm(xeus::xcomm&& comm); - xcomm(xcomm&& comm) = default; - virtual ~xcomm(); - - std::string comm_id() const; - bool kernel() const; - - void close(const py::args& args, const py::kwargs& kwargs); - void send(const py::args& args, const py::kwargs& kwargs); - void on_msg(const python_callback_type& callback); - void on_close(const python_callback_type& callback); - - private: - - xeus::xtarget* target(const py::kwargs& kwargs) const; - xeus::xguid id(const py::kwargs& kwargs) const; - cpp_callback_type cpp_callback(const python_callback_type& callback) const; - - xeus::xcomm m_comm; - }; - - struct xcomm_manager - { - xcomm_manager() = default; - - void register_target(const py::str& target_name, const py::object& callback); - }; - - /************************ - * xcomm implementation * - ************************/ - - xcomm::xcomm(const py::args& /*args*/, const py::kwargs& kwargs) - : m_comm(target(kwargs), id(kwargs)) - { - m_comm.open( - kwargs.attr("get")("metadata", py::dict()), - kwargs.attr("get")("data", py::dict()), - pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) - ); - } - - xcomm::xcomm(xeus::xcomm&& comm) - : m_comm(std::move(comm)) - { - } - - xcomm::~xcomm() - { - } - - std::string xcomm::comm_id() const - { - return m_comm.id(); - } - - bool xcomm::kernel() const - { - return true; - } - - void xcomm::close(const py::args& /*args*/, const py::kwargs& kwargs) - { - m_comm.close( - kwargs.attr("get")("metadata", py::dict()), - kwargs.attr("get")("data", py::dict()), - pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) - ); - } - - void xcomm::send(const py::args& /*args*/, const py::kwargs& kwargs) - { - m_comm.send( - kwargs.attr("get")("metadata", py::dict()), - kwargs.attr("get")("data", py::dict()), - pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list())) - ); - } - - void xcomm::on_msg(const python_callback_type& callback) - { - m_comm.on_message(cpp_callback(callback)); - } - - void xcomm::on_close(const python_callback_type& callback) - { - m_comm.on_close(cpp_callback(callback)); - } - - xeus::xtarget* xcomm::target(const py::kwargs& kwargs) const - { - std::string target_name = kwargs["target_name"].cast(); - return xeus::get_interpreter().comm_manager().target(target_name); - } - - xeus::xguid xcomm::id(const py::kwargs& kwargs) const - { - if (py::hasattr(kwargs, "comm_id")) - { - // TODO: prevent copy - return xeus::xguid(kwargs["comm_id"].cast()); - } - else - { - return xeus::new_xguid(); - } - } - - auto xcomm::cpp_callback(const python_callback_type& py_callback) const -> cpp_callback_type - { - return [this, py_callback](const xeus::xmessage& msg) { - XPYT_HOLDING_GIL(py_callback(cppmessage_to_pymessage(msg))) - }; - } - - void xcomm_manager::register_target(const py::str& target_name, const py::object& callback) - { - auto target_callback = [&callback] (xeus::xcomm&& comm, const xeus::xmessage& msg) { - XPYT_HOLDING_GIL(callback(xcomm(std::move(comm)), cppmessage_to_pymessage(msg))); - }; - - xeus::get_interpreter().comm_manager().register_comm_target( - static_cast(target_name), target_callback - ); - } - - /**************** - * mock objects * - ****************/ - - namespace detail - { - struct compiler_object - { - py::module builtins; - - py::object compile(py::object source, py::str filename, py::str mode, int flags=0) - { - return builtins.attr("compile")(source, filename, mode, flags); - } - - compiler_object() - { - builtins = py::module::import("builtins"); - } - - py::object operator()(py::object source, py::str filename, py::str mode) - { - return compile(source, filename, mode, 0); - } - - py::object ast_parse( - py::str source, - py::str filename="", - py::str symbol="exec") - { - auto ast = py::module::import("ast"); - return compile(source, filename, symbol, py::cast(ast.attr("PyCF_ONLY_AST"))); - } - }; - } - - struct xmock_ipython - { - void register_post_execute(py::args, py::kwargs) {}; - void enable_gui(py::args, py::kwargs) {}; - void observe(py::args, py::kwargs) {}; - void showtraceback(py::args, py::kwargs) {}; - }; - - struct xmock_kernel - { - xmock_kernel() = default; - - inline py::object parent_header() const - { - return py::dict(py::arg("header")=xeus::get_interpreter().parent_header().get()); - } - - xcomm_manager m_comm_manager; - }; - - /***************** - * kernel module * - *****************/ - - void bind_history_manager(py::module& kernel_module) - { - py::class_(kernel_module, "HistoryManager") - .def_property_readonly("session_number", [](xeus::xhistory_manager&){return 0;}) - .def("get_range", [](xeus::xhistory_manager& me, - int session, - int start, - int stop, - bool raw, - bool output) {return me.get_range(session, start, stop, raw, output)["history"];}, - py::arg("session")=0, - py::arg("start")=0, - py::arg("stop")=1000, - py::arg("raw")=true, - py::arg("output")=false) - .def("get_range_by_str", - [](const xeus::xhistory_manager& me, py::str range_str, bool raw, bool output) - { - py::list range_split = range_str.attr("split")("-"); - int start = std::stoi(py::cast(range_split[0])); - int stop = (range_split.size() > 1) ? std::stoi(py::cast(range_split[1])) : start + 1; - int session = 0; - return me.get_range(session, start - 1, stop - 1, raw, output)["history"]; - }, - py::arg("range_str"), - py::arg("raw")=true, - py::arg("output")=false) - .def("get_tail", - [](const xeus::xhistory_manager& me, int last_n, bool raw, bool output) - { - return me.get_tail(last_n, raw, output)["history"]; - }, - py::arg("last_n"), - py::arg("raw")=true, - py::arg("output")=false - ) - .def("search", - [](const xeus::xhistory_manager& me, std::string pattern, bool raw, bool output, py::object py_n, bool unique) - { - int n = py_n.is_none() ? 1000 : py::cast(py_n); - return me.search(pattern, raw, output, n, unique)["history"]; - }, - py::arg("pattern")="*", - py::arg("raw")=true, - py::arg("output")=false, - py::arg("n") = py::none(), - py::arg("unique")=false); - } - - void bind_interactive_shell(py::module& kernel_module) - { - // define compiler class for timeit magic - py::class_ Compiler(kernel_module, "Compiler"); - Compiler.def(py::init<>()) - .def("__call__", &detail::compiler_object::operator(), py::is_operator()) - .def("ast_parse", &detail::compiler_object::ast_parse, - py::arg("code"), - py::arg("filename")="", - py::arg("symbol")="exec"); - - py::class_(kernel_module, "XInteractiveShell", py::dynamic_attr()) - .def(py::init<>()) - .def_property_readonly("magics_manager", &xinteractive_shell::get_magics_manager) - .def_property_readonly("extension_manager", &xinteractive_shell::get_extension_manager) - .def_property_readonly("hooks", &xinteractive_shell::get_hooks) - .def_property_readonly("db", &xinteractive_shell::get_db) - .def_property_readonly("user_ns", &xinteractive_shell::get_user_ns) - .def_property_readonly("user_global_ns", &xinteractive_shell::get_user_global_ns) - .def_property_readonly("builtin_trap", &xinteractive_shell::get_builtin_trap) - .def_property_readonly("ipython_dir", &xinteractive_shell::get_ipython_dir) - .def_property_readonly("dir_stack", &xinteractive_shell::get_dir_stack) - .def_property_readonly("home_dir", &xinteractive_shell::get_home_dir) - .def_property_readonly("history_manager", &xinteractive_shell::get_history_manager) - .def("run_line_magic", &xinteractive_shell::run_line_magic) - .def("run_cell_magic", &xinteractive_shell::run_cell_magic) - // magic method is deprecated but some magic functions still use it - .def("magic", &xinteractive_shell::run_line_magic, "name"_a, "arg"_a="") - .def("system", &xinteractive_shell::system) - .def("getoutput", &xinteractive_shell::getoutput) - .def("register_post_execute", &xinteractive_shell::register_post_execute) - .def("enable_gui", &xinteractive_shell::enable_gui) - .def("showtraceback", &xinteractive_shell::showtraceback) - .def("observe", &xinteractive_shell::observe) - // for timeit timeit - .def("transform_cell", [](xinteractive_shell &, py::str raw_cell){return raw_cell;}) - .def("transform_ast", [](xinteractive_shell &, py::object ast){return ast;}) - // for pinfo (?magic) - .def("_inspect", &xinteractive_shell::inspect) - // generic magics code - .def("run_cell", &xinteractive_shell::run_cell, - py::arg("code"), - py::arg("store_history")=false) - .def("ex", &xinteractive_shell::ex, py::arg("cmd")) - .def("ev", &xinteractive_shell::ev, py::arg("expr")) - .def("register_magic_function", - &xinteractive_shell::register_magic_function, - "Register magic function", - py::arg("func"), - py::arg("magic_kind")="line", - py::arg("magic_name")=py::none()) - .def("register_magics", &xinteractive_shell::register_magics) - .def("set_next_input", &xinteractive_shell::set_next_input, - py::arg("text"), - py::arg("replace")=false) - .attr("compile") = Compiler(); - } - - void bind_comm(py::module& kernel_module) - { - py::class_(kernel_module, "Comm") - .def(py::init()) - .def("close", &xcomm::close) - .def("send", &xcomm::send) - .def("on_msg", &xcomm::on_msg) - .def("on_close", &xcomm::on_close) - .def_property_readonly("comm_id", &xcomm::comm_id) - .def_property_readonly("kernel", &xcomm::kernel); - - py::class_(kernel_module, "CommManager") - .def(py::init<>()) - .def("register_target", &xcomm_manager::register_target); - } - - void bind_mock_objects(py::module& kernel_module) - { - py::class_(kernel_module, "MockKernel", py::dynamic_attr()) - .def(py::init<>()) - .def_property_readonly("_parent_header", &xmock_kernel::parent_header) - .def_readwrite("comm_manager", &xmock_kernel::m_comm_manager); - - py::class_(kernel_module, "MockIPython") - .def("register_post_execute", &xmock_ipython::register_post_execute) - .def("enable_gui", &xmock_ipython::enable_gui) - .def("observe", &xmock_ipython::observe) - .def("showtraceback", &xmock_ipython::showtraceback); - } - - struct ipython_instance - { - ipython_instance() : m_instance(py::none()) - { - } - - py::object get_instance(const py::module& kernel_module) const - { - if (m_instance.is(py::none())) - { - // The first import of IPython will throw if IPython has not been installed. - // In this case we fallback on the mock_ipython object. - try - { - py::module::import("IPython.core.interactiveshell").attr("InteractiveShellABC").attr("register")( - kernel_module.attr("XInteractiveShell")); - m_instance = kernel_module.attr("XInteractiveShell")(); - kernel_module.attr("has_ipython") = py::bool_(true); - } - catch(...) - { - m_instance = kernel_module.attr("MockIPython"); - kernel_module.attr("has_ipython") = py::bool_(false); - } - m_instance.attr("kernel") = kernel_module.attr("MockKernel")(); - } - return m_instance; - } - - private: - - mutable py::object m_instance; - }; - - py::module get_kernel_module_impl() - { - py::module kernel_module = create_module("kernel"); - - py::class_(kernel_module, "Hooks") - .def_static("show_in_pager", &hooks_object::show_in_pager); - - bind_history_manager(kernel_module); - bind_interactive_shell(kernel_module); - bind_comm(kernel_module); - bind_mock_objects(kernel_module); - - // To keep ipywidgets working, we must not import any module from IPython - // before the kernel module has been defined and IPython.core has been - // monkey patched. Otherwise, any call to register_target_comm will execute - // that of IPython instead of that of xeus. Thereafter any call to register_comm - // will execute that of xeus, where the target has not been registered, resulting - // in a segmentation fault. - // Initializing the xeus_python object as a memoized variable ensures the initialization - // of the interactive shell (which imports a lot of module from IPython) will - // occur AFTER IPython.core has been monkey_patched. - // Notice that using a static variable in the lambda to achieve the memoization - // results in a random crash at kernel shutdown. - // Also notice that using an attribute of kernel_module to memoize results - // in random segfault in the interpreter. - ipython_instance ipyinstance; - kernel_module.def("get_ipython", [ipyinstance, kernel_module]() { - return ipyinstance.get_instance(kernel_module); - }); - - return kernel_module; - } - - py::module get_kernel_module() - { - static py::module kernel_module = get_kernel_module_impl(); - return kernel_module; - } -} diff --git a/src/xtraceback.cpp b/src/xtraceback.cpp index a95a4bd0..896aa8a7 100644 --- a/src/xtraceback.cpp +++ b/src/xtraceback.cpp @@ -73,29 +73,20 @@ namespace xpyt get_filename_map()[filename] = execution_count; } - xerror extract_error(py::error_already_set& error) + xerror extract_error(const py::object& type, const py::object& value, const py::object& traceback) { xerror out; - // Fetch the error message, it must be released by the C++ exception first - error.restore(); - - py::object py_type; - py::object py_value; - py::object py_tb; - PyErr_Fetch(&py_type.ptr(), &py_value.ptr(), &py_tb.ptr()); - // This should NOT happen - if (py_type.is_none()) + if (type.is_none()) { out.m_ename = "Error"; - out.m_evalue = error.what(); - out.m_traceback.push_back(error.what()); + out.m_evalue = ""; } else { - out.m_ename = py::str(py_type.attr("__name__")); - out.m_evalue = py::str(py_value); + out.m_ename = py::str(type.attr("__name__")); + out.m_evalue = py::str(value); std::size_t first_frame_size(75); std::string delimiter(first_frame_size, '-'); @@ -107,9 +98,9 @@ namespace xpyt << red_text(out.m_ename) << first_frame_padding << traceback_msg; out.m_traceback.push_back(first_frame.str()); - if (py_tb.ptr() != nullptr && !py_tb.is_none()) + if (traceback.ptr() != nullptr && !traceback.is_none()) { - for (py::handle py_frame : py::module::import("traceback").attr("extract_tb")(py_tb)) + for (py::handle py_frame : py::module::import("traceback").attr("extract_tb")(traceback)) { std::string filename; std::string lineno; @@ -164,4 +155,54 @@ namespace xpyt return out; } + + xerror extract_error(py::error_already_set& error) + { + return extract_error(error.type(), error.value(), error.trace()); + } + + /******************** + * traceback module * + ********************/ + + py::module get_traceback_module_impl() + { + py::module traceback_module = create_module("traceback"); + + traceback_module.def("register_filename_mapping", + register_filename_mapping, + py::arg("filename"), + py::arg("execution_count") + ); + + exec(py::str(R"( +last_error = None + + +def reset_last_error(): + global last_error + + last_error = None + + +def set_last_error(type, value, traceback): + global last_error + + last_error = (type, value, traceback) + + +def get_last_error(): + global last_error + + return last_error + )"), traceback_module.attr("__dict__")); + + return traceback_module; + } + + py::module get_traceback_module() + { + static py::module traceback_module = get_traceback_module_impl(); + return traceback_module; + } } diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 00000000..5aacf83b --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = test +addopts = --nbval --current-env diff --git a/test/test_debugger.cpp b/test/test_debugger.cpp index afd887bf..6bd86d17 100644 --- a/test/test_debugger.cpp +++ b/test/test_debugger.cpp @@ -598,7 +598,7 @@ bool debugger_client::test_stack_trace() ++seq; nl::json stackframes = m_client.receive_on_control(); - bool res = stackframes["content"]["body"]["stackFrames"].size() == 1; + bool res = stackframes["content"]["body"]["stackFrames"].size() != 0; continue_exec(seq); m_client.wait_for_debug_event("stopped"); continue_exec(seq); @@ -1083,4 +1083,3 @@ TEST(debugger, variables) notify_done(); } } - diff --git a/test/test_xeus_python.ipynb b/test/test_xeus_python.ipynb new file mode 100644 index 00000000..b6a859c7 --- /dev/null +++ b/test/test_xeus_python.ipynb @@ -0,0 +1,100 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "south-reserve", + "metadata": {}, + "source": [ + "# This is a test run by nbval\n", + "\n", + "Please keep the cell outputs. You can regenerate the outputs by running the Notebook again and saving it." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "australian-penetration", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = 3\n", + "a" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "blond-assessment", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello World!\n" + ] + } + ], + "source": [ + "print(\"Hello World!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "interesting-negotiation", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f73612297109452a813fe22993a0a87d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib widget\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "fig = plt.figure()\n", + "plt.plot(np.sin(np.linspace(0, 20, 100)));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9 (XPython)", + "language": "python", + "name": "xpython" + }, + "language_info": { + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/test_xeus_python_kernel.py b/test/test_xeus_python_kernel.py deleted file mode 100644 index 2648cefa..00000000 --- a/test/test_xeus_python_kernel.py +++ /dev/null @@ -1,136 +0,0 @@ -############################################################################# -# Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and # -# Wolf Vollprecht # -# Copyright (c) 2018, QuantStack # -# # -# Distributed under the terms of the BSD 3-Clause License. # -# # -# The full license is in the file LICENSE, distributed with this software. # -############################################################################# - -import tempfile -import unittest -import jupyter_kernel_test - - -class XeusPythonTests(jupyter_kernel_test.KernelTests): - - kernel_name = "xpython" - language_name = "python" - - code_hello_world = "print('hello, world')" - - code_page_something = "?print" - - completion_samples = [ - {'text': 'pri', 'matches': {'print'}}, - {'text': 'from sys imp', 'matches': {'import'}}, - {'text': 'se', 'matches': {'set', 'setattr'}}, - ] - - complete_code_samples = ['1', "print('hello, world')", "def f(x):\n return x*2\n\n\n"] - incomplete_code_samples = ["print('''hello", "def f(x):\n x*2"] - invalid_code_samples = ['import = 7q'] - - code_inspect_sample = "open" - - def test_xeus_python_line_magic(self): - reply, output_msgs = self.execute_helper(code="%pwd") - self.assertEqual(output_msgs[0]['msg_type'], 'execute_result') - self.assertEqual(reply['content']['status'], 'ok') - - # line magic expressions - reply, output_msgs = self.execute_helper(code="a = %pwd\nassert a") - self.assertEqual(reply['content']['status'], 'ok') - self.assertFalse(output_msgs) - - def test_xeus_python_cell_magic(self): - """Test calling cell magic""" - with tempfile.NamedTemporaryFile(delete=False) as tmpfile: - temp_filename = tmpfile.name - reply, output_msgs = self.execute_helper(code="%%writefile {}\ntest file".format(temp_filename)) - # we don't care about the actual function of the magic - # only check if execution succeeded - self.assertEqual(reply['content']['status'], 'ok') - - def test_xeus_python_shell_magic(self): - """Test calling shell escape (! or !!) magic.""" - - # the shell command does not exist, but the magic execution should succeed - reply, output_msgs = self.execute_helper(code="!does_not_exist -params") - self.assertEqual(reply['content']['status'], 'ok') - - reply, output_msgs = self.execute_helper(code="!!does_not_exist -params") - self.assertEqual(reply['content']['status'], 'ok') - - def test_xeus_python_user_defined_magics(self): - """Test user-defined magic.""" - - magic_def_code = """ - from IPython.core.magic import register_line_magic, register_cell_magic - @register_line_magic - def lmagic(line): - return line - @register_cell_magic - def cmagic(line, body): - return body""" - - reply, output_msgs = self.execute_helper(code=magic_def_code) - self.assertEqual(reply['content']['status'], 'ok') - - reply, output_msgs = self.execute_helper(code="%lmagic hello") - self.assertEqual(reply['content']['status'], 'ok') - self.assertTrue('hello' in output_msgs[0]['content']['data']['text/plain']) - - # cant call cell magic as line magic - reply, output_msgs = self.execute_helper(code="%cmagic hello") - self.assertEqual(reply['content']['status'], 'error') - - reply, output_msgs = self.execute_helper(code="%%cmagic\nworld") - self.assertEqual(reply['content']['status'], 'ok') - self.assertTrue('world' in output_msgs[0]['content']['data']['text/plain']) - - # cant call line magic as as cell magic - reply, output_msgs = self.execute_helper(code="%%lmagic hello") - self.assertEqual(reply['content']['status'], 'error') - - def test_xeus_python_missing_magic(self): - reply, output_msgs = self.execute_helper(code="%missing_magic") - self.assertRegex(output_msgs[0]['content']['evalue'], "magics not found") - - def test_xeus_python_load_ext_magic(self): - """Test loading extensions""" - reply, output_msgs = self.execute_helper(code="%load_ext example_magic") - reply, output_msgs = self.execute_helper(code="%abra line") - self.assertEqual(reply['content']['status'], 'ok') - - def test_xeus_python_history_manager(self): - reply, output_msgs = self.execute_helper(code="assert get_ipython().history_manager is not None") - self.assertEqual(reply['content']['status'], 'ok') - - def test_xeus_python_stdout(self): - reply, output_msgs = self.execute_helper(code='print(3)') - self.assertEqual(output_msgs[0]['msg_type'], 'stream') - self.assertEqual(output_msgs[0]['content']['name'], 'stdout') - self.assertEqual(output_msgs[0]['content']['text'], '3') - - def test_xeus_python_stderr(self): - reply, output_msgs = self.execute_helper(code='a = []; a.push_back(3)') - self.assertEqual(output_msgs[0]['msg_type'], 'error') - self.assertEqual(output_msgs[0]['content']['ename'], 'AttributeError') - self.assertEqual(output_msgs[0]['content']['evalue'], "'list' object has no attribute 'push_back'") - traceback = output_msgs[0]['content']['traceback'] - self.assertEqual( - "\033[0;31m---------------------------------------------------------------------------\033[0m\n\033[0;31mAttributeError\033[0m Traceback (most recent call last)", - traceback[0] - ) - self.assertTrue( - "a = []; a.push_back" in traceback[1] - ) - self.assertEqual( - "\033[0;31mAttributeError\033[0m: 'list' object has no attribute 'push_back'\n\033[0;31m---------------------------------------------------------------------------\033[0m", - traceback[2] - ) - -if __name__ == '__main__': - unittest.main() diff --git a/test/xeus_client.cpp b/test/xeus_client.cpp index 0ae2d45f..f8f75bf0 100644 --- a/test/xeus_client.cpp +++ b/test/xeus_client.cpp @@ -308,4 +308,3 @@ void xeus_logger_client::log_message(nl::json msg) std::ofstream out(m_file_name, std::ios_base::app); out << msg.dump(4) << std::endl; } -