Draft: Use IPython

CMakeLists.txt

@@ -72,7 +72,7 @@ OPTION(XPYT_DOWNLOAD_GTEST "build gtest from downloaded sources" OFF)
 set(xtl_REQUIRED_VERSION 0.6.23)
 set(xeus_REQUIRED_VERSION 0.25.0)
 set(pybind11_REQUIRED_VERSION 2.6.0)
-set(pybind11_json_REQUIRED_VERSION 0.2.2)
+set(pybind11_json_REQUIRED_VERSION 0.2.8)
 
 if (NOT TARGET xtl)
     find_package(xtl ${xtl_REQUIRED_VERSION} REQUIRED)
@@ -120,29 +120,21 @@ endif ()
 # ============
 
 set(XEUS_PYTHON_SRC
+    src/xcomm.cpp
+    src/xcomm.hpp
+    src/xcompiler.cpp
+    src/xcompiler.hpp
     src/xdebugger.cpp
     src/xdebugpy_client.hpp
     src/xdebugpy_client.cpp
     src/xdisplay.cpp
     src/xdisplay.hpp
     src/xinput.cpp
     src/xinput.hpp
-    src/xinspect.cpp
-    src/xinspect.hpp
-    src/xinteractiveshell.cpp
-    src/xinteractiveshell.hpp
     src/xinternal_utils.cpp
     src/xinternal_utils.hpp
     src/xinterpreter.cpp
-    src/xis_complete.cpp
-    src/xis_complete.hpp
-    src/xlinecache.cpp
-    src/xlinecache.hpp
-    src/xnullcontext.cpp
-    src/xnullcontext.hpp
     src/xpaths.cpp
-    src/xpython_kernel.cpp
-    src/xpython_kernel.hpp
     src/xstream.cpp
     src/xstream.hpp
     src/xtraceback.cpp

README.md

@@ -134,7 +134,7 @@ xeus-python does not cover 100% of the features of ipykernel. For example, only
 
 | `xeus-python`|   `xeus`         |      `xtl`      | `cppzmq` | `nlohmann_json` | `pybind11`     | `pybind11_json`   | `jedi`            | `pygments`        | `ptvsd` | `debugpy` |
 |--------------|------------------|-----------------|----------|-----------------|----------------|-------------------|-------------------|-------------------|---------|-----------|
-|  master      |  >=1.0.0,<0.26   |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<4.0    | >=2.6.0,<3.0   | >=0.2.6,<0.3      | >=0.15.1,<0.19    | >=2.3.1,<3.0.0    |         | >=1.1.0   |
+|  master      |  >=1.0.0,<0.26   |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<4.0    | >=2.6.0,<3.0   | >=0.2.8,<0.3      | >=0.15.1,<0.19    | >=2.3.1,<3.0.0    |         | >=1.1.0   |
 |  0.10.2      |  >=1.0.0,<0.26   |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<4.0    | >=2.6.0,<3.0   | >=0.2.6,<0.3      | >=0.15.1,<0.19    | >=2.3.1,<3.0.0    |         | >=1.1.0   |
 |  0.10.1      |  >=1.0.0,<0.26   |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<4.0    | >=2.6.0,<3.0   | >=0.2.6,<0.3      | >=0.18.0,<0.19    | >=2.3.1,<3.0.0    |         | >=1.1.0   |
 |  0.10.0      |  >=1.0.0,<0.26   |  >=0.7.0,<0.8   | ~4.4.1   | >=3.6.1,<4.0    | >=2.6.0,<3.0   | >=0.2.6,<0.3      | >=0.18.0,<0.19    | >=2.3.1,<3.0.0    |         | >=1.1.0   |

include/xeus-python/xinterpreter.hpp

@@ -74,9 +74,9 @@ namespace xpyt
 
         void redirect_output();
         void redirect_display(bool install_hook=true);
-        void load_extensions();
 
         py::object m_displayhook;
+        py::object m_ipython_shell;
 
         // The interpreter has the same scope as a `gil_scoped_release` instance
         // so that the GIL is not held by default, it will only be held when the

src/xcomm.cpp

@@ -0,0 +1,203 @@
+/***************************************************************************
+* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and     *
+* Wolf Vollprecht                                                          *
+* Copyright (c) 2018, QuantStack                                           *
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#include <string>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+
+#include "xeus/xcomm.hpp"
+#include "xeus/xinterpreter.hpp"
+
+#include "pybind11_json/pybind11_json.hpp"
+
+#include "pybind11/pybind11.h"
+#include "pybind11/functional.h"
+#include "pybind11/eval.h"
+
+#include "xeus-python/xutils.hpp"
+
+#include "xcomm.hpp"
+#include "xinternal_utils.hpp"
+
+namespace py = pybind11;
+namespace nl = nlohmann;
+
+namespace xpyt
+{
+    /*********************
+     * xcomm declaration *
+     ********************/
+
+    class xcomm
+    {
+    public:
+
+        using python_callback_type = std::function<void(py::object)>;
+        using cpp_callback_type = std::function<void(const xeus::xmessage&)>;
+        using zmq_buffers_type = std::vector<zmq::message_t>;
+
+        xcomm(const py::args& args, const py::kwargs& kwargs);
+        xcomm(xeus::xcomm&& comm);
+        xcomm(xcomm&& comm) = default;
+        virtual ~xcomm();
+
+        std::string comm_id() const;
+        bool kernel() const;
+
+        void close(const py::args& args, const py::kwargs& kwargs);
+        void send(const py::args& args, const py::kwargs& kwargs);
+        void on_msg(const python_callback_type& callback);
+        void on_close(const python_callback_type& callback);
+
+    private:
+
+        xeus::xtarget* target(const py::kwargs& kwargs) const;
+        xeus::xguid id(const py::kwargs& kwargs) const;
+        cpp_callback_type cpp_callback(const python_callback_type& callback) const;
+
+        xeus::xcomm m_comm;
+    };
+
+    struct xcomm_manager
+    {
+        xcomm_manager() = default;
+
+        void register_target(const py::str& target_name, const py::object& callback);
+    };
+
+    /************************
+     * xcomm implementation *
+     ************************/
+
+    xcomm::xcomm(const py::args& /*args*/, const py::kwargs& kwargs)
+        : m_comm(target(kwargs), id(kwargs))
+    {
+        m_comm.open(
+            kwargs.attr("get")("metadata", py::dict()),
+            kwargs.attr("get")("data", py::dict()),
+            pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list()))
+        );
+    }
+
+    xcomm::xcomm(xeus::xcomm&& comm)
+        : m_comm(std::move(comm))
+    {
+    }
+
+    xcomm::~xcomm()
+    {
+    }
+
+    std::string xcomm::comm_id() const
+    {
+        return m_comm.id();
+    }
+
+    bool xcomm::kernel() const
+    {
+        return true;
+    }
+
+    void xcomm::close(const py::args& /*args*/, const py::kwargs& kwargs)
+    {
+        m_comm.close(
+            kwargs.attr("get")("metadata", py::dict()),
+            kwargs.attr("get")("data", py::dict()),
+            pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list()))
+        );
+    }
+
+    void xcomm::send(const py::args& /*args*/, const py::kwargs& kwargs)
+    {
+        m_comm.send(
+            kwargs.attr("get")("metadata", py::dict()),
+            kwargs.attr("get")("data", py::dict()),
+            pylist_to_zmq_buffers(kwargs.attr("get")("buffers", py::list()))
+        );
+    }
+
+    void xcomm::on_msg(const python_callback_type& callback)
+    {
+        m_comm.on_message(cpp_callback(callback));
+    }
+
+    void xcomm::on_close(const python_callback_type& callback)
+    {
+        m_comm.on_close(cpp_callback(callback));
+    }
+
+    xeus::xtarget* xcomm::target(const py::kwargs& kwargs) const
+    {
+        std::string target_name = kwargs["target_name"].cast<std::string>();
+        return xeus::get_interpreter().comm_manager().target(target_name);
+    }
+
+    xeus::xguid xcomm::id(const py::kwargs& kwargs) const
+    {
+        if (py::hasattr(kwargs, "comm_id"))
+        {
+            // TODO: prevent copy
+            return xeus::xguid(kwargs["comm_id"].cast<std::string>());
+        }
+        else
+        {
+            return xeus::new_xguid();
+        }
+    }
+
+    auto xcomm::cpp_callback(const python_callback_type& py_callback) const -> cpp_callback_type
+    {
+        return [this, py_callback](const xeus::xmessage& msg) {
+            XPYT_HOLDING_GIL(py_callback(cppmessage_to_pymessage(msg)))
+        };
+    }
+
+    void xcomm_manager::register_target(const py::str& target_name, const py::object& callback)
+    {
+        auto target_callback = [&callback] (xeus::xcomm&& comm, const xeus::xmessage& msg) {
+            XPYT_HOLDING_GIL(callback(xcomm(std::move(comm)), cppmessage_to_pymessage(msg)));
+        };
+
+        xeus::get_interpreter().comm_manager().register_comm_target(
+            static_cast<std::string>(target_name), target_callback
+        );
+    }
+
+    /***************
+     * comm module *
+     ***************/
+
+    py::module get_comm_module_impl()
+    {
+        py::module comm_module = create_module("comm");
+
+        py::class_<xcomm>(comm_module, "Comm")
+            .def(py::init<py::args, py::kwargs>())
+            .def("close", &xcomm::close)
+            .def("send", &xcomm::send)
+            .def("on_msg", &xcomm::on_msg)
+            .def("on_close", &xcomm::on_close)
+            .def_property_readonly("comm_id", &xcomm::comm_id)
+            .def_property_readonly("kernel", &xcomm::kernel);
+
+        py::class_<xcomm_manager>(comm_module, "CommManager")
+            .def(py::init<>())
+            .def("register_target", &xcomm_manager::register_target);
+
+        return comm_module;
+    }
+
+    py::module get_comm_module()
+    {
+        static py::module comm_module = get_comm_module_impl();
+        return comm_module;
+    }
+}

src/xis_complete.hpp → src/xcomm.hpp

@@ -8,16 +8,16 @@
 * The full license is in the file LICENSE, distributed with this software. *
 ****************************************************************************/
 
-#ifndef XPYT_COMPLETION_HPP
-#define XPYT_COMPLETION_HPP
+#ifndef XPYT_COMM_HPP
+#define XPYT_COMM_HPP
 
 #include "pybind11/pybind11.h"
 
 namespace py = pybind11;
 
 namespace xpyt
 {
-    py::module get_completion_module();
+    py::module get_comm_module();
 }
 
 #endif

src/xcompiler.cpp

@@ -0,0 +1,69 @@
+/***************************************************************************
+* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and     *
+* Wolf Vollprecht                                                          *
+* Copyright (c) 2018, QuantStack                                           *
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#include <iostream>
+#include <string>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+
+#include "xeus/xcomm.hpp"
+#include "xeus/xinterpreter.hpp"
+
+#include "pybind11_json/pybind11_json.hpp"
+
+#include "pybind11/pybind11.h"
+#include "pybind11/functional.h"
+#include "pybind11/eval.h"
+
+#include "xeus-python/xutils.hpp"
+#include "xeus-python/xtraceback.hpp"
+
+#include "xcompiler.hpp"
+#include "xinternal_utils.hpp"
+
+namespace py = pybind11;
+namespace nl = nlohmann;
+
+namespace xpyt
+{
+
+    py::str get_filename(const py::str& raw_code)
+    {
+        return get_cell_tmp_file(raw_code);
+    }
+
+    /*******************
+     * compiler module *
+     *******************/
+
+    py::module get_compiler_module_impl()
+    {
+        py::module compiler_module = create_module("compiler");
+
+        compiler_module.def("get_filename", get_filename);
+
+        exec(py::str(R"(
+from IPython.core.compilerop import CachingCompiler
+
+class XCachingCompiler(CachingCompiler):
+    def get_code_name(self, raw_code, code, number):
+        return get_filename(raw_code)
+        )"), compiler_module.attr("__dict__"));
+
+        return compiler_module;
+    }
+
+    py::module get_compiler_module()
+    {
+        static py::module compiler_module = get_compiler_module_impl();
+        return compiler_module;
+    }
+}

src/xpython_kernel.hpp → src/xcompiler.hpp

@@ -8,17 +8,16 @@
 * The full license is in the file LICENSE, distributed with this software. *
 ****************************************************************************/
 
-#ifndef XPYT_KERNEL_HPP
-#define XPYT_KERNEL_HPP
+#ifndef XPYT_COMPILER_HPP
+#define XPYT_COMPILER_HPP
 
 #include "pybind11/pybind11.h"
-#include "xeus/xhistory_manager.hpp"
 
 namespace py = pybind11;
 
 namespace xpyt
 {
-    py::module get_kernel_module();
+    py::module get_compiler_module();
 }
 
 #endif