Merge pull request #46 from quantopian/ipython6-compat

Jupyter notebook 5 compatibility

.travis.yml

@@ -1,11 +1,15 @@
 language: python
 sudo: false
+
 python:
   - "2.7"
   - "3.4"
+  - "3.6"
 env:
-  - IPYTHON=3
-  - IPYTHON=4
+  - NB_VERSION=4
+  - NB_VERSION=5
+  - FLAKE8=true NB_VERSION=5
+  - NOTEST=true NB_VERSION=5
 
 addons:
   postgresql: "9.3"
@@ -14,8 +18,11 @@ install:
   - pip install tox
 
 script:
-  - if [[ $TRAVIS_PYTHON_VERSION = '2.7' ]]; then tox -e py27-ipython$IPYTHON,flake8; fi
-  - if [[ $TRAVIS_PYTHON_VERSION = '3.4' ]]; then tox -e py34-ipython$IPYTHON,flake8; fi
+  - if [[ $TRAVIS_PYTHON_VERSION = '2.7' ]]; then tox -e py27-notebook$NB_VERSION; fi
+  - if [[ $TRAVIS_PYTHON_VERSION = '3.4' ]]; then tox -e py34-notebook$NB_VERSION; fi
+  - if [[ $TRAVIS_PYTHON_VERSION = '3.6' ]]; then tox -e py36-notebook$NB_VERSION; fi
+  - if [[ $FLAKE8 = 'true' ]]; then tox -e notest; fi
+  - if [[ $NOTEST = 'true' ]]; then tox -e flake8; fi
 
 branches:
   only:

README.rst

@@ -3,18 +3,23 @@ PGContents
 
 PGContents is a PostgreSQL-backed implementation of `IPEP 27 <https://github.com/ipython/ipython/wiki/IPEP-27:-Contents-Service>`_.  It aims to a be a transparent, drop-in replacement for IPython's standard filesystem-backed storage system.  PGContents' `PostgresContentsManager` class can be used to replace all local filesystem storage with database-backed storage, while its `PostgresCheckpoints` class can be used to replace just IPython's checkpoint storage.  These features are useful when running IPython in environments where you either don't have access to—or don't trust the reliability of—the local filesystem of your notebook server.
 
-This repository is under development as part of the `Quantopian Research Environment <https://www.quantopian.com/research>`_, currently in Open Beta.
+This repository developed as part of the `Quantopian Research Environment <https://www.quantopian.com/research>`_.
 
 Getting Started
 ---------------
 **Prerequisites:**
  - Write access to an empty `PostgreSQL <http://www.postgresql.org>`_ database.
- - A Python installation with `IPython <https://github.com/ipython/ipython>`_ 3.x 
-   or `Jupyter Notebook <https://github.com/jupyter/notebook>`_ >= 4.0.
+ - A Python installation with `Jupyter Notebook <https://github.com/jupyter/notebook>`_ >= 4.0.
 
 **Installation:**
 
-0. Install `pgcontents` from PyPI via `pip install pgcontents[ipy4]`. (This will install pgcontents in a manner compatible with a recent Jupyter Notebook version.  To install pgcontents with support for the legacy IPython 3.x series, run `pip install pgcontents[ipy3]`).
-1. Run `pgcontents init` to configure your database.  You will be prompted for a database URL for pgcontents to use for storage.  (Alternatively, you can set the PGCONTENTS_DB_URL environment variable, or pass `--db-url` on the command line).
-2. Configure IPython/Jupyter to use pgcontents as its storage backend.  This can be done from the command line or by modifying your notebook config file.  For IPython 3.x on a Unix-like system, your notebok config will be located located at ``~/.ipython/profile_default/ipython_notebook_config.py``.  For Jupyter Notebook, it will will be located at ``~/.jupyter/jupyter_notebook_config.py``. See the ``examples`` directory for example configuration files.
-3. Enjoy your filesystem-free IPython experience!
+0. Install ``pgcontents`` from PyPI via ``pip install pgcontents``.
+1. Run ``pgcontents init`` to configure your database.  You will be prompted for a database URL for pgcontents to use for storage.  (Alternatively, you can set the ``PGCONTENTS_DB_URL`` environment variable, or pass ``--db-url`` on the command line).
+2. Configure Jupyter to use pgcontents as its storage backend.  This can be done from the command line or by modifying your notebook config file. On a Unix-like system, your notebook config will be located at ``~/.jupyter/jupyter_notebook_config.py``. See the ``examples`` directory for example configuration files.
+3. Enjoy your filesystem-free Jupyter experience!
+
+Demo Video
+----------
+You can see a demo of PGContents in action in `this presentation from JupyterCon 2017`_.
+
+.. _`this presentation from JupyterCon 2017` : https://youtu.be/TtsbspKHJGo?t=917

notebook4_constraints.txt

@@ -0,0 +1,2 @@
+notebook<5
+tornado<5

notebook5_constraints.txt

@@ -0,0 +1 @@
+notebook<6

pgcontents/pgmanager.py

@@ -57,6 +57,7 @@
     save_file,
 )
 from .utils.ipycompat import Bool, ContentsManager, from_dict
+from traitlets import default
 
 
 class PostgresContentsManager(PostgresManagerMixin, ContentsManager):
@@ -69,11 +70,18 @@ class PostgresContentsManager(PostgresManagerMixin, ContentsManager):
         help="Create a root directory automatically?",
     )
 
-    def _checkpoints_class_default(self):
+    @default('checkpoints_class')
+    def _default_checkpoints_class(self):
         return PostgresCheckpoints
 
-    def _checkpoints_kwargs_default(self):
-        kw = super(PostgresContentsManager, self)._checkpoints_kwargs_default()
+    @default('checkpoints_kwargs')
+    def _default_checkpoints_kwargs(self):
+        klass = PostgresContentsManager
+        try:
+            kw = super(klass, self)._checkpoints_kwargs_default()
+        except AttributeError:
+            kw = super(klass, self)._default_checkpoints_kwargs()
+
         kw.update({
             'create_user_on_startup': self.create_user_on_startup,
             'crypto': self.crypto,
@@ -83,7 +91,8 @@ def _checkpoints_kwargs_default(self):
         })
         return kw
 
-    def _create_directory_on_startup_default(self):
+    @default('create_directory_on_startup')
+    def _default_create_directory_on_startup(self):
         return self.create_user_on_startup
 
     def __init__(self, *args, **kwargs):

pgcontents/tests/test_pgcontents_api.py

@@ -51,7 +51,7 @@
 )
 from ..utils.ipycompat import (
     APITest, Config, FileContentsManager, GenericFileCheckpoints, to_os_path,
-)
+    assert_http_error)
 from ..utils.sync import walk, walk_dirs
 
 
@@ -163,6 +163,45 @@ def test_list_checkpoints_sorting(self):
             )
         )
 
+    # ContentsManager has different behaviour in notebook 5.5+
+    # https://github.com/jupyter/notebook/pull/3108...it now allows
+    # non-empty directories to be deleted.
+    #
+    # PostgresContentsManager should continue to work the old way and
+    # prevent non-empty directories from being deleted, since it doesn't
+    # support backing up the deleted directory in the OS trash can.
+    # FileContentsManager should allow non-empty directories to be deleted.
+    def test_delete_non_empty_dir(self):
+        if isinstance(self.notebook.contents_manager,
+                      PostgresContentsManager):
+            # make sure non-empty directories cannot be deleted with
+            # PostgresContentsManager
+            _test_delete_non_empty_dir_fail(self, u'å b')
+        elif isinstance(self.notebook.contents_manager,
+                        HybridContentsManager):
+            # check that one of the non-empty subdirectories owned by the
+            # PostgresContentsManager cannnot be deleted
+            _test_delete_non_empty_dir_fail(self, 'Directory with spaces in')
+        else:
+            # for all other contents managers that we test (in this case it
+            # will just be FileContentsManager) use the super class
+            # implementation of this test (i.e. make sure non-empty dirs can
+            # be deleted)
+            super(_APITestBase, self).test_delete_non_empty_dir()
+
+
+def _test_delete_non_empty_dir_fail(self, path):
+    with assert_http_error(400):
+        self.api.delete(path)
+
+
+def _test_delete_non_empty_dir_pass(self, path):
+    # Test that non empty directory can be deleted
+    self.api.delete(path)
+    # Check if directory has actually been deleted
+    with assert_http_error(404):
+        self.api.list(path)
+
 
 def postgres_contents_config():
     """
@@ -441,12 +480,12 @@ def _method(self, api_path, *args):
         'isfile',
         'isdir',
     ]
-    l = locals()
+    locs = locals()
     for method_name in __methods_to_multiplex:
-        l[method_name] = __api_path_dispatch(method_name)
+        locs[method_name] = __api_path_dispatch(method_name)
     del __methods_to_multiplex
     del __api_path_dispatch
-    del l
+    del locs
 
     # Override to not delete the root of the file subsystem.
     def test_delete_dirs(self):

pgcontents/utils/ipycompat.py

@@ -1,93 +1,51 @@
 """
-Utilities for managing IPython 3/4 compat.
+Utilities for managing compat between notebook versions.
 """
-import IPython
+import notebook
+if notebook.version_info[0] >= 6:  # noqa
+    raise ImportError("Jupyter Notebook versions 6 and up are not supported.")
 
-SUPPORTED_VERSIONS = {3, 4, 5}
-IPY_MAJOR = IPython.version_info[0]
-if IPY_MAJOR not in SUPPORTED_VERSIONS:
-    raise ImportError("IPython version %d is not supported." % IPY_MAJOR)
+from traitlets.config import Config
+from notebook.services.contents.checkpoints import (
+    Checkpoints,
+    GenericCheckpointsMixin,
+)
+from notebook.services.contents.filemanager import FileContentsManager
+from notebook.services.contents.filecheckpoints import (
+    GenericFileCheckpoints
+)
+from notebook.services.contents.manager import ContentsManager
+from notebook.services.contents.tests.test_manager import (
+    TestContentsManager
+)
+from notebook.services.contents.tests.test_contents_api import (
+    APITest
+)
+from notebook.tests.launchnotebook import assert_http_error
+from notebook.utils import to_os_path
+from nbformat import from_dict, reads, writes
+from nbformat.v4.nbbase import (
+    new_code_cell,
+    new_markdown_cell,
+    new_notebook,
+    new_raw_cell,
+)
+from nbformat.v4.rwbase import strip_transient
+from traitlets import (
+    Any,
+    Bool,
+    Dict,
+    Instance,
+    Integer,
+    HasTraits,
+    Unicode,
+)
 
-IPY3 = (IPY_MAJOR == 3)
-
-if IPY3:
-    from IPython.config import Config
-    from IPython.html.services.contents.manager import ContentsManager
-    from IPython.html.services.contents.checkpoints import (
-        Checkpoints,
-        GenericCheckpointsMixin,
-    )
-    from IPython.html.services.contents.filemanager import FileContentsManager
-    from IPython.html.services.contents.filecheckpoints import (
-        GenericFileCheckpoints
-    )
-    from IPython.html.services.contents.tests.test_manager import (
-        TestContentsManager
-    )
-    from IPython.html.services.contents.tests.test_contents_api import (
-        APITest
-    )
-    from IPython.html.utils import to_os_path
-    from IPython.nbformat import from_dict, reads, writes
-    from IPython.nbformat.v4.nbbase import (
-        new_code_cell,
-        new_markdown_cell,
-        new_notebook,
-        new_raw_cell,
-    )
-    from IPython.nbformat.v4.rwbase import strip_transient
-    from IPython.utils.traitlets import (
-        Any,
-        Bool,
-        Dict,
-        Instance,
-        Integer,
-        HasTraits,
-        Unicode,
-    )
-else:
-    import notebook
-    if notebook.version_info[0] >= 5:
-        raise ImportError("Notebook versions 5 and up are not supported.")
-
-    from traitlets.config import Config
-    from notebook.services.contents.checkpoints import (
-        Checkpoints,
-        GenericCheckpointsMixin,
-    )
-    from notebook.services.contents.filemanager import FileContentsManager
-    from notebook.services.contents.filecheckpoints import (
-        GenericFileCheckpoints
-    )
-    from notebook.services.contents.manager import ContentsManager
-    from notebook.services.contents.tests.test_manager import (
-        TestContentsManager
-    )
-    from notebook.services.contents.tests.test_contents_api import (
-        APITest
-    )
-    from notebook.utils import to_os_path
-    from nbformat import from_dict, reads, writes
-    from nbformat.v4.nbbase import (
-        new_code_cell,
-        new_markdown_cell,
-        new_notebook,
-        new_raw_cell,
-    )
-    from nbformat.v4.rwbase import strip_transient
-    from traitlets import (
-        Any,
-        Bool,
-        Dict,
-        Instance,
-        Integer,
-        HasTraits,
-        Unicode,
-    )
 
 __all__ = [
     'APITest',
     'Any',
+    'assert_http_error',
     'Bool',
     'Checkpoints',
     'Config',

requirements.txt

@@ -5,4 +5,4 @@ cryptography>=1.4
 psycopg2>=2.6.1
 requests>=2.7.0
 six>=1.9.0
-tox>=2.3
+notebook[test]>=4.0

setup.py

@@ -47,8 +47,6 @@ def main():
         install_requires=reqs,
         extras_require={
             'test': test_reqs,
-            'ipy3': ['ipython[test,notebook]<4.0'],
-            'ipy4': ['ipython<6.0', 'notebook[test]>=4.0,<5.0'],
         },
         scripts=[
             'bin/pgcontents',

tox.ini

@@ -1,19 +1,23 @@
 [tox]
-envlist=py{27,34}-ipython{3,4},flake8
+envlist=py{27,34}-notebook4,py{27,34,36}-notebook5,flake8,notest
 skip_missing_interpreters=True
 
 [testenv]
 whitelist_externals =
     createdb
 
+install_command =
+    py{27,34,36}-notebook4: pip install -c notebook4_constraints.txt {opts} {packages}
+    py{27,34,36}-notebook5: pip install -c notebook5_constraints.txt {opts} {packages}
+    flake8,notest: pip install {opts} {packages}
+
 deps =
-    py{27,34}-ipython3: .[test,ipy3]
-    py{27,34}-ipython4: .[test,ipy4]
+    py{27,34,36}-notebook{4,5}: .[test]
     flake8: flake8
-    notest: .[ipy4]
+    notest: .
 
 commands =
-    py{27,34}-ipython{3,4}: -createdb pgcontents_testing
-    py{27,34}-ipython{3,4}: nosetests pgcontents/tests
+    py{27,34,36}-notebook{4,5}: -createdb pgcontents_testing
+    py{27,34,36}-notebook{4,5}: nosetests pgcontents/tests
     flake8: flake8 pgcontents
     notest: python -c 'from pgcontents.utils.ipycompat import *'