Repo2docker dask fixes

base-notebook/binder/dask_config.yaml

@@ -13,26 +13,33 @@ distributed:
       limit: 5s
 
 kubernetes:
+  name: dask-{JUPYTERHUB_USER}-{uuid}
   worker-template:
-    metadata:
     spec:
       restartPolicy: Never
       containers:
-      - args:
-          - dask-worker
-          - --nthreads
-          - '2'
-          - --no-bokeh
-          - --memory-limit
-          - 7GB
-          - --death-timeout
-          - '60'
-        image: ${JUPYTER_IMAGE_SPEC}
-        name: dask-worker
-        resources:
-          limits:
-            cpu: "1.75"
-            memory: 7G
-          requests:
-            cpu: 1
-            memory: 7G
+        - name: dask-${JUPYTERHUB_USER}
+          image: ${JUPYTER_IMAGE_SPEC}
+          args:
+            - dask-worker
+            - --nthreads
+            - '2'
+            - --no-bokeh
+            - --memory-limit
+            - 7GB
+            - --death-timeout
+            - '60'
+          resources:
+            limits:
+              cpu: "1.75"
+              memory: 7G
+            requests:
+              cpu: 1
+              memory: 7G
+
+labextension:
+  factory:
+    module: dask_kubernetes
+    class: KubeCluster
+    args: []
+    kwargs: {}

base-notebook/binder/environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - dask-kubernetes>=0.8.0
   - nbserverproxy
   - ipywidgets
+  - graphviz
   - pip
   - pip:
-    - dask_labextension==0.3.3
-    - graphviz
+    - dask_labextension==0.3.1

base-notebook/binder/tests/test_dask_env.py

@@ -10,3 +10,11 @@ def client():
 def test_check_dask_version(client):
     print(client)
     versions = client.get_versions(check=True)
+
+
+def test_dask_config():
+    import dask
+
+    assert '/srv/conda/envs/notebook/etc/dask' in dask.config.paths
+    assert dask.config.config['labextension']['factory']['class'] == 'KubeCluster'
+    assert 'worker-template' in dask.config.config['kubernetes']

base-notebook/binder/tests/test_imports.py

@@ -1,5 +1,6 @@
 import importlib
-
+import sys
+from distutils.version import LooseVersion
 import pytest
 
 packages = [
@@ -10,3 +11,17 @@
 @pytest.mark.parametrize('package_name', packages, ids=packages)
 def test_import(package_name):
     importlib.import_module(package_name)
+
+# for current repo2docker config
+def test_conda_environment():
+    assert sys.prefix == '/srv/conda/envs/notebook'
+
+# would be better to automatically get these from environment.yml
+def test_pinned_versions():
+    import tornado
+    import dask_kubernetes
+    import dask_labextension
+
+    assert LooseVersion(tornado.version) == LooseVersion('5.1.1')
+    assert LooseVersion(dask_kubernetes.__version__) >= LooseVersion('0.8')
+    assert LooseVersion(dask_labextension.__version__) == LooseVersion('0.3.1')

onbuild/r2d_overlay.py

@@ -32,12 +32,13 @@ def binder_path(path):
 
 @become(NB_UID)
 def apply_environment():
+    NB_PYTHON_PREFIX = os.environ['NB_PYTHON_PREFIX']
     env_path = binder_path('environment.yml')
     if os.path.exists(env_path):
         return [
-            f'conda env update -n root -f {env_path}',
-            f'conda clean -tipsy',
-            f'conda list -n root',
+            f'conda env update -p {NB_PYTHON_PREFIX} -f {env_path}',
+            f'conda clean --all -f -y',
+            f'conda list -p {NB_PYTHON_PREFIX}',
             f'rm -rf /srv/conda/pkgs'
         ]
 
@@ -46,10 +47,10 @@ def apply_environment():
 def apply_requirements():
     req_path = binder_path('requirements.txt')
     env_path = binder_path('environment.yml')
-
+    NB_PYTHON_PREFIX = os.environ['NB_PYTHON_PREFIX']
     if os.path.exists(req_path) and not os.path.exists(env_path):
         return [
-            f'python3 -m pip install --no-cache-dir -r {req_path}'
+            f'{NB_PYTHON_PREFIX}/bin/pip install --no-cache-dir -r {req_path}'
         ]
 
 
@@ -124,4 +125,4 @@ def main():
         raise Exception("start isn't implemented yet")
 
 if __name__ == '__main__':
-    main()
+    main()

pangeo-notebook/binder/apt.txt

@@ -1 +1,2 @@
 graphviz
+vim