From c99c6babfdba5b24fa3377c2f8d80938f1f9aa88 Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 4 Apr 2020 21:31:59 +0200 Subject: [PATCH 1/3] [WIP] Trying to switch to bionic travis dist Tring to swithch to the last bionic Travis build environment. https://docs.travis-ci.com/user/reference/bionic/ --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0130968878..720984b1fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: xenial +dist: bionic language: python python: - 3.7 From d458afffaf230eca7ceb355c04c8a05f7d2ab8e0 Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 4 Apr 2020 23:04:45 +0200 Subject: [PATCH 2/3] Big upgrade - Bump root container version - Bump packages of - scipy-notebook - r-notebook - datascience-notebook - add -y flag to jupyter lab commands in scipy-notebook - Fix a bug in check outdated packages when packages installed through pip --- all-spark-notebook/Dockerfile | 4 ++-- base-notebook/Dockerfile | 2 +- datascience-notebook/Dockerfile | 8 ++++---- r-notebook/Dockerfile | 4 ++-- scipy-notebook/Dockerfile | 10 +++++----- test/helpers.py | 4 +++- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index b0e56d349f..bb111817a0 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -23,8 +23,8 @@ USER $NB_UID # R packages RUN conda install --quiet --yes \ - 'r-base=3.6.2' \ - 'r-ggplot2=3.2*' \ + 'r-base=3.6.3' \ + 'r-ggplot2=3.3*' \ 'r-irkernel=1.1*' \ 'r-rcurl=1.98*' \ 'r-sparklyr=1.1*' \ diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index b45672ae15..8a38968d3b 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -4,7 +4,7 @@ # Ubuntu 18.04 (bionic) # https://hub.docker.com/_/ubuntu/?tab=tags&name=bionic # OS/ARCH: linux/amd64 -ARG ROOT_CONTAINER=ubuntu:bionic-20200219@sha256:0925d086715714114c1988f7c947db94064fd385e171a63c07730f1fa014e6f9 +ARG ROOT_CONTAINER=ubuntu:bionic-20200311@sha256:e5dd9dbb37df5b731a6688fa49f4003359f6f126958c9c928f937bec69836320 ARG BASE_CONTAINER=$ROOT_CONTAINER FROM $BASE_CONTAINER diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index d3744b0eb7..c3b1a61ecb 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -45,11 +45,11 @@ USER $NB_UID # R packages including IRKernel which gets installed globally. RUN conda install --quiet --yes \ - 'r-base=3.6.2' \ + 'r-base=3.6.3' \ 'r-caret=6.0*' \ 'r-crayon=1.3*' \ 'r-devtools=2.2*' \ - 'r-forecast=8.10*' \ + 'r-forecast=8.11*' \ 'r-hexbin=1.28*' \ 'r-htmltools=0.4*' \ 'r-htmlwidgets=1.5*' \ @@ -60,8 +60,8 @@ RUN conda install --quiet --yes \ 'r-rcurl=1.98*' \ 'r-reshape2=1.4*' \ 'r-rmarkdown=2.1*' \ - 'r-rsqlite=2.1*' \ - 'r-shiny=1.3*' \ + 'r-rsqlite=2.2*' \ + 'r-shiny=1.4*' \ 'r-tidyverse=1.3*' \ 'rpy2=3.1*' \ && \ diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 8b29f82375..bf11abd4c3 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -25,7 +25,7 @@ USER $NB_UID # R packages RUN conda install --quiet --yes \ - 'r-base=3.6.2' \ + 'r-base=3.6.3' \ 'r-caret=6.0*' \ 'r-crayon=1.3*' \ 'r-devtools=2.2*' \ @@ -41,7 +41,7 @@ RUN conda install --quiet --yes \ 'r-reshape2=1.4*' \ 'r-rmarkdown=2.1*' \ 'r-rodbc=1.3*' \ - 'r-rsqlite=2.1*' \ + 'r-rsqlite=2.2*' \ 'r-shiny=1.4*' \ 'r-tidyverse=1.3*' \ 'unixodbc=2.3.*' \ diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index c6022e3a01..a6f116158a 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -18,16 +18,16 @@ USER $NB_UID RUN conda install --quiet --yes \ 'beautifulsoup4=4.8.*' \ 'conda-forge::blas=*=openblas' \ - 'bokeh=1.4.*' \ + 'bokeh=2.0.*' \ 'cloudpickle=1.3.*' \ 'cython=0.29.*' \ - 'dask=2.11.*' \ + 'dask=2.14.*' \ 'dill=0.3.*' \ 'h5py=2.10.*' \ 'hdf5=1.10.*' \ 'ipywidgets=7.5.*' \ 'ipympl=0.5.*'\ - 'matplotlib-base=3.1.*' \ + 'matplotlib-base=3.2.*' \ 'numba=0.48.*' \ 'numexpr=2.7.*' \ 'pandas=1.0.*' \ @@ -53,8 +53,8 @@ RUN conda install --quiet --yes \ jupyter labextension install @jupyter-widgets/jupyterlab-manager@^2.0.0 --no-build && \ jupyter labextension install @bokeh/jupyter_bokeh@^2.0.0 --no-build && \ jupyter labextension install jupyter-matplotlib@^0.7.2 --no-build && \ - jupyter lab build && \ - jupyter lab clean && \ + jupyter lab build -y && \ + jupyter lab clean -y && \ npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ diff --git a/test/helpers.py b/test/helpers.py index b3600a4154..d3bb0c8131 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -89,7 +89,9 @@ def _packages_from_json(env_export): """Extract packages and versions from the lines returned by the list of specifications""" dependencies = json.loads(env_export).get("dependencies") packages_dict = dict() - for split in map(lambda x: x.split("=", 1), dependencies): + for split in map(lambda x: x.split("=", 1) if isinstance(x, str) else None, dependencies): + if split is None: + break # default values package = split[0] version = set() From 8de52359adc001426731c0f5fb42406393e4d9b8 Mon Sep 17 00:00:00 2001 From: romainx Date: Mon, 6 Apr 2020 19:26:58 +0200 Subject: [PATCH 3/3] Fix a bug in the helper method and also make the code more readable. --- test/helpers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/helpers.py b/test/helpers.py index d3bb0c8131..eb410c403a 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -87,11 +87,13 @@ def _execute_command(self, command): @staticmethod def _packages_from_json(env_export): """Extract packages and versions from the lines returned by the list of specifications""" + #dependencies = filter(lambda x: isinstance(x, str), json.loads(env_export).get("dependencies")) dependencies = json.loads(env_export).get("dependencies") + # Filtering packages installed through pip in this case it's a dict {'pip': ['toree==0.3.0']} + # Since we only manage packages installed through conda here + dependencies = filter(lambda x: isinstance(x, str), dependencies) packages_dict = dict() - for split in map(lambda x: x.split("=", 1) if isinstance(x, str) else None, dependencies): - if split is None: - break + for split in map(lambda x: x.split("=", 1), dependencies): # default values package = split[0] version = set()