diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml index d319b47a6..11e91e22c 100644 --- a/.github/workflows/build_push.yaml +++ b/.github/workflows/build_push.yaml @@ -212,4 +212,4 @@ jobs: uses: act10ns/slack@v1 with: status: failure - message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} + message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b2b27547b..1696ccd57 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ __pycache__ .venv .ipynb_checkpoints/ .idea +node_modules +package-lock.json +package.json \ No newline at end of file diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 34493e037..9d03a02ab 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -4,17 +4,29 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6e278cb46946ccf60d4d2 # For compatibility with docker stacks -ARG NB_USER="jovyan" ARG HOME=/home/$NB_USER -ENV NB_UID="1000" -ENV NB_GID="100" +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +ENV NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + CONDA_DIR=/opt/conda \ + PATH=$PATH:/opt/conda/bin \ + NB_USER="jovyan" \ + HOME="/home/${NB_USER}" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" #Fix-permissions COPY remote-desktop/fix-permissions /usr/bin/fix-permissions -RUN chmod u+x /usr/bin/fix-permissions +#clean up +COPY clean-layer.sh /usr/bin/clean-layer.sh + +RUN chmod u+x /usr/bin/fix-permissions \ + && chmod +x /usr/bin/clean-layer.sh RUN apt-get update --yes \ && apt-get install --yes python3-pip tini language-pack-fr \ @@ -26,3 +38,63 @@ RUN /rocker_scripts/install_shiny_server.sh \ # Users should install R packages in their home directory RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ + + +# ARG CONDA_VERSION=py38_4.10.3 +# ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c +ARG PYTHON_VERSION=3.11 + +# #Install Miniconda +# #Has to be appended, else messes with qgis +# RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ +# echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ +# if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ +# mkdir -p /opt && \ +# sh miniconda.sh -b -p /opt/conda && \ +# rm miniconda.sh miniconda.md5 && \ +# ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ +# echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ +# echo "conda activate base" >> ~/.bashrc && \ +# find /opt/conda/ -follow -type f -name '*.a' -delete && \ +# find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ +# /opt/conda/bin/conda clean -afy && \ +# chown -R $NB_UID:$NB_GID /opt/conda +# +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + clean-layer.sh && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" \ No newline at end of file diff --git a/docker-bits/0_cpu.Dockerfile b/docker-bits/0_cpu.Dockerfile index 23c989a38..78efa1545 100644 --- a/docker-bits/0_cpu.Dockerfile +++ b/docker-bits/0_cpu.Dockerfile @@ -10,13 +10,10 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh \ No newline at end of file diff --git a/docker-bits/0_cpu_sas.Dockerfile b/docker-bits/0_cpu_sas.Dockerfile index f96c29355..85f75878b 100644 --- a/docker-bits/0_cpu_sas.Dockerfile +++ b/docker-bits/0_cpu_sas.Dockerfile @@ -12,12 +12,15 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh #updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 #TODO: Evaluate if this is still necessary when updating the base image RUN pip install --force-reinstall cryptography==39.0.1 && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions /home/$NB_USER \ No newline at end of file diff --git a/docker-bits/2_cpu.Dockerfile b/docker-bits/2_cpu.Dockerfile index b5e930840..4bfa9d62b 100644 --- a/docker-bits/2_cpu.Dockerfile +++ b/docker-bits/2_cpu.Dockerfile @@ -1,6 +1,6 @@ # Create conda environment (CPU only) with many useful packages. -RUN conda create -n pycpu --yes \ +RUN mamba create -n pycpu --yes \ python==3.11.0 ipython==8.11.0 sphinx==6.1.3 \ boto==2.49.0 s3fs==2023.3.0 \ dos2unix==7.4.1 parallel==20230122 \ @@ -12,6 +12,6 @@ RUN conda create -n pycpu --yes \ gensim==4.3.0 nltk==3.8.1 spacy==3.5.0 \ pytorch==1.13.1 torchaudio==0.13.1 torchvision==0.14.1 cpuonly==2.0 \ -c pytorch -c conda-forge && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/2_pytorch.Dockerfile b/docker-bits/2_pytorch.Dockerfile index 5b73592bf..ab2bdabf2 100644 --- a/docker-bits/2_pytorch.Dockerfile +++ b/docker-bits/2_pytorch.Dockerfile @@ -10,7 +10,7 @@ RUN mamba create -n torch && \ gputil \ # pytorch-cuda are the nvidia cuda drivers pytorch-cuda=11.8 && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ python -m ipykernel install --user --name torch --display-name "PyTorch" diff --git a/docker-bits/2_tensorflow.Dockerfile b/docker-bits/2_tensorflow.Dockerfile index 6b087f518..a4b12df0e 100644 --- a/docker-bits/2_tensorflow.Dockerfile +++ b/docker-bits/2_tensorflow.Dockerfile @@ -1,8 +1,9 @@ # Install Tensorflow -RUN pip install --quiet \ - 'tensorflow' \ - 'keras' \ - 'ipykernel==6.21.3' \ +RUN mamba install --quiet --yes \ + tensorflow \ + keras \ + ipykernel \ && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 5135f201c..61fed3f5c 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -1,27 +1,17 @@ USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/docker-bits/4_CLI.Dockerfile b/docker-bits/4_CLI.Dockerfile index d22374109..1873c0c90 100644 --- a/docker-bits/4_CLI.Dockerfile +++ b/docker-bits/4_CLI.Dockerfile @@ -1,12 +1,14 @@ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -15,20 +17,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -40,26 +38,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/docker-bits/5_DB-Drivers.Dockerfile b/docker-bits/5_DB-Drivers.Dockerfile index 4a73cb4da..96e55819e 100644 --- a/docker-bits/5_DB-Drivers.Dockerfile +++ b/docker-bits/5_DB-Drivers.Dockerfile @@ -1,7 +1,7 @@ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index a981b6389..6f9f8fbd3 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -27,9 +27,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -48,16 +49,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -67,21 +64,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -93,31 +85,33 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ - 'r-languageserver' \ + mamba install -c conda-forge \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ + 'yaml-language-server' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ @@ -128,7 +122,7 @@ RUN apt-get install --yes sqlite3 \ ARG NODE_OPTIONS=--openssl-legacy-provider RUN sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ && sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ - && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ + && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/traverse@7.23.2\ && npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ && rm -r ${OMPP_INSTALL_DIR}/html \ && mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \ diff --git a/docker-bits/6_remote-desktop.Dockerfile b/docker-bits/6_remote-desktop.Dockerfile index b3a9d7531..c5161d293 100644 --- a/docker-bits/6_remote-desktop.Dockerfile +++ b/docker-bits/6_remote-desktop.Dockerfile @@ -6,9 +6,6 @@ ENV XDG_DATA_HOME=/etc/share ENV VSCODE_DIR=$XDG_DATA_HOME/code ENV VSCODE_EXTENSIONS=$VSCODE_DIR/extensions -COPY clean-layer.sh /usr/bin/clean-layer.sh -RUN chmod +x /usr/bin/clean-layer.sh - ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update \ && apt-get install -y dbus-x11 \ @@ -284,7 +281,8 @@ ENV OMPP_GROUP=100 ENV OMPP_UID=$NB_UID ENV OMPP_GID=$NB_GID # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && tar -xf /tmp/ompp.tar.gz -C /tmp/ \ @@ -339,26 +337,6 @@ RUN apt-get update && apt-get install --yes websockify \ && cp /usr/lib/websockify/rebind.cpython-38-x86_64-linux-gnu.so /usr/lib/websockify/rebind.so \ && clean-layer.sh -#Install Miniconda -#Has to be appended, else messes with qgis -ENV PATH $PATH:/opt/conda/bin - -ARG CONDA_VERSION=py38_4.10.3 -ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ - echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ - if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ - mkdir -p /opt && \ - sh miniconda.sh -b -p /opt/conda && \ - rm miniconda.sh miniconda.md5 && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - find /opt/conda/ -follow -type f -name '*.a' -delete && \ - find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ - /opt/conda/bin/conda clean -afy && \ - chown -R $NB_UID:$NB_GID /opt/conda #Set Defaults ENV HOME=/home/$NB_USER diff --git a/docker-bits/6_rstudio.Dockerfile b/docker-bits/6_rstudio.Dockerfile index 365964568..c84bf18c7 100644 --- a/docker-bits/6_rstudio.Dockerfile +++ b/docker-bits/6_rstudio.Dockerfile @@ -7,7 +7,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -19,7 +19,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/docker-stacks-datascience-notebook/clean-layer.sh b/output/docker-stacks-datascience-notebook/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/docker-stacks-datascience-notebook/clean-layer.sh +++ b/output/docker-stacks-datascience-notebook/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index d0c3e8b8d..d2df693ce 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -20,48 +20,35 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -69,13 +56,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -84,20 +73,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -109,26 +94,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -159,8 +140,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -198,9 +179,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -219,16 +201,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -238,21 +216,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -264,31 +237,33 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ - 'r-languageserver' \ + mamba install -c conda-forge \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ + 'yaml-language-server' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ @@ -299,7 +274,7 @@ RUN apt-get install --yes sqlite3 \ ARG NODE_OPTIONS=--openssl-legacy-provider RUN sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ && sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ - && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ + && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/traverse@7.23.2\ && npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ && rm -r ${OMPP_INSTALL_DIR}/html \ && mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \ diff --git a/output/jupyterlab-cpu/clean-layer.sh b/output/jupyterlab-cpu/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-cpu/clean-layer.sh +++ b/output/jupyterlab-cpu/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index c428f9df4..c7398e006 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -20,16 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/2_pytorch.Dockerfile @@ -47,7 +44,7 @@ RUN mamba create -n torch && \ gputil \ # pytorch-cuda are the nvidia cuda drivers pytorch-cuda=11.8 && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ python -m ipykernel install --user --name torch --display-name "PyTorch" @@ -58,32 +55,22 @@ RUN mamba create -n torch && \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -91,13 +78,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -106,20 +95,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -131,26 +116,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -181,8 +162,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -220,9 +201,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -241,16 +223,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -260,21 +238,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -286,31 +259,33 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ - 'r-languageserver' \ + mamba install -c conda-forge \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ + 'yaml-language-server' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ @@ -321,7 +296,7 @@ RUN apt-get install --yes sqlite3 \ ARG NODE_OPTIONS=--openssl-legacy-provider RUN sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ && sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ - && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ + && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/traverse@7.23.2\ && npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ && rm -r ${OMPP_INSTALL_DIR}/html \ && mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \ diff --git a/output/jupyterlab-pytorch/clean-layer.sh b/output/jupyterlab-pytorch/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-pytorch/clean-layer.sh +++ b/output/jupyterlab-pytorch/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 670557265..864053df1 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -20,16 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/1_CUDA-11.8.0.Dockerfile @@ -151,11 +148,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ############################### # Install Tensorflow -RUN pip install --quiet \ - 'tensorflow' \ - 'keras' \ - 'ipykernel==6.21.3' \ +RUN mamba install --quiet --yes \ + tensorflow \ + keras \ + ipykernel \ && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -164,32 +162,22 @@ RUN pip install --quiet \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -197,13 +185,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -212,20 +202,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -237,26 +223,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -287,8 +269,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -326,9 +308,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -347,16 +330,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -366,21 +345,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -392,31 +366,33 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ - 'r-languageserver' \ + mamba install -c conda-forge \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ + 'yaml-language-server' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ @@ -427,7 +403,7 @@ RUN apt-get install --yes sqlite3 \ ARG NODE_OPTIONS=--openssl-legacy-provider RUN sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ && sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ - && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ + && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/traverse@7.23.2\ && npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ && rm -r ${OMPP_INSTALL_DIR}/html \ && mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \ diff --git a/output/jupyterlab-tensorflow/clean-layer.sh b/output/jupyterlab-tensorflow/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-tensorflow/clean-layer.sh +++ b/output/jupyterlab-tensorflow/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 55af1f51f..e7bd96ab7 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -9,17 +9,29 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6e278cb46946ccf60d4d2 # For compatibility with docker stacks -ARG NB_USER="jovyan" ARG HOME=/home/$NB_USER -ENV NB_UID="1000" -ENV NB_GID="100" +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +ENV NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + CONDA_DIR=/opt/conda \ + PATH=$PATH:/opt/conda/bin \ + NB_USER="jovyan" \ + HOME="/home/${NB_USER}" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" #Fix-permissions COPY remote-desktop/fix-permissions /usr/bin/fix-permissions -RUN chmod u+x /usr/bin/fix-permissions +#clean up +COPY clean-layer.sh /usr/bin/clean-layer.sh + +RUN chmod u+x /usr/bin/fix-permissions \ + && chmod +x /usr/bin/clean-layer.sh RUN apt-get update --yes \ && apt-get install --yes python3-pip tini language-pack-fr \ @@ -32,37 +44,87 @@ RUN /rocker_scripts/install_shiny_server.sh \ # Users should install R packages in their home directory RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ + +# ARG CONDA_VERSION=py38_4.10.3 +# ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c +ARG PYTHON_VERSION=3.11 + +# #Install Miniconda +# #Has to be appended, else messes with qgis +# RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ +# echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ +# if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ +# mkdir -p /opt && \ +# sh miniconda.sh -b -p /opt/conda && \ +# rm miniconda.sh miniconda.md5 && \ +# ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ +# echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ +# echo "conda activate base" >> ~/.bashrc && \ +# find /opt/conda/ -follow -type f -name '*.a' -delete && \ +# find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ +# /opt/conda/bin/conda clean -afy && \ +# chown -R $NB_UID:$NB_GID /opt/conda +# +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + clean-layer.sh && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -70,13 +132,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -85,20 +149,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -110,26 +170,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -166,9 +222,6 @@ ENV XDG_DATA_HOME=/etc/share ENV VSCODE_DIR=$XDG_DATA_HOME/code ENV VSCODE_EXTENSIONS=$VSCODE_DIR/extensions -COPY clean-layer.sh /usr/bin/clean-layer.sh -RUN chmod +x /usr/bin/clean-layer.sh - ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update \ && apt-get install -y dbus-x11 \ @@ -444,7 +497,8 @@ ENV OMPP_GROUP=100 ENV OMPP_UID=$NB_UID ENV OMPP_GID=$NB_GID # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && tar -xf /tmp/ompp.tar.gz -C /tmp/ \ @@ -499,26 +553,6 @@ RUN apt-get update && apt-get install --yes websockify \ && cp /usr/lib/websockify/rebind.cpython-38-x86_64-linux-gnu.so /usr/lib/websockify/rebind.so \ && clean-layer.sh -#Install Miniconda -#Has to be appended, else messes with qgis -ENV PATH $PATH:/opt/conda/bin - -ARG CONDA_VERSION=py38_4.10.3 -ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ - echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ - if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ - mkdir -p /opt && \ - sh miniconda.sh -b -p /opt/conda && \ - rm miniconda.sh miniconda.md5 && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - find /opt/conda/ -follow -type f -name '*.a' -delete && \ - find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ - /opt/conda/bin/conda clean -afy && \ - chown -R $NB_UID:$NB_GID /opt/conda #Set Defaults ENV HOME=/home/$NB_USER diff --git a/output/remote-desktop/clean-layer.sh b/output/remote-desktop/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/remote-desktop/clean-layer.sh +++ b/output/remote-desktop/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/remote-desktop/initial-condarc b/output/remote-desktop/initial-condarc new file mode 100644 index 000000000..66ecf6081 --- /dev/null +++ b/output/remote-desktop/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index a08092193..92603ba54 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -20,48 +20,35 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -69,13 +56,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -84,20 +73,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -109,26 +94,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -159,8 +140,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -202,7 +183,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -214,7 +195,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/rstudio/clean-layer.sh b/output/rstudio/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/rstudio/clean-layer.sh +++ b/output/rstudio/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 65c1d0a20..2f4c14bf4 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -17,9 +17,12 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh #updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 #TODO: Evaluate if this is still necessary when updating the base image @@ -32,32 +35,22 @@ RUN pip install --force-reinstall cryptography==39.0.1 && \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ - 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ +RUN mamba install --quiet \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 's3fs' \ + 'fire==0.5.0' \ + 'graphviz' && \ + pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ - 'fire==0.5.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile @@ -65,13 +58,15 @@ RUN chmod +x /usr/local/bin/aaw-suspend-server.sh USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -80,20 +75,16 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -105,26 +96,22 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ @@ -155,8 +142,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -194,9 +181,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -215,16 +203,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -234,21 +218,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -260,31 +239,33 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ - 'r-languageserver' \ + mamba install -c conda-forge \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ + 'yaml-language-server' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ @@ -295,7 +276,7 @@ RUN apt-get install --yes sqlite3 \ ARG NODE_OPTIONS=--openssl-legacy-provider RUN sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ && sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.conf.js \ - && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ + && npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/traverse@7.23.2\ && npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \ && rm -r ${OMPP_INSTALL_DIR}/html \ && mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \ @@ -347,7 +328,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -359,7 +340,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/sas/clean-layer.sh b/output/sas/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/sas/clean-layer.sh +++ b/output/sas/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/resources/common/clean-layer.sh b/resources/common/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/resources/common/clean-layer.sh +++ b/resources/common/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/resources/remote-desktop/initial-condarc b/resources/remote-desktop/initial-condarc new file mode 100644 index 000000000..66ecf6081 --- /dev/null +++ b/resources/remote-desktop/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file diff --git a/tests/general/test_packages.py b/tests/general/test_packages.py index 79e9b5f89..589cf4612 100644 --- a/tests/general/test_packages.py +++ b/tests/general/test_packages.py @@ -95,6 +95,7 @@ "pillow", # import yaml, not import pyyaml "pyyaml", + "graphviz", ]