From b15e48c0c091044cf96b779b7ee96abc27038e1d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 5 Aug 2022 11:14:40 +0200 Subject: [PATCH 01/16] Major revision to base the image on the jupyter docker stack. - The PGSQL and RabbitMQ services are expected to started in separate containers. - Do not separate the Jupyter and AiiDA Python kernels; use shared requirements file. - Use requirements file for environment-wide pinning (conda and pip). - Supports AiiDA 2.0. Co-authored-by: Aliaksandr Yakutovich --- .github/workflows/build_and_test_on_pr.yml | 5 + Dockerfile | 189 ++++++++---------- .../factory_reset.sh | 12 +- before-notebook.d/prepare-aiida.sh | 77 +++++++ before-notebook.d/prepare-aiidalab.sh | 66 ++++++ config-quick-setup.yaml | 15 ++ my_init.d/prepare-aiidalab.sh | 7 - my_my_init | 3 - opt/aiidalab-singleuser | 34 ---- opt/prepare-aiidalab.sh | 96 --------- requirements-server.txt | 17 -- requirements.txt | 13 +- service/jupyter-notebook | 4 - 13 files changed, 256 insertions(+), 282 deletions(-) rename {my_init.d => before-notebook.d}/factory_reset.sh (78%) create mode 100755 before-notebook.d/prepare-aiida.sh create mode 100755 before-notebook.d/prepare-aiidalab.sh create mode 100644 config-quick-setup.yaml delete mode 100755 my_init.d/prepare-aiidalab.sh delete mode 100755 my_my_init delete mode 100755 opt/aiidalab-singleuser delete mode 100755 opt/prepare-aiidalab.sh delete mode 100644 requirements-server.txt delete mode 100755 service/jupyter-notebook diff --git a/.github/workflows/build_and_test_on_pr.yml b/.github/workflows/build_and_test_on_pr.yml index 4e1b510a..c852e0b2 100644 --- a/.github/workflows/build_and_test_on_pr.yml +++ b/.github/workflows/build_and_test_on_pr.yml @@ -48,8 +48,13 @@ jobs: echo "::set-output name=docker_id_first_run::${DOCKERID}" docker exec --tty --user root $DOCKERID wait-for-services docker exec --tty --user aiida $DOCKERID wait-for-services +<<<<<<< HEAD docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/opt/conda/envs/pgsql/bin/pg_ctl -D /home/$SYSTEM_USER/.postgresql status' # Check that postgres is up. docker exec --tty --user root $DOCKERID /bin/bash -l -c '/opt/conda/envs/rmq/bin/rabbitmqctl status' # Check that rabbitmq is up. +======= + docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/usr/lib/postgresql/10/bin/pg_ctl -D /home/$NB_USER/.postgresql status' # Check that postgres is up. + docker exec --tty --user root $DOCKERID /bin/bash -l -c 'service rabbitmq-server status' # Check that rabbitmq is up. +>>>>>>> ee7ece1 (Major revision to base the image on the jupyter docker stack.) docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda create -y -n test_env python=3.8' # Check that one can create a new conda environment. docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda activate test_env' # Check that new environment works. sudo cp tmp/.ssh/id_rsa . # Copy id_rsa file from the mounted folder. diff --git a/Dockerfile b/Dockerfile index 2e2a554e..1b4fcf68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,78 @@ -FROM aiidateam/aiida-core:1.6.9 +FROM jupyter/minimal-notebook:python-3.9.12 LABEL maintainer="AiiDAlab Team " -# Specify default factory reset (not set): -ENV AIIDALAB_FACTORY_RESET "" +USER root +WORKDIR /opt/ + +# Install the shared requirements. +COPY requirements.txt . +RUN mamba install --yes \ + --file requirements.txt && \ + mamba clean -all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Pin shared requirements in the base environemnt. +RUN cat requirements.txt | xargs -I{} conda config --system --add pinned_packages {} + +# Configure pip to use requirements file as constraints file. +ENV PIP_CONSTRAINT=/opt/requirements.txt + +# TODO: should be converted to a conda package +# ARG aiidalab_version=aiida-2.0 +RUN pip install --quiet --no-cache-dir \ + # TODO: switch to release version + "aiidalab@git+https://github.com/aiidalab/aiidalab@main" && \ + # "aiidalab==${aiidalab_version}" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install the aiidalab-home app. +ARG aiidalab_home_version=develop +RUN git clone https://github.com/aiidalab/aiidalab-home && \ + cd aiidalab-home && \ + git checkout "${aiidalab_home_version}" && \ + pip install --quiet --no-cache-dir "./" && \ + fix-permissions "./" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install and enable appmode. +RUN git clone https://github.com/oschuett/appmode.git && \ + cd appmode && \ + git checkout v0.8.0 +COPY gears.svg ./appmode/appmode/static/gears.svg +RUN pip install ./appmode --no-cache-dir && \ + jupyter nbextension enable --py --sys-prefix appmode && \ + jupyter serverextension enable --py --sys-prefix appmode + +# Enable verdi autocompletion. +RUN echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + fix-permissions "${CONDA_DIR}" + +# Configure AiiDA profile. +COPY config-quick-setup.yaml . +COPY before-notebook.d/prepare-aiida.sh /usr/local/bin/before-notebook.d/ + +# Perform factory reset if needed. +COPY before-notebook.d/factory_reset.sh /usr/local/bin/before-notebook.d/ + +# Prepare user's folders for AiiDAlab launch. +COPY before-notebook.d/prepare-aiidalab.sh /usr/local/bin/before-notebook.d/ -# Configure environment. -ENV AIIDALAB_HOME /home/${SYSTEM_USER} +# Configure AiiDA. +ENV SETUP_DEFAULT_AIIDA_PROFILE true +ENV AIIDA_PROFILE_NAME default +ENV AIIDA_USER_EMAIL aiida@localhost +ENV AIIDA_USER_FIRST_NAME Giuseppe +ENV AIIDA_USER_LAST_NAME Verdi +ENV AIIDA_USER_INSTITUTION Khedivial + + +# Configure AiiDAlab environment. +ENV AIIDALAB_HOME /home/${NB_USER} ENV AIIDALAB_APPS ${AIIDALAB_HOME}/apps ENV AIIDALAB_DEFAULT_GIT_BRANCH master @@ -20,108 +86,23 @@ ENV AIIDALAB_DEFAULT_GIT_BRANCH master # # Please note that multiple entries must be whitespace delimited. # Please see `aiidalab install --help` for more information. -ENV AIIDALAB_DEFAULT_APPS "aiidalab-widgets-base~=1.0" - -USER root -WORKDIR /opt/ - -# Install OS dependencies. -# Not clear whether libssl-dev and libffi-dev are still needed. -# povray needed for structure editor widget. -RUN apt-get update && apt-get install -y \ - ca-certificates \ - file \ - libssl-dev \ - libffi-dev \ - povray \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Dependencies needed for Jupyter Lab. -RUN apt-get update && apt-get install -y \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -# Install ngrok to be able to proxy AiiDA RESTful API server. -# Currently not used by the home app, but used in tutorials -RUN wget --quiet -P /tmp/ \ - https://bin.equinox.io/a/dnxFaDKQgP4/ngrok-2.3.35-linux-amd64.zip \ - && unzip /tmp/ngrok-2.3.35-linux-amd64.zip \ - && mv ./ngrok /usr/local/bin/ \ - && rm -f /tmp/ngrok-2.3.35-linux-amd64.zip - -# Get recent version of pip (needed for `pip cache` command). -# New pip executable is installed into /usr/local/bin -RUN /usr/bin/pip3 install --upgrade pip - -# Jupyter dependencies installed into system python environment -# which runs the jupyter notebook server. -COPY requirements-server.txt . -RUN /usr/local/bin/pip install -r /opt/requirements-server.txt \ - && /usr/local/bin/pip cache purge - -# Install and enable appmode. -RUN git clone https://github.com/oschuett/appmode.git && cd appmode && git reset --hard v0.8.0 -COPY gears.svg ./appmode/appmode/static/gears.svg -RUN /usr/local/bin/pip install ./appmode -RUN /usr/local/bin/jupyter nbextension enable --py --sys-prefix appmode -RUN /usr/local/bin/jupyter serverextension enable --py --sys-prefix appmode - -# Install jupyterlab theme (takes about 4 minutes and 10 seconds). -#WORKDIR /opt/jupyterlab-theme -#RUN git clone https://github.com/aiidalab/jupyterlab-theme && \ -# cd jupyterlab-theme && \ -# npm install && \ -# npm run build && \ -# npm run build:webpack && \ -# npm pack ./ && \ -# /usr/local/bin/jupyter labextension install *.tgz && \ -# cd .. - -## Configure user environment - -# Install some useful packages that are not available on PyPi. -RUN conda install --yes -c conda-forge \ - openbabel==3.1.1 \ - rdkit==2021.09.2 \ - && conda clean --all - -# Install AiiDAlab Python packages into user conda environment and populate reentry cache. -COPY requirements.txt . -ARG extra_requirements -RUN pip install --upgrade pip -RUN pip install -r requirements.txt $extra_requirements -RUN reentry scan +# ENV AIIDALAB_DEFAULT_APPS "aiidalab-widgets-base~=1.0" +ENV AIIDALAB_DEFAULT_APPS "" -# Configure pip to use requirements file as constraints file. -RUN conda env config vars set PIP_CONSTRAINT=/opt/requirements.txt - -# Install python kernel from the conda environment (comes with the aiidalab package). -RUN python -m ipykernel install - - -# Perform factory reset if needed. -COPY my_init.d/factory_reset.sh /etc/my_init.d/09_factory_reset.sh - -# Prepare user's folders for AiiDAlab launch. -COPY opt/aiidalab-singleuser /opt/ -COPY opt/prepare-aiidalab.sh /opt/ -COPY my_init.d/prepare-aiidalab.sh /etc/my_init.d/80_prepare-aiidalab.sh +# Specify default factory reset (not set): +ENV AIIDALAB_FACTORY_RESET "" -# Install the aiidalab-home app. -ARG aiidalab_home_version=v22.01.0 -RUN git clone https://github.com/aiidalab/aiidalab-home && cd aiidalab-home && git checkout $aiidalab_home_version -RUN chmod 774 aiidalab-home +USER ${NB_USER} -# Copy scripts to start Jupyter notebook. -COPY opt/start-notebook.sh /opt/ -COPY service/jupyter-notebook /etc/service/jupyter-notebook/run +WORKDIR "/home/${NB_USER}" -# Expose port 8888. -EXPOSE 8888 +# Make sure that the known_hosts file is present inside the .ssh folder. +RUN mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ + touch /home/${NB_USER}/.ssh/known_hosts -# Remove when the following issue is fixed: https://github.com/jupyterhub/dockerspawner/issues/319. -COPY my_my_init /sbin/my_my_init +# Create apps folder. +RUN mkdir -p /home/${NB_USER}/apps -CMD ["/sbin/my_my_init"] +ENV NOTEBOOK_ARGS \ + "--NotebookApp.default_url='/apps/apps/home/start.ipynb'" \ + "--ContentsManager.allow_hidden=True" diff --git a/my_init.d/factory_reset.sh b/before-notebook.d/factory_reset.sh similarity index 78% rename from my_init.d/factory_reset.sh rename to before-notebook.d/factory_reset.sh index 4f8049ff..50404249 100755 --- a/my_init.d/factory_reset.sh +++ b/before-notebook.d/factory_reset.sh @@ -9,9 +9,9 @@ export SHELL=/bin/bash # 0b010 2 - Remove all files and directories within the users home directory. # If the ~/AIIDALAB_FACTORY_RESET file exists, parse it (and remove it). -if [ -e "/home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET" ]; then - RESET_MODE="`cat /home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET`" - rm -f "/home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET" # Remove file after parsing. +if [ -e "/home/${NB_USER}/AIIDALAB_FACTORY_RESET" ]; then + RESET_MODE="`cat /home/${NB_USER}/AIIDALAB_FACTORY_RESET`" + rm -f "/home/${NB_USER}/AIIDALAB_FACTORY_RESET" # Remove file after parsing. fi # If the AIIDALAB_FACTORY_RESET environment variable is set, it takes preference (default mode=0): @@ -32,12 +32,12 @@ fi # mode & 001 -> delete ~/apps/ and ~/.local/ if (( (${RESET_MODE} & 0x1) == 0x1)); then echo "factory reset: Remove apps (~/apps/) and local software installation (~/.local/)." - rm -rf "/home/${SYSTEM_USER}/apps" - rm -rf "/home/${SYSTEM_USER}/.local" + rm -rf "/home/${NB_USER}/apps" + rm -rf "/home/${NB_USER}/.local" fi # mode & 010 -> delete all home directory contents if (( (${RESET_MODE} & 0x2) == 0x2)); then echo "factory reset: Remove user home directory contents." - find "/home/${SYSTEM_USER}/" -mindepth 1 -delete + find "/home/${NB_USER}/" -mindepth 1 -delete fi diff --git a/before-notebook.d/prepare-aiida.sh b/before-notebook.d/prepare-aiida.sh new file mode 100755 index 00000000..0f5b91b8 --- /dev/null +++ b/before-notebook.d/prepare-aiida.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# This script is executed whenever the docker container is (re)started. + +# Debugging. +set -x + +# Environment. +export SHELL=/bin/bash + +# Check if user requested to set up AiiDA profile (and if it exists already) +if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then + NEED_SETUP_PROFILE=true; +else + NEED_SETUP_PROFILE=false; +fi + +# Setup AiiDA profile if needed. +if [[ ${NEED_SETUP_PROFILE} == true ]]; then + + # Create AiiDA profile. + verdi quicksetup \ + --non-interactive \ + --profile "${AIIDA_PROFILE_NAME}" \ + --email "${AIIDA_USER_EMAIL}" \ + --first-name "${AIIDA_USER_FIRST_NAME}" \ + --last-name "${AIIDA_USER_LAST_NAME}" \ + --institution "${AIIDA_USER_INSTITUTION}" \ + --config /opt/config-quick-setup.yaml + + # Setup and configure local computer. + computer_name=localhost + + # Determine the number of physical cores as a default for the number of + # available MPI ranks on the localhost. We do not count "logical" cores, + # since MPI parallelization over hyper-threaded cores is typically + # associated with a significant performance penalty. We use the + # `psutil.cpu_count(logical=False)` function as opposed to simply + # `os.cpu_count()` since the latter would include hyperthreaded (logical + # cores). + NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) + LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} + + if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then + echo "Unable to automatically determine the number of logical CPUs on this " + echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " + echo "explicitly set the number of available MPI ranks." + exit 1 + fi + + verdi computer show ${computer_name} || verdi computer setup \ + --non-interactive \ + --label "${computer_name}" \ + --description "this computer" \ + --hostname "${computer_name}" \ + --transport local \ + --scheduler direct \ + --work-dir /home/aiida/aiida_run/ \ + --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ + --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ + verdi computer configure local "${computer_name}" \ + --non-interactive \ + --safe-interval 0.0 +fi + + +# Show the default profile +verdi profile show || echo "The default profile is not set." + +# Make sure that the daemon is not running, otherwise the migration will abort. +verdi daemon stop + +# Migration will run for the default profile. +verdi storage migrate --force + +# Daemon will start only if the database exists and is migrated to the latest version. +verdi daemon start || echo "AiiDA daemon is not running." diff --git a/before-notebook.d/prepare-aiidalab.sh b/before-notebook.d/prepare-aiidalab.sh new file mode 100755 index 00000000..66766e76 --- /dev/null +++ b/before-notebook.d/prepare-aiidalab.sh @@ -0,0 +1,66 @@ +#!/bin/bash -e + +# Debugging. +set -x + +# Environment. +export SHELL=/bin/bash + +# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225 +if [ -L /home/${NB_USER}/${NB_USER} ]; then + rm /home/${NB_USER}/${NB_USER} +fi + +if [[ ! -f /home/${NB_USER}/.ssh/id_rsa ]]; then + # Generate ssh key that works with `paramiko` + # See: https://aiida.readthedocs.io/projects/aiida-core/en/latest/get_started/computers.html#remote-computer-requirements + ssh-keygen -f /home/${NB_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' +fi + +# Install the home app. +if [ ! -e /home/${NB_USER}/apps/home ]; then + echo "Install home app." + # The home app is installed in system space and linked to from user space. + # That ensures that users are not inadvertently running the wrong version of + # the home app for a given system environment, but still makes it possible to + # manually install a specific version of the home app in between upgrades, e.g., + # for development work, by simply replacing the link with a clone of the repository. + ln -s /opt/aiidalab-home /home/${NB_USER}/apps/home +elif [[ -d /home/${NB_USER}/apps/home && ! -L /home/${NB_USER}/apps/home ]]; then + # Backup an existing repository of the home app and replace with link to /opt/aiidalab-home. + # This mechanism preserves potential development work on a manually installed repository + # of the home app and also constitutes a migration path for existing aiidalab accounts, where + # the home app was installed directly into user space by default. + mv /home/${NB_USER}/apps/home /home/${NB_USER}/apps/.home~`date --iso-8601=seconds` \ + && ln -s /opt/aiidalab-home /home/${NB_USER}/apps/home || echo "WARNING: Unable to install home app." +fi + + +# Install default apps (see the Dockerfile for an explanation of the +# AIIDALAB_DEFAULT_APPS variable). +if [[ ${INITIAL_SETUP} == 1 ]]; then + + # Iterate over lines in AIIDALAB_DEFAULT_APPS variable. + for app in ${AIIDALAB_DEFAULT_APPS:-}; do + aiidalab install --yes "${app}" + done +fi + +# Clear user trash directory. +if [ -e /home/${NB_USER}/.trash ]; then + rm -rf /home/${NB_USER}/.trash/* +fi + +# Remove old apps_meta.sqlite requests cache files. +find -L /home/${NB_USER} -maxdepth 3 -name apps_meta.sqlite -writable -delete + +# Remove old temporary notebook files. +find -L /home/${NB_USER}/apps -maxdepth 2 -type f -name .*.ipynb -writable -delete + +# Uninstall aiidalab from user packages (if present). +# Would otherwise interfere with the system package. +USER_AIIDALAB_PACKAGE="$(python -c 'import site; print(site.USER_SITE)')/aiidalab" +if [ -e ${USER_AIIDALAB_PACKAGE} ]; then + echo "Uninstall local installation of aiidalab package." + pip uninstall --yes aiidalab +fi diff --git a/config-quick-setup.yaml b/config-quick-setup.yaml new file mode 100644 index 00000000..23d36d89 --- /dev/null +++ b/config-quick-setup.yaml @@ -0,0 +1,15 @@ +--- +db_engine: postgresql_psycopg2 +db_backend: psql_dos +db_host: database +db_port: 5432 +su_db_username: pguser +su_db_password: password +su_db_name: template1 +db_name: aiida_db +db_username: aiida +db_password: password +broker_host: messaging +broker_port: 5672 +broker_username: guest +broker_password: guest diff --git a/my_init.d/prepare-aiidalab.sh b/my_init.d/prepare-aiidalab.sh deleted file mode 100755 index eb40bbff..00000000 --- a/my_init.d/prepare-aiidalab.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -em - -# Change group of the aiidalab-home folder -chown root:${SYSTEM_USER} -R /opt/aiidalab-home - -su -c /opt/prepare-aiidalab.sh ${SYSTEM_USER} diff --git a/my_my_init b/my_my_init deleted file mode 100755 index edcdc77f..00000000 --- a/my_my_init +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/sbin/my_init diff --git a/opt/aiidalab-singleuser b/opt/aiidalab-singleuser deleted file mode 100755 index fe3e4332..00000000 --- a/opt/aiidalab-singleuser +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -from jupyterhub import singleuser - -# https://github.com/jupyterhub/jupyterhub/blob/master/scripts/jupyterhub-singleuser -# https://github.com/jupyterhub/jupyterhub/blob/master/jupyterhub/singleuser.py - -matcloud_page_template = """ -{% extends "templates/page.html" %} -{% block header_buttons %} -{{super()}} - - - Control Panel - - - - - Materials Cloud - - -{% endblock %} -{% block logo %} -Jupyter Notebook -{% endblock logo %} -""" - -if __name__ == '__main__': - singleuser.page_template = matcloud_page_template - singleuser.main() diff --git a/opt/prepare-aiidalab.sh b/opt/prepare-aiidalab.sh deleted file mode 100755 index 2a3e086b..00000000 --- a/opt/prepare-aiidalab.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -e - -# Debugging. -set -x - -# Environment. -export SHELL=/bin/bash - -# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225 -if [ -L /home/${SYSTEM_USER}/${SYSTEM_USER} ]; then - rm /home/${SYSTEM_USER}/${SYSTEM_USER} -fi - -# Setup AiiDA jupyter extension. -# Don't forget to copy this file to .ipython/profile_default/startup/ -# aiida/tools/ipython/aiida_magic_register.py -if [ ! -e /home/${SYSTEM_USER}/.ipython/profile_default/startup/aiida_magic_register.py ]; then - mkdir -p /home/${SYSTEM_USER}/.ipython/profile_default/startup/ - cat << EOF > /home/${SYSTEM_USER}/.ipython/profile_default/startup/aiida_magic_register.py -if __name__ == "__main__": - - try: - import aiida - del aiida - except ImportError: - pass - else: - import IPython - # pylint: disable=ungrouped-imports - from aiida.tools.ipython.ipython_magics import load_ipython_extension - - # Get the current Ipython session - IPYSESSION = IPython.get_ipython() - - # Register the line magic - load_ipython_extension(IPYSESSION) -EOF -fi - -# Create apps folder and make its subfolders importable from Python. -if [ ! -e /home/${SYSTEM_USER}/apps ]; then - # Create apps folder and make it importable from python. - mkdir -p /home/${SYSTEM_USER}/apps - INITIAL_SETUP=1 -fi - -# Install the home app. -if [ ! -e /home/${SYSTEM_USER}/apps/home ]; then - echo "Install home app." - # The home app is installed in system space and linked to from user space. - # That ensures that users are not inadvertently running the wrong version of - # the home app for a given system environment, but still makes it possible to - # manually install a specific version of the home app in between upgrades, e.g., - # for development work, by simply replacing the link with a clone of the repository. - ln -s /opt/aiidalab-home /home/${SYSTEM_USER}/apps/home -elif [[ -d /home/${SYSTEM_USER}/apps/home && ! -L /home/${SYSTEM_USER}/apps/home ]]; then - # Backup an existing repository of the home app and replace with link to /opt/aiidalab-home. - # This mechanism preserves potential development work on a manually installed repository - # of the home app and also constitutes a migration path for existing aiidalab accounts, where - # the home app was installed directly into user space by default. - mv /home/${SYSTEM_USER}/apps/home /home/${SYSTEM_USER}/apps/.home~`date --iso-8601=seconds` \ - && ln -s /opt/aiidalab-home /home/${SYSTEM_USER}/apps/home || echo "WARNING: Unable to install home app." -fi - - -# Install default apps (see the Dockerfile for an explanation of the -# AIIDALAB_DEFAULT_APPS variable). -if [[ ${INITIAL_SETUP} == 1 ]]; then - - # Iterate over lines in AIIDALAB_DEFAULT_APPS variable. - for app in ${AIIDALAB_DEFAULT_APPS:-}; do - aiidalab install --yes "${app}" - done -fi - -# Update reentry. -reentry scan - -# Clear user trash directory. -if [ -e /home/${SYSTEM_USER}/.trash ]; then - rm -rf /home/${SYSTEM_USER}/.trash/* -fi - -# Remove old apps_meta.sqlite requests cache files. -find -L /home/${SYSTEM_USER} -maxdepth 3 -name apps_meta.sqlite -writable -delete - -# Remove old temporary notebook files. -find -L /home/${SYSTEM_USER}/apps -maxdepth 2 -type f -name .*.ipynb -writable -delete - -# Uninstall aiidalab from user packages (if present). -# Would otherwise interfere with the system package. -USER_AIIDALAB_PACKAGE="$(/opt/conda/bin/python -c 'import site; print(site.USER_SITE)')/aiidalab" -if [ -e ${USER_AIIDALAB_PACKAGE} ]; then - echo "Uninstall local installation of aiidalab package." - /opt/conda/bin/python -m pip uninstall --yes aiidalab -fi diff --git a/requirements-server.txt b/requirements-server.txt deleted file mode 100644 index 9a4e0733..00000000 --- a/requirements-server.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Dependencies for notebook server and interaction with JupyterHub -jupyterhub==1.5.0 -jupyterlab==3.0.17 -notebook==6.4.12 -# Detect notebooks written in MyST Markdown -jupytext==1.13.4 -# Dependencies for jupyter widgets -bqplot==0.12.25 -ipytree==0.1.8 -ipywidgets-extended==1.0.5 -nglview==2.7.7 -widget-periodictable~=3.0 -# Used for exposing AiiDA REST API to the outside world -jupyter-server-proxy==3.2.1 -# Install voila package and AiiDAlab voila template. -voila==0.2.10 -voila-aiidalab-template==0.2.1 diff --git a/requirements.txt b/requirements.txt index aee1d1f6..f93ca67f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,2 @@ -aiidalab==22.7.1 -aiidalab-widgets-base~=1.3.4 -binaryornot~=0.4 -bokeh~=2.0 -bqplot~=0.12 -cookiecutter~=1.6 -markdown~=3.1 -pip~=22.0,<22.1 -pysmiles~=1.0 -pythreejs~=2.1 -widget-periodictable~=3.0 +aiida-core>=2.0.0 +pip==22.0.4 diff --git a/service/jupyter-notebook b/service/jupyter-notebook deleted file mode 100755 index ae758a7a..00000000 --- a/service/jupyter-notebook +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -em - -su -c /opt/start-notebook.sh ${SYSTEM_USER} From fd1fc00d230a8da8cce9200fe19c5f45a0a24e14 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 5 Aug 2022 15:13:47 +0200 Subject: [PATCH 02/16] Add black to pre-commit hooks. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6c86134..d959ae40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,8 @@ repos: rev: 0.17.0 hooks: - id: check-github-workflows + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black From 667fda5686046bf6e8933ed868974fb66b495907 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 5 Aug 2022 14:35:52 +0200 Subject: [PATCH 03/16] Implement tests. --- .github/workflows/build_and_test_on_pr.yml | 35 ++++------------ docker-compose.yml | 42 ++++++++++++++++++++ tests/conftest.py | 46 ++++++++++++++++++++++ tests/docker-compose.yml | 1 + tests/requirements.txt | 4 ++ tests/test_aiidalab.py | 26 ++++++++++++ 6 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 docker-compose.yml create mode 100644 tests/conftest.py create mode 120000 tests/docker-compose.yml create mode 100644 tests/requirements.txt create mode 100644 tests/test_aiidalab.py diff --git a/.github/workflows/build_and_test_on_pr.yml b/.github/workflows/build_and_test_on_pr.yml index c852e0b2..32e74b07 100644 --- a/.github/workflows/build_and_test_on_pr.yml +++ b/.github/workflows/build_and_test_on_pr.yml @@ -40,32 +40,11 @@ jobs: tags: aiidalab-docker-stack:latest cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - - name: Start and test the container - id: test_run + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: pip # caching pip dependencies + - run: pip install -r tests/requirements.txt + - name: Run tests run: | - mkdir tmp - export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest` - echo "::set-output name=docker_id_first_run::${DOCKERID}" - docker exec --tty --user root $DOCKERID wait-for-services - docker exec --tty --user aiida $DOCKERID wait-for-services -<<<<<<< HEAD - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/opt/conda/envs/pgsql/bin/pg_ctl -D /home/$SYSTEM_USER/.postgresql status' # Check that postgres is up. - docker exec --tty --user root $DOCKERID /bin/bash -l -c '/opt/conda/envs/rmq/bin/rabbitmqctl status' # Check that rabbitmq is up. -======= - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/usr/lib/postgresql/10/bin/pg_ctl -D /home/$NB_USER/.postgresql status' # Check that postgres is up. - docker exec --tty --user root $DOCKERID /bin/bash -l -c 'service rabbitmq-server status' # Check that rabbitmq is up. ->>>>>>> ee7ece1 (Major revision to base the image on the jupyter docker stack.) - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda create -y -n test_env python=3.8' # Check that one can create a new conda environment. - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda activate test_env' # Check that new environment works. - sudo cp tmp/.ssh/id_rsa . # Copy id_rsa file from the mounted folder. - docker stop $DOCKERID # Stop the container. - export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest` # Start a new container using the same mounted folder. - echo "::set-output name=docker_id_second_run::${DOCKERID}" - docker exec --tty $DOCKERID wait-for-services - sudo diff id_rsa tmp/.ssh/id_rsa # Check that the id_rsa file wasn't modified. - - name: Show the container log (first run). - if: always() - run: docker logs "${{ steps.test_run.outputs.docker_id_first_run }}" - - name: Show the container log (second run). - if: always() - run: docker logs "${{ steps.test_run.outputs.docker_id_second_run }}" + pytest -v diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..819bc394 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,42 @@ +--- +version: '3.4' + +services: + + database: + image: postgres:12.3 + environment: + POSTGRES_USER: pguser + POSTGRES_PASSWORD: password + volumes: + - aiida-postgres-db:/var/lib/postgresql/data + + messaging: + image: rabbitmq:3.8.3-management + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: + - aiida-rmq-data:/var/lib/rabbitmq/ + + aiidalab: + build: .. + environment: + RMQHOST: messaging + TZ: Europe/Zurich + DOCKER_STACKS_JUPYTER_CMD: notebook + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + AIIDALAB_DEFAULT_APPS: '' + volumes: + - aiidalab-home-folder:/home/jovyan + depends_on: + - database + - messaging + ports: + - 8888 + + +volumes: + aiida-postgres-db: + aiida-rmq-data: + aiidalab-home-folder: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..5be857f1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +import pytest +import requests + +from requests.exceptions import ConnectionError + + +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code == 200: + return True + except ConnectionError: + return False + + +@pytest.fixture(scope="session") +def notebook_service(docker_ip, docker_services): + """Ensure that HTTP service is up and responsive.""" + port = docker_services.port_for("aiidalab", 8888) + url = f"http://{docker_ip}:{port}" + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive(url) + ) + return url + + +@pytest.fixture(scope="session") +def docker_compose(docker_services): + return docker_services._docker_compose + + +@pytest.fixture +def aiidalab_exec(docker_compose): + def execute(command, user=None, **kwargs): + if user: + command = f"exec -T --user={user} aiidalab {command}" + else: + command = f"exec -T aiidalab {command}" + return docker_compose.execute(command, **kwargs) + + return execute + + +@pytest.fixture +def nb_user(aiidalab_exec): + return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip() diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 120000 index 00000000..5c8318ef --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1 @@ +../docker-compose.yml \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..ade94cbc --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +docker-compose==1.29.2 +pytest==7.1.2 +pytest-docker==1.0.0 +requests==2.28.1 diff --git a/tests/test_aiidalab.py b/tests/test_aiidalab.py new file mode 100644 index 00000000..14ad7f86 --- /dev/null +++ b/tests/test_aiidalab.py @@ -0,0 +1,26 @@ +import requests + + +def test_notebook_service_available(notebook_service): + response = requests.get(f"{notebook_service}/") + assert response.status_code == 200 + + +def test_pip_check(aiidalab_exec): + aiidalab_exec("pip check") + + +def test_aiidalab_available(aiidalab_exec, nb_user): + output = aiidalab_exec("aiidalab --version", user=nb_user).decode().strip().lower() + assert "aiidalab" in output + + +def test_create_conda_environment(aiidalab_exec, nb_user): + output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip() + assert "conda activate tmp" in output + + +def test_verdi_status(aiidalab_exec, nb_user): + output = aiidalab_exec("verdi status", user=nb_user).decode().strip() + assert "Connected to RabbitMQ" in output + assert "Daemon is running" in output From e80b7a3ba95ee07f97c9ffa88c6055dcdd5cba99 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 18 Aug 2022 19:36:00 +0200 Subject: [PATCH 04/16] Refactor into image stack. --- .github/workflows/build_and_test_on_pr.yml | 52 +++++++++----- .github/workflows/release_image.yml | 58 ++++++++++++---- .gitignore | 1 + .pre-commit-config.yaml | 1 + README.md | 26 +++++-- build.py | 69 +++++++++++++++++++ build.yml | 6 ++ bumpver.toml | 21 ++++++ docker-compose.yml | 3 +- dodo.py | 45 ++++++++++++ environment.yml | 13 ++++ stack/base/Dockerfile | 51 ++++++++++++++ .../base/before-notebook.d}/prepare-aiida.sh | 0 .../base/config-quick-setup.yaml | 0 .../base/requirements.txt | 0 Dockerfile => stack/lab/Dockerfile | 43 ++---------- .../lab/before-notebook.d}/factory_reset.sh | 0 .../before-notebook.d}/prepare-aiidalab.sh | 0 gears.svg => stack/lab/gears.svg | 0 19 files changed, 309 insertions(+), 80 deletions(-) create mode 100755 build.py create mode 100644 build.yml create mode 100644 bumpver.toml create mode 100644 dodo.py create mode 100644 environment.yml create mode 100644 stack/base/Dockerfile rename {before-notebook.d => stack/base/before-notebook.d}/prepare-aiida.sh (100%) rename config-quick-setup.yaml => stack/base/config-quick-setup.yaml (100%) rename requirements.txt => stack/base/requirements.txt (100%) rename Dockerfile => stack/lab/Dockerfile (63%) rename {before-notebook.d => stack/lab/before-notebook.d}/factory_reset.sh (100%) rename {before-notebook.d => stack/lab/before-notebook.d}/prepare-aiidalab.sh (100%) rename gears.svg => stack/lab/gears.svg (100%) diff --git a/.github/workflows/build_and_test_on_pr.yml b/.github/workflows/build_and_test_on_pr.yml index 32e74b07..10180dcf 100644 --- a/.github/workflows/build_and_test_on_pr.yml +++ b/.github/workflows/build_and_test_on_pr.yml @@ -8,7 +8,7 @@ # 4. Copy id_rsa file from the docker container to local folder. # 5. Restart the container and check that the id_rsa file didn't change. -name: build-and-test-image-from-pull-request +name: build-and-test on: [pull_request] @@ -17,34 +17,52 @@ jobs: build-and-test: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 15 + + services: + registry: + image: registry:2 + ports: + - 5000:5000 steps: - uses: actions/checkout@v2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - name: Obtain Docker build args + id: meta_extra + run: | + echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)" + - name: Build base (AiiDA) image + id: build_base_image + uses: docker/build-push-action@v3 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Build image locally - uses: docker/build-push-action@v2 + context: stack/base + tags: localhost:5000/aiidalab/base:latest + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + push: true + - name: Build lab image + id: build_lab_image + uses: docker/build-push-action@v3 with: - load: true - push: false - tags: aiidalab-docker-stack:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache + context: stack/lab + tags: localhost:5000/aiidalab/lab:latest + push: true + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + BASE_IMAGE=localhost:5000/aiidalab/base@${{ steps.build_base_image.outputs.digest }} - uses: actions/setup-python@v4 with: python-version: '3.10' cache: pip # caching pip dependencies - run: pip install -r tests/requirements.txt - name: Run tests + env: + AIIDALAB_IMAGE: 'localhost:5000/aiidalab/lab@${{ steps.build_lab_image.outputs.digest }}' run: | pytest -v diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml index cddb7bcd..73de6c36 100644 --- a/.github/workflows/release_image.yml +++ b/.github/workflows/release_image.yml @@ -16,31 +16,59 @@ jobs: build-docker-image: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 15 steps: - uses: actions/checkout@v2 - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ github.repository }} - tags: | - type=ref,event=branch - type=pep440,pattern={{version}} + - name: Install Conda environment from environment.yml + uses: mamba-org/provision-with-micromamba@v12 + - name: Docker meta exta + id: meta_extra + run: | + echo "::set-output name=tags::$(./build.py tags --github-actions)" + echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 + - name: Docker meta base + id: meta_base + uses: docker/metadata-action@v4 + with: + images: aiidalab/base + tags: | + type=ref,event=branch + ${{ steps.meta_extra.outputs.tags }} + - name: Build and push base image + id: build_base_image + uses: docker/build-push-action@v3 + with: + context: stack/base + push: true + tags: ${{ steps.meta_base.outputs.tags }} + platforms: linux/amd64, linux/arm64 + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + - name: Docker meta lab + id: meta_lab + uses: docker/metadata-action@v4 + with: + images: aiidalab/lab + tags: | + type=ref,event=branch + ${{ steps.meta_extra.outputs.tags }} + - name: Build and push lab image + uses: docker/build-push-action@v3 with: + context: stack/lab push: true + tags: ${{ steps.meta_lab.outputs.tags }} platforms: linux/amd64, linux/arm64 - tags: ${{ steps.meta.outputs.tags }} + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + BASE_IMAGE=aiidalab/base@${{ steps.build_base_image.outputs.digest }} diff --git a/.gitignore b/.gitignore index b2f83705..2c29dd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -275,3 +275,4 @@ submit_test # Custom. Pipfile.lock +.doit* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d959ae40..71e3f849 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: rev: 0.2.2 hooks: - id: yamlfmt + args: [--preserve-quotes] - repo: https://github.com/sirosen/check-jsonschema rev: 0.17.0 diff --git a/README.md b/README.md index ec438c65..d61f3420 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,32 @@ # Docker Stack for AiiDAlab -This repository contains the Dockerfile for the official AiiDAlab docker images. +This repository contains the Dockerfiles for the official AiiDAlab docker images. -Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/aiidalab-docker-stack with the following tags: +Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/ with the following tags: - `latest` – the latest tagged release. - `` – a specific tagged release, example: `21.12.0`. - `master`/`develop` – the latest commit on the corresponding branches with the same name. -## Get started +## Build images locally -### Local deployment +To build the images locally, setup a build end testing environment with [conda](https://docs.conda.io/en/latest/miniconda.html) (or [mamba](https://mamba.readthedocs.io/en/latest/installation.html)): -To run AiiDAlab on your own workstation or laptop you can either -- run the image directly with: `docker run aiidalab-docker-stack -p 8888:8888`, or -- _(recommended)_ use the `aiidalab-launch` tool which is a thin docker wrapper. +```console +conda env create -f environment.yml +``` + +Then activate the environment with +```console +conda activate aiidalab-docker-stack +``` + +To build the images, run `doit build`. = +You can then run automated tests with `doit tests`. + +For local testing, you can start the images with `doit up`, however please refer to the next section for a production-ready local deployment of AiiDAlab with aiidalab-launch. + +## Run AiiDAlab in production The `aiidalab-launch` tool provides a convenient and robust method of both launching and managing one or multiple AiiDAlab instances on your computer. To use it, simply install it via pip diff --git a/build.py b/build.py new file mode 100755 index 00000000..0751533f --- /dev/null +++ b/build.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +import os +from pathlib import Path + +import click +from yaml import SafeLoader, load + +BUILD_CONFIG = load(Path("build.yml").read_text(), Loader=SafeLoader) + + +def get_organization(): + return BUILD_CONFIG["organization"] + + +def get_version(): + return BUILD_CONFIG["version"] + + +def get_tags(): + yield BUILD_CONFIG["version"] # The version of the stack. + # The versions of dependencies: + for name, version in BUILD_CONFIG["versions"].items(): + yield f"{name}-{version}" + + +def get_docker_build_args(): + yield f"VERSION={BUILD_CONFIG['version']}" + for name, version in BUILD_CONFIG["versions"].items(): + yield f"{name.upper()}_VERSION={version}" + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option( + "--github-actions", + is_flag=True, + help="Output tags in a format convenient for GitHub actions.", +) +def tags(github_actions): + if github_actions: + enable = str(os.environ.get("GITHUB_REF_TYPE", None) == "tag").lower() + click.echo( + r"%0A".join( + f"type=raw,enable={enable},event=tag,value={tag}" for tag in get_tags() + ) + ) + else: + click.echo("\n".join(get_tags())) + + +@cli.command() +@click.option( + "--github-actions", + is_flag=True, + help="Output tags in a format convenient for GitHub actions.", +) +def docker_build_args(github_actions): + if github_actions: + click.echo(r"%0A".join(get_docker_build_args())) + else: + click.echo(" ".join(f"--build-arg {arg}" for arg in get_docker_build_args())) + + +if __name__ == "__main__": + cli() diff --git a/build.yml b/build.yml new file mode 100644 index 00000000..7bc37f32 --- /dev/null +++ b/build.yml @@ -0,0 +1,6 @@ +--- +organization: aiidalab +version: '2022.1001' +versions: + python: '3.9.4' + aiida: '2.0.0' diff --git a/bumpver.toml b/bumpver.toml new file mode 100644 index 00000000..f05eb065 --- /dev/null +++ b/bumpver.toml @@ -0,0 +1,21 @@ +[bumpver] +current_version = "2022.1001" +version_pattern = "YYYY.BUILD[-TAG]" +commit_message = "Bump version {old_version} -> {new_version}." +commit = true +tag = false +push = false + +[bumpver.file_patterns] +"build.yml" = [ + "version: '{version}'" +] +"bumpver.toml" = [ + 'current_version = "{version}"', +] +"docker-compose.yml" = [ + "aiidalab/lab:{version}" +] +"stack/lab/Dockerfile" = [ + 'ARG VERSION={version}' +] diff --git a/docker-compose.yml b/docker-compose.yml index 819bc394..18842db6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiidalab: - build: .. + image: ${AIIDALAB_IMAGE:-aiidalab/lab:2022.1001} environment: RMQHOST: messaging TZ: Europe/Zurich @@ -35,7 +35,6 @@ services: ports: - 8888 - volumes: aiida-postgres-db: aiida-rmq-data: diff --git a/dodo.py b/dodo.py new file mode 100644 index 00000000..fadb3b26 --- /dev/null +++ b/dodo.py @@ -0,0 +1,45 @@ +from pathlib import Path + +from build import get_docker_build_args, get_organization, get_tags, get_version + +DOIT_CONFIG = {"default_tasks": ["build"]} + + +def task_build(): + """Build all docker images.""" + + contexts = [p.parent for p in sorted(Path("stack").glob("*/Dockerfile"))] + organization = get_organization() + version = get_version() # The version of the stack. + + for context in contexts: + image = f"{organization}/{context.name}" + + build_action = ["docker", "build"] + build_action.extend([f"-t {image}:{tag}" for tag in get_tags()]) + build_action.extend(f"--build-arg {arg}" for arg in get_docker_build_args()) + build_action.append(str(context)) + build_action = " ".join(build_action) + + deps = ["build.yml", "build.py"] + [ + p for p in context.glob("**/*") if p.is_file() + ] + + yield { + "name": f"{image}:{version}", + "actions": [build_action], + "file_dep": deps, + "verbosity": 2, + } + + +def task_tests(): + """Run tests with pytest.""" + + return {"actions": ["pytest"]} + + +def task_up(): + """Start AiiDAlab server for testing.""" + + return {"actions": ["docker-compose up --detach"], "verbosity": 2} diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..fdde6d71 --- /dev/null +++ b/environment.yml @@ -0,0 +1,13 @@ +--- +name: aiidalab-docker-stack +channels: + - conda-forge +dependencies: + - bumpver=2022.1118 + - docker-compose=1.29.2 + - doit=0.36.0 + - pip=22.2.2 + - pytest=7.1.2 + - pip: + - pytest-docker==1.0.0 +prefix: /home/sadorf/miniconda3/envs/aiidalab-docker-stack diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile new file mode 100644 index 00000000..1af8ea59 --- /dev/null +++ b/stack/base/Dockerfile @@ -0,0 +1,51 @@ +ARG PYTHON_VERSION=3.9.4 +FROM jupyter/minimal-notebook:python-${PYTHON_VERSION} + +LABEL maintainer="AiiDAlab Team " + +USER root +WORKDIR /opt/ + +ARG AIIDA_VERSION=2.0.0 + +# Install the shared requirements. +COPY requirements.txt . +RUN mamba install --yes \ + aiida-core=${AIIDA_VERSION} \ + --file requirements.txt \ + && mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + + +# Pin shared requirements in the base environemnt. +RUN cat requirements.txt | xargs -I{} conda config --system --add pinned_packages {} + +# Configure pip to use requirements file as constraints file. +ENV PIP_CONSTRAINT=/opt/requirements.txt + +# Enable verdi autocompletion. +RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ + echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + fix-permissions "${CONDA_DIR}" + +# Configure AiiDA profile. +COPY config-quick-setup.yaml . +COPY before-notebook.d/prepare-aiida.sh /usr/local/bin/before-notebook.d/ + +# Configure AiiDA. +ENV SETUP_DEFAULT_AIIDA_PROFILE true +ENV AIIDA_PROFILE_NAME default +ENV AIIDA_USER_EMAIL aiida@localhost +ENV AIIDA_USER_FIRST_NAME Giuseppe +ENV AIIDA_USER_LAST_NAME Verdi +ENV AIIDA_USER_INSTITUTION Khedivial + +USER ${NB_USER} + +WORKDIR "/home/${NB_USER}" + +# Make sure that the known_hosts file is present inside the .ssh folder. +RUN mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ + touch /home/${NB_USER}/.ssh/known_hosts diff --git a/before-notebook.d/prepare-aiida.sh b/stack/base/before-notebook.d/prepare-aiida.sh similarity index 100% rename from before-notebook.d/prepare-aiida.sh rename to stack/base/before-notebook.d/prepare-aiida.sh diff --git a/config-quick-setup.yaml b/stack/base/config-quick-setup.yaml similarity index 100% rename from config-quick-setup.yaml rename to stack/base/config-quick-setup.yaml diff --git a/requirements.txt b/stack/base/requirements.txt similarity index 100% rename from requirements.txt rename to stack/base/requirements.txt diff --git a/Dockerfile b/stack/lab/Dockerfile similarity index 63% rename from Dockerfile rename to stack/lab/Dockerfile index 1b4fcf68..c8eede58 100644 --- a/Dockerfile +++ b/stack/lab/Dockerfile @@ -1,24 +1,12 @@ -FROM jupyter/minimal-notebook:python-3.9.12 +ARG VERSION=2022.1001 +ARG BASE_IMAGE=aiidalab/base:${VERSION} +FROM ${BASE_IMAGE} LABEL maintainer="AiiDAlab Team " USER root WORKDIR /opt/ -# Install the shared requirements. -COPY requirements.txt . -RUN mamba install --yes \ - --file requirements.txt && \ - mamba clean -all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Pin shared requirements in the base environemnt. -RUN cat requirements.txt | xargs -I{} conda config --system --add pinned_packages {} - -# Configure pip to use requirements file as constraints file. -ENV PIP_CONSTRAINT=/opt/requirements.txt - # TODO: should be converted to a conda package # ARG aiidalab_version=aiida-2.0 RUN pip install --quiet --no-cache-dir \ @@ -29,7 +17,7 @@ RUN pip install --quiet --no-cache-dir \ fix-permissions "/home/${NB_USER}" # Install the aiidalab-home app. -ARG aiidalab_home_version=develop +ARG aiidalab_home_version=v22.08.0 RUN git clone https://github.com/aiidalab/aiidalab-home && \ cd aiidalab-home && \ git checkout "${aiidalab_home_version}" && \ @@ -47,30 +35,12 @@ RUN pip install ./appmode --no-cache-dir && \ jupyter nbextension enable --py --sys-prefix appmode && \ jupyter serverextension enable --py --sys-prefix appmode -# Enable verdi autocompletion. -RUN echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ - chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ - fix-permissions "${CONDA_DIR}" - -# Configure AiiDA profile. -COPY config-quick-setup.yaml . -COPY before-notebook.d/prepare-aiida.sh /usr/local/bin/before-notebook.d/ - # Perform factory reset if needed. COPY before-notebook.d/factory_reset.sh /usr/local/bin/before-notebook.d/ # Prepare user's folders for AiiDAlab launch. COPY before-notebook.d/prepare-aiidalab.sh /usr/local/bin/before-notebook.d/ -# Configure AiiDA. -ENV SETUP_DEFAULT_AIIDA_PROFILE true -ENV AIIDA_PROFILE_NAME default -ENV AIIDA_USER_EMAIL aiida@localhost -ENV AIIDA_USER_FIRST_NAME Giuseppe -ENV AIIDA_USER_LAST_NAME Verdi -ENV AIIDA_USER_INSTITUTION Khedivial - - # Configure AiiDAlab environment. ENV AIIDALAB_HOME /home/${NB_USER} ENV AIIDALAB_APPS ${AIIDALAB_HOME}/apps @@ -96,11 +66,6 @@ USER ${NB_USER} WORKDIR "/home/${NB_USER}" -# Make sure that the known_hosts file is present inside the .ssh folder. -RUN mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ - touch /home/${NB_USER}/.ssh/known_hosts - -# Create apps folder. RUN mkdir -p /home/${NB_USER}/apps ENV NOTEBOOK_ARGS \ diff --git a/before-notebook.d/factory_reset.sh b/stack/lab/before-notebook.d/factory_reset.sh similarity index 100% rename from before-notebook.d/factory_reset.sh rename to stack/lab/before-notebook.d/factory_reset.sh diff --git a/before-notebook.d/prepare-aiidalab.sh b/stack/lab/before-notebook.d/prepare-aiidalab.sh similarity index 100% rename from before-notebook.d/prepare-aiidalab.sh rename to stack/lab/before-notebook.d/prepare-aiidalab.sh diff --git a/gears.svg b/stack/lab/gears.svg similarity index 100% rename from gears.svg rename to stack/lab/gears.svg From 02ca1bb7dc34f616d428479f1941efb6e11a2c83 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 18 Aug 2022 19:41:57 +0200 Subject: [PATCH 05/16] Install aiidalab from conda-forge. --- stack/lab/Dockerfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index c8eede58..fd5049dd 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -7,12 +7,10 @@ LABEL maintainer="AiiDAlab Team " USER root WORKDIR /opt/ -# TODO: should be converted to a conda package -# ARG aiidalab_version=aiida-2.0 -RUN pip install --quiet --no-cache-dir \ - # TODO: switch to release version - "aiidalab@git+https://github.com/aiidalab/aiidalab@main" && \ - # "aiidalab==${aiidalab_version}" && \ +ARG AIIDALAB_VERSION=22.08.0 +RUN mamba install --yes \ + aiidalab=${AIIDALAB_VERSION} \ + && mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" From 68770bbff0ba75be53744587c0571a48be70e3e8 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 07:21:22 +0200 Subject: [PATCH 06/16] Expose AiiDAlab on host for doit up task. Port is configurable via a command line argument ('-p' / '--port). --- docker-compose.yml | 2 +- dodo.py | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 18842db6..70ed4694 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: - database - messaging ports: - - 8888 + - "0.0.0.0:${AIIDALAB_PORT:-}:8888" volumes: aiida-postgres-db: diff --git a/dodo.py b/dodo.py index fadb3b26..76f01d63 100644 --- a/dodo.py +++ b/dodo.py @@ -41,5 +41,22 @@ def task_tests(): def task_up(): """Start AiiDAlab server for testing.""" - - return {"actions": ["docker-compose up --detach"], "verbosity": 2} + return { + "actions": ["AIIDALAB_PORT=%(port)i docker-compose up --detach"], + "params": [ + { + "name": "port", + "short": "p", + "long": "port", + "type": int, + "default": 8888, + "help": "Specify the AiiDAlab host port.", + }, + ], + "verbosity": 2, + } + + +def task_down(): + """Stop AiiDAlab server.""" + return {"actions": ["docker-compose down"], "verbosity": 2} From 716dbcfdb801c03e83b80a70ab5a9265a9d860da Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 08:26:25 +0200 Subject: [PATCH 07/16] Pin the installed aiidalab version. --- stack/lab/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index fd5049dd..0c237274 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -14,6 +14,10 @@ RUN mamba install --yes \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" +# Pin aiidalab version. +RUN echo "aiidalab==${AIIDALAB_VERSION}" >> /opt/requirements.txt +RUN conda config --system --add pinned_packages "aiidalab=${AIIDALAB_VERSION}" + # Install the aiidalab-home app. ARG aiidalab_home_version=v22.08.0 RUN git clone https://github.com/aiidalab/aiidalab-home && \ From adec7769b44ac18dd5106663f5de8dfdbc6bba80 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 08:26:38 +0200 Subject: [PATCH 08/16] Start home app by default. Using an older jupyter stack version we cannot use the ENV_NOTEBOOK_ARGS environment variable to configure the startup command. --- stack/lab/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index 0c237274..8850d97a 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -70,6 +70,9 @@ WORKDIR "/home/${NB_USER}" RUN mkdir -p /home/${NB_USER}/apps -ENV NOTEBOOK_ARGS \ - "--NotebookApp.default_url='/apps/apps/home/start.ipynb'" \ - "--ContentsManager.allow_hidden=True" +# Switch to NOTEBOOK_ARGS approach (see below) +# for newer jupyter docker stack versions. +RUN echo 'c.NotebookApp.default_url="/apps/apps/home/start.ipynb"' >> /etc/jupyter/jupyter_notebook_config.py +# ENV NOTEBOOK_ARGS \ + # "--NotebookApp.default_url='/apps/apps/home/start.ipynb'" \ + # "--ContentsManager.allow_hidden=True" From 256e7a39bb2fd5c90417dd3c7c391e208e39b437 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 10:51:01 +0200 Subject: [PATCH 09/16] Use consistent casing for Docker args. --- stack/lab/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index 8850d97a..318407f3 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -19,10 +19,10 @@ RUN echo "aiidalab==${AIIDALAB_VERSION}" >> /opt/requirements.txt RUN conda config --system --add pinned_packages "aiidalab=${AIIDALAB_VERSION}" # Install the aiidalab-home app. -ARG aiidalab_home_version=v22.08.0 +ARG AIIDALAB_HOME_VERSION=v22.08.0 RUN git clone https://github.com/aiidalab/aiidalab-home && \ cd aiidalab-home && \ - git checkout "${aiidalab_home_version}" && \ + git checkout "${AIIDALAB_HOME_VERSION}" && \ pip install --quiet --no-cache-dir "./" && \ fix-permissions "./" && \ fix-permissions "${CONDA_DIR}" && \ From f941537dac755b705b7ede6c886903d65b9bd560 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 11:29:39 +0200 Subject: [PATCH 10/16] Add tests for package pinning and constraints. --- tests/conftest.py | 13 +++++++++++++ tests/test_aiidalab.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5be857f1..a5a4dd28 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ +from pathlib import Path + import pytest import requests +from yaml import SafeLoader, load from requests.exceptions import ConnectionError @@ -44,3 +47,13 @@ def execute(command, user=None, **kwargs): @pytest.fixture def nb_user(aiidalab_exec): return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip() + + +@pytest.fixture(scope="session") +def _build_config(): + return load(Path("build.yml").read_text(), Loader=SafeLoader) + + +@pytest.fixture(scope="session") +def aiida_version(_build_config): + return _build_config["versions"]["aiida"] diff --git a/tests/test_aiidalab.py b/tests/test_aiidalab.py index 14ad7f86..87d4be89 100644 --- a/tests/test_aiidalab.py +++ b/tests/test_aiidalab.py @@ -1,4 +1,7 @@ +import pytest import requests +import json +from packaging.version import parse def test_notebook_service_available(notebook_service): @@ -20,6 +23,39 @@ def test_create_conda_environment(aiidalab_exec, nb_user): assert "conda activate tmp" in output +def test_correct_aiida_version_installed(aiidalab_exec, aiida_version): + info = json.loads(aiidalab_exec("mamba list --json aiida-core").decode())[0] + assert info["name"] == "aiida-core" + assert parse(info["version"]) == parse(aiida_version) + + +@pytest.mark.parametrize("package_manager", ["mamba", "pip"]) +@pytest.mark.parametrize("incompatible_version", ["1.6.3"]) +def test_prevent_installation_of_incompatible_aiida_version( + aiidalab_exec, nb_user, aiida_version, package_manager, incompatible_version +): + assert parse(aiida_version) != parse(incompatible_version) + # Expected to succeed: + aiidalab_exec( + f"{package_manager} install aiida-core=={aiida_version}", user=nb_user + ) + with pytest.raises(Exception): + aiidalab_exec( + f"{package_manager} install aiida-core={incompatible_version}", user=nb_user + ) + + +@pytest.mark.parametrize("package_manager", ["mamba", "pip"]) +@pytest.mark.parametrize("incompatible_version", ["22.7.1"]) +def test_prevent_installation_of_incompatible_aiidalab_version( + aiidalab_exec, nb_user, package_manager, incompatible_version +): + with pytest.raises(Exception): + aiidalab_exec( + f"{package_manager} install aiidalab={incompatible_version}", user=nb_user + ) + + def test_verdi_status(aiidalab_exec, nb_user): output = aiidalab_exec("verdi status", user=nb_user).decode().strip() assert "Connected to RabbitMQ" in output From c0b626e6b3578bee85ad1b8184e54a479ef06b57 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 11:32:45 +0200 Subject: [PATCH 11/16] Increase verbosity for 'doit tests'. --- dodo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dodo.py b/dodo.py index 76f01d63..48faaf38 100644 --- a/dodo.py +++ b/dodo.py @@ -36,7 +36,7 @@ def task_build(): def task_tests(): """Run tests with pytest.""" - return {"actions": ["pytest"]} + return {"actions": ["pytest -v"], "verbosity": 2} def task_up(): From 38f0e3edf6c7f8e09b352600541b7fb6a755cd16 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 11:40:04 +0200 Subject: [PATCH 12/16] Trigger CI. From 8267f680141636db82f8a40455098fe2133f022d Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Fri, 19 Aug 2022 16:02:27 +0200 Subject: [PATCH 13/16] Remove prefix from conda environment file. --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index fdde6d71..322f804c 100644 --- a/environment.yml +++ b/environment.yml @@ -10,4 +10,3 @@ dependencies: - pytest=7.1.2 - pip: - pytest-docker==1.0.0 -prefix: /home/sadorf/miniconda3/envs/aiidalab-docker-stack From 9ed064e043fe4a64fa6da7e0f4c48de391e079e5 Mon Sep 17 00:00:00 2001 From: Aliaksandr Yakutovich Date: Fri, 19 Aug 2022 16:08:50 +0200 Subject: [PATCH 14/16] Update stack/base/requirements.txt Co-authored-by: Carl Simon Adorf --- stack/base/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/base/requirements.txt b/stack/base/requirements.txt index f93ca67f..c4ecc4b0 100644 --- a/stack/base/requirements.txt +++ b/stack/base/requirements.txt @@ -1,2 +1,2 @@ -aiida-core>=2.0.0 +aiida-core>=2.0.0,<3 pip==22.0.4 From 342b22795b74b39edbc90bd1a520526f00db85c4 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 22 Aug 2022 14:14:22 +0200 Subject: [PATCH 15/16] Trigger build of entire stack on change to any of the images. Example: The lab image should be rebuilt when the base image is changed. No need to further declare depencies since the process is already optimized through docker layer caching. --- dodo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dodo.py b/dodo.py index 48faaf38..346b596a 100644 --- a/dodo.py +++ b/dodo.py @@ -12,6 +12,10 @@ def task_build(): organization = get_organization() version = get_version() # The version of the stack. + deps = ["build.yml", "build.py"] + [ + p for p in Path("stack").glob("**/*") if p.is_file() + ] + for context in contexts: image = f"{organization}/{context.name}" @@ -21,10 +25,6 @@ def task_build(): build_action.append(str(context)) build_action = " ".join(build_action) - deps = ["build.yml", "build.py"] + [ - p for p in context.glob("**/*") if p.is_file() - ] - yield { "name": f"{image}:{version}", "actions": [build_action], From 323bb51359a3b453be87afae33f5487744d76f94 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 22 Aug 2022 14:16:33 +0200 Subject: [PATCH 16/16] Move ssh configuration into base image. And install the ssh-singleagent.sh script. --- stack/base/Dockerfile | 16 +++++++++++----- stack/base/before-notebook.d/setup-ssh.sh | 15 +++++++++++++++ stack/lab/before-notebook.d/prepare-aiidalab.sh | 6 ------ 3 files changed, 26 insertions(+), 11 deletions(-) create mode 100755 stack/base/before-notebook.d/setup-ssh.sh diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile index 1af8ea59..975c880a 100644 --- a/stack/base/Dockerfile +++ b/stack/base/Dockerfile @@ -32,7 +32,7 @@ RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ # Configure AiiDA profile. COPY config-quick-setup.yaml . -COPY before-notebook.d/prepare-aiida.sh /usr/local/bin/before-notebook.d/ +COPY before-notebook.d/* /usr/local/bin/before-notebook.d/ # Configure AiiDA. ENV SETUP_DEFAULT_AIIDA_PROFILE true @@ -42,10 +42,16 @@ ENV AIIDA_USER_FIRST_NAME Giuseppe ENV AIIDA_USER_LAST_NAME Verdi ENV AIIDA_USER_INSTITUTION Khedivial +# Install the load-singlesshagent.sh script as described here: +# https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/howto/ssh.html#starting-the-ssh-agent +# The startup of this script is configured in the before-notebook.d/setup-ssh.sh file. +RUN wget --quiet --directory-prefix=/opt/bin/ \ + "https://aiida.readthedocs.io/projects/aiida-core/en/v${AIIDA_VERSION}/_downloads/4265ec5a42c3a3dba586dd460c0db95e/load-singlesshagent.sh" \ + && echo $'\n# Load singlesshagent on shell startup.\n\ +if [ -f /opt/bin/load-singlesshagent.sh ]; then\n\ + . /opt/bin/load-singlesshagent.sh\n\ +fi\n' >> "/home/${NB_USER}/.bashrc" + USER ${NB_USER} WORKDIR "/home/${NB_USER}" - -# Make sure that the known_hosts file is present inside the .ssh folder. -RUN mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ - touch /home/${NB_USER}/.ssh/known_hosts diff --git a/stack/base/before-notebook.d/setup-ssh.sh b/stack/base/before-notebook.d/setup-ssh.sh new file mode 100755 index 00000000..00bc4a02 --- /dev/null +++ b/stack/base/before-notebook.d/setup-ssh.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Make sure that the known_hosts file is present inside the .ssh folder. +mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ + touch /home/${NB_USER}/.ssh/known_hosts + + +if [[ ! -f /home/${NB_USER}/.ssh/id_rsa ]]; then + # Generate ssh key that works with `paramiko` + # See: https://aiida.readthedocs.io/projects/aiida-core/en/latest/get_started/computers.html#remote-computer-requirements + ssh-keygen -f /home/${NB_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' +fi + +# Start the ssh-agent. +eval `ssh-agent` diff --git a/stack/lab/before-notebook.d/prepare-aiidalab.sh b/stack/lab/before-notebook.d/prepare-aiidalab.sh index 66766e76..76f83cab 100755 --- a/stack/lab/before-notebook.d/prepare-aiidalab.sh +++ b/stack/lab/before-notebook.d/prepare-aiidalab.sh @@ -11,12 +11,6 @@ if [ -L /home/${NB_USER}/${NB_USER} ]; then rm /home/${NB_USER}/${NB_USER} fi -if [[ ! -f /home/${NB_USER}/.ssh/id_rsa ]]; then - # Generate ssh key that works with `paramiko` - # See: https://aiida.readthedocs.io/projects/aiida-core/en/latest/get_started/computers.html#remote-computer-requirements - ssh-keygen -f /home/${NB_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' -fi - # Install the home app. if [ ! -e /home/${NB_USER}/apps/home ]; then echo "Install home app."