diff --git a/.github/workflows/build_and_test_on_pr.yml b/.github/workflows/build_and_test_on_pr.yml index 4e1b510a..10180dcf 100644 --- a/.github/workflows/build_and_test_on_pr.yml +++ b/.github/workflows/build_and_test_on_pr.yml @@ -8,7 +8,7 @@ # 4. Copy id_rsa file from the docker container to local folder. # 5. Restart the container and check that the id_rsa file didn't change. -name: build-and-test-image-from-pull-request +name: build-and-test on: [pull_request] @@ -17,50 +17,52 @@ jobs: build-and-test: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 15 + + services: + registry: + image: registry:2 + ports: + - 5000:5000 steps: - uses: actions/checkout@v2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - name: Obtain Docker build args + id: meta_extra + run: | + echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)" + - name: Build base (AiiDA) image + id: build_base_image + uses: docker/build-push-action@v3 + with: + context: stack/base + tags: localhost:5000/aiidalab/base:latest + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + push: true + - name: Build lab image + id: build_lab_image + uses: docker/build-push-action@v3 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Build image locally - uses: docker/build-push-action@v2 + context: stack/lab + tags: localhost:5000/aiidalab/lab:latest + push: true + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + BASE_IMAGE=localhost:5000/aiidalab/base@${{ steps.build_base_image.outputs.digest }} + - uses: actions/setup-python@v4 with: - load: true - push: false - tags: aiidalab-docker-stack:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - name: Start and test the container - id: test_run + python-version: '3.10' + cache: pip # caching pip dependencies + - run: pip install -r tests/requirements.txt + - name: Run tests + env: + AIIDALAB_IMAGE: 'localhost:5000/aiidalab/lab@${{ steps.build_lab_image.outputs.digest }}' run: | - mkdir tmp - export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest` - echo "::set-output name=docker_id_first_run::${DOCKERID}" - docker exec --tty --user root $DOCKERID wait-for-services - docker exec --tty --user aiida $DOCKERID wait-for-services - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/opt/conda/envs/pgsql/bin/pg_ctl -D /home/$SYSTEM_USER/.postgresql status' # Check that postgres is up. - docker exec --tty --user root $DOCKERID /bin/bash -l -c '/opt/conda/envs/rmq/bin/rabbitmqctl status' # Check that rabbitmq is up. - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda create -y -n test_env python=3.8' # Check that one can create a new conda environment. - docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda activate test_env' # Check that new environment works. - sudo cp tmp/.ssh/id_rsa . # Copy id_rsa file from the mounted folder. - docker stop $DOCKERID # Stop the container. - export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest` # Start a new container using the same mounted folder. - echo "::set-output name=docker_id_second_run::${DOCKERID}" - docker exec --tty $DOCKERID wait-for-services - sudo diff id_rsa tmp/.ssh/id_rsa # Check that the id_rsa file wasn't modified. - - name: Show the container log (first run). - if: always() - run: docker logs "${{ steps.test_run.outputs.docker_id_first_run }}" - - name: Show the container log (second run). - if: always() - run: docker logs "${{ steps.test_run.outputs.docker_id_second_run }}" + pytest -v diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml index cddb7bcd..73de6c36 100644 --- a/.github/workflows/release_image.yml +++ b/.github/workflows/release_image.yml @@ -16,31 +16,59 @@ jobs: build-docker-image: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 15 steps: - uses: actions/checkout@v2 - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ github.repository }} - tags: | - type=ref,event=branch - type=pep440,pattern={{version}} + - name: Install Conda environment from environment.yml + uses: mamba-org/provision-with-micromamba@v12 + - name: Docker meta exta + id: meta_extra + run: | + echo "::set-output name=tags::$(./build.py tags --github-actions)" + echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 + - name: Docker meta base + id: meta_base + uses: docker/metadata-action@v4 + with: + images: aiidalab/base + tags: | + type=ref,event=branch + ${{ steps.meta_extra.outputs.tags }} + - name: Build and push base image + id: build_base_image + uses: docker/build-push-action@v3 + with: + context: stack/base + push: true + tags: ${{ steps.meta_base.outputs.tags }} + platforms: linux/amd64, linux/arm64 + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + - name: Docker meta lab + id: meta_lab + uses: docker/metadata-action@v4 + with: + images: aiidalab/lab + tags: | + type=ref,event=branch + ${{ steps.meta_extra.outputs.tags }} + - name: Build and push lab image + uses: docker/build-push-action@v3 with: + context: stack/lab push: true + tags: ${{ steps.meta_lab.outputs.tags }} platforms: linux/amd64, linux/arm64 - tags: ${{ steps.meta.outputs.tags }} + build-args: | + ${{ steps.meta_extra.outputs.build_args }} + BASE_IMAGE=aiidalab/base@${{ steps.build_base_image.outputs.digest }} diff --git a/.gitignore b/.gitignore index b2f83705..2c29dd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -275,3 +275,4 @@ submit_test # Custom. Pipfile.lock +.doit* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6c86134..71e3f849 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,14 @@ repos: rev: 0.2.2 hooks: - id: yamlfmt + args: [--preserve-quotes] - repo: https://github.com/sirosen/check-jsonschema rev: 0.17.0 hooks: - id: check-github-workflows + + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 2e2a554e..00000000 --- a/Dockerfile +++ /dev/null @@ -1,127 +0,0 @@ -FROM aiidateam/aiida-core:1.6.9 - -LABEL maintainer="AiiDAlab Team " - -# Specify default factory reset (not set): -ENV AIIDALAB_FACTORY_RESET "" - -# Configure environment. -ENV AIIDALAB_HOME /home/${SYSTEM_USER} -ENV AIIDALAB_APPS ${AIIDALAB_HOME}/apps -ENV AIIDALAB_DEFAULT_GIT_BRANCH master - -# Specify which apps to install in addition to the home app. The -# AIIDALAB_DEFAULT_APPS variable should be a whitespace-delimited variable -# where each entry must follow the specifier format used by `aiidalab install`. -# -# Example for setting the AIIDALAB_DEFAULT_APPS variable: -# -# AIIDALAB_DEFAULT_APPS="aiidalab-widgets-base quantum-espresso==20.12.0" -# -# Please note that multiple entries must be whitespace delimited. -# Please see `aiidalab install --help` for more information. -ENV AIIDALAB_DEFAULT_APPS "aiidalab-widgets-base~=1.0" - -USER root -WORKDIR /opt/ - -# Install OS dependencies. -# Not clear whether libssl-dev and libffi-dev are still needed. -# povray needed for structure editor widget. -RUN apt-get update && apt-get install -y \ - ca-certificates \ - file \ - libssl-dev \ - libffi-dev \ - povray \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Dependencies needed for Jupyter Lab. -RUN apt-get update && apt-get install -y \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -# Install ngrok to be able to proxy AiiDA RESTful API server. -# Currently not used by the home app, but used in tutorials -RUN wget --quiet -P /tmp/ \ - https://bin.equinox.io/a/dnxFaDKQgP4/ngrok-2.3.35-linux-amd64.zip \ - && unzip /tmp/ngrok-2.3.35-linux-amd64.zip \ - && mv ./ngrok /usr/local/bin/ \ - && rm -f /tmp/ngrok-2.3.35-linux-amd64.zip - -# Get recent version of pip (needed for `pip cache` command). -# New pip executable is installed into /usr/local/bin -RUN /usr/bin/pip3 install --upgrade pip - -# Jupyter dependencies installed into system python environment -# which runs the jupyter notebook server. -COPY requirements-server.txt . -RUN /usr/local/bin/pip install -r /opt/requirements-server.txt \ - && /usr/local/bin/pip cache purge - -# Install and enable appmode. -RUN git clone https://github.com/oschuett/appmode.git && cd appmode && git reset --hard v0.8.0 -COPY gears.svg ./appmode/appmode/static/gears.svg -RUN /usr/local/bin/pip install ./appmode -RUN /usr/local/bin/jupyter nbextension enable --py --sys-prefix appmode -RUN /usr/local/bin/jupyter serverextension enable --py --sys-prefix appmode - -# Install jupyterlab theme (takes about 4 minutes and 10 seconds). -#WORKDIR /opt/jupyterlab-theme -#RUN git clone https://github.com/aiidalab/jupyterlab-theme && \ -# cd jupyterlab-theme && \ -# npm install && \ -# npm run build && \ -# npm run build:webpack && \ -# npm pack ./ && \ -# /usr/local/bin/jupyter labextension install *.tgz && \ -# cd .. - -## Configure user environment - -# Install some useful packages that are not available on PyPi. -RUN conda install --yes -c conda-forge \ - openbabel==3.1.1 \ - rdkit==2021.09.2 \ - && conda clean --all - -# Install AiiDAlab Python packages into user conda environment and populate reentry cache. -COPY requirements.txt . -ARG extra_requirements -RUN pip install --upgrade pip -RUN pip install -r requirements.txt $extra_requirements -RUN reentry scan - -# Configure pip to use requirements file as constraints file. -RUN conda env config vars set PIP_CONSTRAINT=/opt/requirements.txt - -# Install python kernel from the conda environment (comes with the aiidalab package). -RUN python -m ipykernel install - - -# Perform factory reset if needed. -COPY my_init.d/factory_reset.sh /etc/my_init.d/09_factory_reset.sh - -# Prepare user's folders for AiiDAlab launch. -COPY opt/aiidalab-singleuser /opt/ -COPY opt/prepare-aiidalab.sh /opt/ -COPY my_init.d/prepare-aiidalab.sh /etc/my_init.d/80_prepare-aiidalab.sh - -# Install the aiidalab-home app. -ARG aiidalab_home_version=v22.01.0 -RUN git clone https://github.com/aiidalab/aiidalab-home && cd aiidalab-home && git checkout $aiidalab_home_version -RUN chmod 774 aiidalab-home - -# Copy scripts to start Jupyter notebook. -COPY opt/start-notebook.sh /opt/ -COPY service/jupyter-notebook /etc/service/jupyter-notebook/run - -# Expose port 8888. -EXPOSE 8888 - -# Remove when the following issue is fixed: https://github.com/jupyterhub/dockerspawner/issues/319. -COPY my_my_init /sbin/my_my_init - -CMD ["/sbin/my_my_init"] diff --git a/README.md b/README.md index ec438c65..d61f3420 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,32 @@ # Docker Stack for AiiDAlab -This repository contains the Dockerfile for the official AiiDAlab docker images. +This repository contains the Dockerfiles for the official AiiDAlab docker images. -Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/aiidalab-docker-stack with the following tags: +Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/ with the following tags: - `latest` – the latest tagged release. - `` – a specific tagged release, example: `21.12.0`. - `master`/`develop` – the latest commit on the corresponding branches with the same name. -## Get started +## Build images locally -### Local deployment +To build the images locally, setup a build end testing environment with [conda](https://docs.conda.io/en/latest/miniconda.html) (or [mamba](https://mamba.readthedocs.io/en/latest/installation.html)): -To run AiiDAlab on your own workstation or laptop you can either -- run the image directly with: `docker run aiidalab-docker-stack -p 8888:8888`, or -- _(recommended)_ use the `aiidalab-launch` tool which is a thin docker wrapper. +```console +conda env create -f environment.yml +``` + +Then activate the environment with +```console +conda activate aiidalab-docker-stack +``` + +To build the images, run `doit build`. = +You can then run automated tests with `doit tests`. + +For local testing, you can start the images with `doit up`, however please refer to the next section for a production-ready local deployment of AiiDAlab with aiidalab-launch. + +## Run AiiDAlab in production The `aiidalab-launch` tool provides a convenient and robust method of both launching and managing one or multiple AiiDAlab instances on your computer. To use it, simply install it via pip diff --git a/build.py b/build.py new file mode 100755 index 00000000..0751533f --- /dev/null +++ b/build.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +import os +from pathlib import Path + +import click +from yaml import SafeLoader, load + +BUILD_CONFIG = load(Path("build.yml").read_text(), Loader=SafeLoader) + + +def get_organization(): + return BUILD_CONFIG["organization"] + + +def get_version(): + return BUILD_CONFIG["version"] + + +def get_tags(): + yield BUILD_CONFIG["version"] # The version of the stack. + # The versions of dependencies: + for name, version in BUILD_CONFIG["versions"].items(): + yield f"{name}-{version}" + + +def get_docker_build_args(): + yield f"VERSION={BUILD_CONFIG['version']}" + for name, version in BUILD_CONFIG["versions"].items(): + yield f"{name.upper()}_VERSION={version}" + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option( + "--github-actions", + is_flag=True, + help="Output tags in a format convenient for GitHub actions.", +) +def tags(github_actions): + if github_actions: + enable = str(os.environ.get("GITHUB_REF_TYPE", None) == "tag").lower() + click.echo( + r"%0A".join( + f"type=raw,enable={enable},event=tag,value={tag}" for tag in get_tags() + ) + ) + else: + click.echo("\n".join(get_tags())) + + +@cli.command() +@click.option( + "--github-actions", + is_flag=True, + help="Output tags in a format convenient for GitHub actions.", +) +def docker_build_args(github_actions): + if github_actions: + click.echo(r"%0A".join(get_docker_build_args())) + else: + click.echo(" ".join(f"--build-arg {arg}" for arg in get_docker_build_args())) + + +if __name__ == "__main__": + cli() diff --git a/build.yml b/build.yml new file mode 100644 index 00000000..7bc37f32 --- /dev/null +++ b/build.yml @@ -0,0 +1,6 @@ +--- +organization: aiidalab +version: '2022.1001' +versions: + python: '3.9.4' + aiida: '2.0.0' diff --git a/bumpver.toml b/bumpver.toml new file mode 100644 index 00000000..f05eb065 --- /dev/null +++ b/bumpver.toml @@ -0,0 +1,21 @@ +[bumpver] +current_version = "2022.1001" +version_pattern = "YYYY.BUILD[-TAG]" +commit_message = "Bump version {old_version} -> {new_version}." +commit = true +tag = false +push = false + +[bumpver.file_patterns] +"build.yml" = [ + "version: '{version}'" +] +"bumpver.toml" = [ + 'current_version = "{version}"', +] +"docker-compose.yml" = [ + "aiidalab/lab:{version}" +] +"stack/lab/Dockerfile" = [ + 'ARG VERSION={version}' +] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..70ed4694 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +--- +version: '3.4' + +services: + + database: + image: postgres:12.3 + environment: + POSTGRES_USER: pguser + POSTGRES_PASSWORD: password + volumes: + - aiida-postgres-db:/var/lib/postgresql/data + + messaging: + image: rabbitmq:3.8.3-management + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: + - aiida-rmq-data:/var/lib/rabbitmq/ + + aiidalab: + image: ${AIIDALAB_IMAGE:-aiidalab/lab:2022.1001} + environment: + RMQHOST: messaging + TZ: Europe/Zurich + DOCKER_STACKS_JUPYTER_CMD: notebook + SETUP_DEFAULT_AIIDA_PROFILE: 'true' + AIIDALAB_DEFAULT_APPS: '' + volumes: + - aiidalab-home-folder:/home/jovyan + depends_on: + - database + - messaging + ports: + - "0.0.0.0:${AIIDALAB_PORT:-}:8888" + +volumes: + aiida-postgres-db: + aiida-rmq-data: + aiidalab-home-folder: diff --git a/dodo.py b/dodo.py new file mode 100644 index 00000000..346b596a --- /dev/null +++ b/dodo.py @@ -0,0 +1,62 @@ +from pathlib import Path + +from build import get_docker_build_args, get_organization, get_tags, get_version + +DOIT_CONFIG = {"default_tasks": ["build"]} + + +def task_build(): + """Build all docker images.""" + + contexts = [p.parent for p in sorted(Path("stack").glob("*/Dockerfile"))] + organization = get_organization() + version = get_version() # The version of the stack. + + deps = ["build.yml", "build.py"] + [ + p for p in Path("stack").glob("**/*") if p.is_file() + ] + + for context in contexts: + image = f"{organization}/{context.name}" + + build_action = ["docker", "build"] + build_action.extend([f"-t {image}:{tag}" for tag in get_tags()]) + build_action.extend(f"--build-arg {arg}" for arg in get_docker_build_args()) + build_action.append(str(context)) + build_action = " ".join(build_action) + + yield { + "name": f"{image}:{version}", + "actions": [build_action], + "file_dep": deps, + "verbosity": 2, + } + + +def task_tests(): + """Run tests with pytest.""" + + return {"actions": ["pytest -v"], "verbosity": 2} + + +def task_up(): + """Start AiiDAlab server for testing.""" + return { + "actions": ["AIIDALAB_PORT=%(port)i docker-compose up --detach"], + "params": [ + { + "name": "port", + "short": "p", + "long": "port", + "type": int, + "default": 8888, + "help": "Specify the AiiDAlab host port.", + }, + ], + "verbosity": 2, + } + + +def task_down(): + """Stop AiiDAlab server.""" + return {"actions": ["docker-compose down"], "verbosity": 2} diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..322f804c --- /dev/null +++ b/environment.yml @@ -0,0 +1,12 @@ +--- +name: aiidalab-docker-stack +channels: + - conda-forge +dependencies: + - bumpver=2022.1118 + - docker-compose=1.29.2 + - doit=0.36.0 + - pip=22.2.2 + - pytest=7.1.2 + - pip: + - pytest-docker==1.0.0 diff --git a/my_init.d/prepare-aiidalab.sh b/my_init.d/prepare-aiidalab.sh deleted file mode 100755 index eb40bbff..00000000 --- a/my_init.d/prepare-aiidalab.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -em - -# Change group of the aiidalab-home folder -chown root:${SYSTEM_USER} -R /opt/aiidalab-home - -su -c /opt/prepare-aiidalab.sh ${SYSTEM_USER} diff --git a/my_my_init b/my_my_init deleted file mode 100755 index edcdc77f..00000000 --- a/my_my_init +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/sbin/my_init diff --git a/opt/aiidalab-singleuser b/opt/aiidalab-singleuser deleted file mode 100755 index fe3e4332..00000000 --- a/opt/aiidalab-singleuser +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -from jupyterhub import singleuser - -# https://github.com/jupyterhub/jupyterhub/blob/master/scripts/jupyterhub-singleuser -# https://github.com/jupyterhub/jupyterhub/blob/master/jupyterhub/singleuser.py - -matcloud_page_template = """ -{% extends "templates/page.html" %} -{% block header_buttons %} -{{super()}} - - - Control Panel - - - - - Materials Cloud - - -{% endblock %} -{% block logo %} -Jupyter Notebook -{% endblock logo %} -""" - -if __name__ == '__main__': - singleuser.page_template = matcloud_page_template - singleuser.main() diff --git a/opt/prepare-aiidalab.sh b/opt/prepare-aiidalab.sh deleted file mode 100755 index 2a3e086b..00000000 --- a/opt/prepare-aiidalab.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -e - -# Debugging. -set -x - -# Environment. -export SHELL=/bin/bash - -# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225 -if [ -L /home/${SYSTEM_USER}/${SYSTEM_USER} ]; then - rm /home/${SYSTEM_USER}/${SYSTEM_USER} -fi - -# Setup AiiDA jupyter extension. -# Don't forget to copy this file to .ipython/profile_default/startup/ -# aiida/tools/ipython/aiida_magic_register.py -if [ ! -e /home/${SYSTEM_USER}/.ipython/profile_default/startup/aiida_magic_register.py ]; then - mkdir -p /home/${SYSTEM_USER}/.ipython/profile_default/startup/ - cat << EOF > /home/${SYSTEM_USER}/.ipython/profile_default/startup/aiida_magic_register.py -if __name__ == "__main__": - - try: - import aiida - del aiida - except ImportError: - pass - else: - import IPython - # pylint: disable=ungrouped-imports - from aiida.tools.ipython.ipython_magics import load_ipython_extension - - # Get the current Ipython session - IPYSESSION = IPython.get_ipython() - - # Register the line magic - load_ipython_extension(IPYSESSION) -EOF -fi - -# Create apps folder and make its subfolders importable from Python. -if [ ! -e /home/${SYSTEM_USER}/apps ]; then - # Create apps folder and make it importable from python. - mkdir -p /home/${SYSTEM_USER}/apps - INITIAL_SETUP=1 -fi - -# Install the home app. -if [ ! -e /home/${SYSTEM_USER}/apps/home ]; then - echo "Install home app." - # The home app is installed in system space and linked to from user space. - # That ensures that users are not inadvertently running the wrong version of - # the home app for a given system environment, but still makes it possible to - # manually install a specific version of the home app in between upgrades, e.g., - # for development work, by simply replacing the link with a clone of the repository. - ln -s /opt/aiidalab-home /home/${SYSTEM_USER}/apps/home -elif [[ -d /home/${SYSTEM_USER}/apps/home && ! -L /home/${SYSTEM_USER}/apps/home ]]; then - # Backup an existing repository of the home app and replace with link to /opt/aiidalab-home. - # This mechanism preserves potential development work on a manually installed repository - # of the home app and also constitutes a migration path for existing aiidalab accounts, where - # the home app was installed directly into user space by default. - mv /home/${SYSTEM_USER}/apps/home /home/${SYSTEM_USER}/apps/.home~`date --iso-8601=seconds` \ - && ln -s /opt/aiidalab-home /home/${SYSTEM_USER}/apps/home || echo "WARNING: Unable to install home app." -fi - - -# Install default apps (see the Dockerfile for an explanation of the -# AIIDALAB_DEFAULT_APPS variable). -if [[ ${INITIAL_SETUP} == 1 ]]; then - - # Iterate over lines in AIIDALAB_DEFAULT_APPS variable. - for app in ${AIIDALAB_DEFAULT_APPS:-}; do - aiidalab install --yes "${app}" - done -fi - -# Update reentry. -reentry scan - -# Clear user trash directory. -if [ -e /home/${SYSTEM_USER}/.trash ]; then - rm -rf /home/${SYSTEM_USER}/.trash/* -fi - -# Remove old apps_meta.sqlite requests cache files. -find -L /home/${SYSTEM_USER} -maxdepth 3 -name apps_meta.sqlite -writable -delete - -# Remove old temporary notebook files. -find -L /home/${SYSTEM_USER}/apps -maxdepth 2 -type f -name .*.ipynb -writable -delete - -# Uninstall aiidalab from user packages (if present). -# Would otherwise interfere with the system package. -USER_AIIDALAB_PACKAGE="$(/opt/conda/bin/python -c 'import site; print(site.USER_SITE)')/aiidalab" -if [ -e ${USER_AIIDALAB_PACKAGE} ]; then - echo "Uninstall local installation of aiidalab package." - /opt/conda/bin/python -m pip uninstall --yes aiidalab -fi diff --git a/requirements-server.txt b/requirements-server.txt deleted file mode 100644 index 9a4e0733..00000000 --- a/requirements-server.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Dependencies for notebook server and interaction with JupyterHub -jupyterhub==1.5.0 -jupyterlab==3.0.17 -notebook==6.4.12 -# Detect notebooks written in MyST Markdown -jupytext==1.13.4 -# Dependencies for jupyter widgets -bqplot==0.12.25 -ipytree==0.1.8 -ipywidgets-extended==1.0.5 -nglview==2.7.7 -widget-periodictable~=3.0 -# Used for exposing AiiDA REST API to the outside world -jupyter-server-proxy==3.2.1 -# Install voila package and AiiDAlab voila template. -voila==0.2.10 -voila-aiidalab-template==0.2.1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index aee1d1f6..00000000 --- a/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -aiidalab==22.7.1 -aiidalab-widgets-base~=1.3.4 -binaryornot~=0.4 -bokeh~=2.0 -bqplot~=0.12 -cookiecutter~=1.6 -markdown~=3.1 -pip~=22.0,<22.1 -pysmiles~=1.0 -pythreejs~=2.1 -widget-periodictable~=3.0 diff --git a/service/jupyter-notebook b/service/jupyter-notebook deleted file mode 100755 index ae758a7a..00000000 --- a/service/jupyter-notebook +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -em - -su -c /opt/start-notebook.sh ${SYSTEM_USER} diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile new file mode 100644 index 00000000..975c880a --- /dev/null +++ b/stack/base/Dockerfile @@ -0,0 +1,57 @@ +ARG PYTHON_VERSION=3.9.4 +FROM jupyter/minimal-notebook:python-${PYTHON_VERSION} + +LABEL maintainer="AiiDAlab Team " + +USER root +WORKDIR /opt/ + +ARG AIIDA_VERSION=2.0.0 + +# Install the shared requirements. +COPY requirements.txt . +RUN mamba install --yes \ + aiida-core=${AIIDA_VERSION} \ + --file requirements.txt \ + && mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + + +# Pin shared requirements in the base environemnt. +RUN cat requirements.txt | xargs -I{} conda config --system --add pinned_packages {} + +# Configure pip to use requirements file as constraints file. +ENV PIP_CONSTRAINT=/opt/requirements.txt + +# Enable verdi autocompletion. +RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ + echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \ + fix-permissions "${CONDA_DIR}" + +# Configure AiiDA profile. +COPY config-quick-setup.yaml . +COPY before-notebook.d/* /usr/local/bin/before-notebook.d/ + +# Configure AiiDA. +ENV SETUP_DEFAULT_AIIDA_PROFILE true +ENV AIIDA_PROFILE_NAME default +ENV AIIDA_USER_EMAIL aiida@localhost +ENV AIIDA_USER_FIRST_NAME Giuseppe +ENV AIIDA_USER_LAST_NAME Verdi +ENV AIIDA_USER_INSTITUTION Khedivial + +# Install the load-singlesshagent.sh script as described here: +# https://aiida.readthedocs.io/projects/aiida-core/en/v2.0.0/howto/ssh.html#starting-the-ssh-agent +# The startup of this script is configured in the before-notebook.d/setup-ssh.sh file. +RUN wget --quiet --directory-prefix=/opt/bin/ \ + "https://aiida.readthedocs.io/projects/aiida-core/en/v${AIIDA_VERSION}/_downloads/4265ec5a42c3a3dba586dd460c0db95e/load-singlesshagent.sh" \ + && echo $'\n# Load singlesshagent on shell startup.\n\ +if [ -f /opt/bin/load-singlesshagent.sh ]; then\n\ + . /opt/bin/load-singlesshagent.sh\n\ +fi\n' >> "/home/${NB_USER}/.bashrc" + +USER ${NB_USER} + +WORKDIR "/home/${NB_USER}" diff --git a/stack/base/before-notebook.d/prepare-aiida.sh b/stack/base/before-notebook.d/prepare-aiida.sh new file mode 100755 index 00000000..0f5b91b8 --- /dev/null +++ b/stack/base/before-notebook.d/prepare-aiida.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# This script is executed whenever the docker container is (re)started. + +# Debugging. +set -x + +# Environment. +export SHELL=/bin/bash + +# Check if user requested to set up AiiDA profile (and if it exists already) +if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then + NEED_SETUP_PROFILE=true; +else + NEED_SETUP_PROFILE=false; +fi + +# Setup AiiDA profile if needed. +if [[ ${NEED_SETUP_PROFILE} == true ]]; then + + # Create AiiDA profile. + verdi quicksetup \ + --non-interactive \ + --profile "${AIIDA_PROFILE_NAME}" \ + --email "${AIIDA_USER_EMAIL}" \ + --first-name "${AIIDA_USER_FIRST_NAME}" \ + --last-name "${AIIDA_USER_LAST_NAME}" \ + --institution "${AIIDA_USER_INSTITUTION}" \ + --config /opt/config-quick-setup.yaml + + # Setup and configure local computer. + computer_name=localhost + + # Determine the number of physical cores as a default for the number of + # available MPI ranks on the localhost. We do not count "logical" cores, + # since MPI parallelization over hyper-threaded cores is typically + # associated with a significant performance penalty. We use the + # `psutil.cpu_count(logical=False)` function as opposed to simply + # `os.cpu_count()` since the latter would include hyperthreaded (logical + # cores). + NUM_PHYSICAL_CORES=$(python -c 'import psutil; print(int(psutil.cpu_count(logical=False)))' 2>/dev/null) + LOCALHOST_MPI_PROCS_PER_MACHINE=${LOCALHOST_MPI_PROCS_PER_MACHINE:-${NUM_PHYSICAL_CORES}} + + if [ -z $LOCALHOST_MPI_PROCS_PER_MACHINE ]; then + echo "Unable to automatically determine the number of logical CPUs on this " + echo "machine. Please set the LOCALHOST_MPI_PROCS_PER_MACHINE variable to " + echo "explicitly set the number of available MPI ranks." + exit 1 + fi + + verdi computer show ${computer_name} || verdi computer setup \ + --non-interactive \ + --label "${computer_name}" \ + --description "this computer" \ + --hostname "${computer_name}" \ + --transport local \ + --scheduler direct \ + --work-dir /home/aiida/aiida_run/ \ + --mpirun-command "mpirun -np {tot_num_mpiprocs}" \ + --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \ + verdi computer configure local "${computer_name}" \ + --non-interactive \ + --safe-interval 0.0 +fi + + +# Show the default profile +verdi profile show || echo "The default profile is not set." + +# Make sure that the daemon is not running, otherwise the migration will abort. +verdi daemon stop + +# Migration will run for the default profile. +verdi storage migrate --force + +# Daemon will start only if the database exists and is migrated to the latest version. +verdi daemon start || echo "AiiDA daemon is not running." diff --git a/stack/base/before-notebook.d/setup-ssh.sh b/stack/base/before-notebook.d/setup-ssh.sh new file mode 100755 index 00000000..00bc4a02 --- /dev/null +++ b/stack/base/before-notebook.d/setup-ssh.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Make sure that the known_hosts file is present inside the .ssh folder. +mkdir -p --mode=0700 /home/${NB_USER}/.ssh && \ + touch /home/${NB_USER}/.ssh/known_hosts + + +if [[ ! -f /home/${NB_USER}/.ssh/id_rsa ]]; then + # Generate ssh key that works with `paramiko` + # See: https://aiida.readthedocs.io/projects/aiida-core/en/latest/get_started/computers.html#remote-computer-requirements + ssh-keygen -f /home/${NB_USER}/.ssh/id_rsa -t rsa -b 4096 -m PEM -N '' +fi + +# Start the ssh-agent. +eval `ssh-agent` diff --git a/stack/base/config-quick-setup.yaml b/stack/base/config-quick-setup.yaml new file mode 100644 index 00000000..23d36d89 --- /dev/null +++ b/stack/base/config-quick-setup.yaml @@ -0,0 +1,15 @@ +--- +db_engine: postgresql_psycopg2 +db_backend: psql_dos +db_host: database +db_port: 5432 +su_db_username: pguser +su_db_password: password +su_db_name: template1 +db_name: aiida_db +db_username: aiida +db_password: password +broker_host: messaging +broker_port: 5672 +broker_username: guest +broker_password: guest diff --git a/stack/base/requirements.txt b/stack/base/requirements.txt new file mode 100644 index 00000000..c4ecc4b0 --- /dev/null +++ b/stack/base/requirements.txt @@ -0,0 +1,2 @@ +aiida-core>=2.0.0,<3 +pip==22.0.4 diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile new file mode 100644 index 00000000..318407f3 --- /dev/null +++ b/stack/lab/Dockerfile @@ -0,0 +1,78 @@ +ARG VERSION=2022.1001 +ARG BASE_IMAGE=aiidalab/base:${VERSION} +FROM ${BASE_IMAGE} + +LABEL maintainer="AiiDAlab Team " + +USER root +WORKDIR /opt/ + +ARG AIIDALAB_VERSION=22.08.0 +RUN mamba install --yes \ + aiidalab=${AIIDALAB_VERSION} \ + && mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Pin aiidalab version. +RUN echo "aiidalab==${AIIDALAB_VERSION}" >> /opt/requirements.txt +RUN conda config --system --add pinned_packages "aiidalab=${AIIDALAB_VERSION}" + +# Install the aiidalab-home app. +ARG AIIDALAB_HOME_VERSION=v22.08.0 +RUN git clone https://github.com/aiidalab/aiidalab-home && \ + cd aiidalab-home && \ + git checkout "${AIIDALAB_HOME_VERSION}" && \ + pip install --quiet --no-cache-dir "./" && \ + fix-permissions "./" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install and enable appmode. +RUN git clone https://github.com/oschuett/appmode.git && \ + cd appmode && \ + git checkout v0.8.0 +COPY gears.svg ./appmode/appmode/static/gears.svg +RUN pip install ./appmode --no-cache-dir && \ + jupyter nbextension enable --py --sys-prefix appmode && \ + jupyter serverextension enable --py --sys-prefix appmode + +# Perform factory reset if needed. +COPY before-notebook.d/factory_reset.sh /usr/local/bin/before-notebook.d/ + +# Prepare user's folders for AiiDAlab launch. +COPY before-notebook.d/prepare-aiidalab.sh /usr/local/bin/before-notebook.d/ + +# Configure AiiDAlab environment. +ENV AIIDALAB_HOME /home/${NB_USER} +ENV AIIDALAB_APPS ${AIIDALAB_HOME}/apps +ENV AIIDALAB_DEFAULT_GIT_BRANCH master + +# Specify which apps to install in addition to the home app. The +# AIIDALAB_DEFAULT_APPS variable should be a whitespace-delimited variable +# where each entry must follow the specifier format used by `aiidalab install`. +# +# Example for setting the AIIDALAB_DEFAULT_APPS variable: +# +# AIIDALAB_DEFAULT_APPS="aiidalab-widgets-base quantum-espresso==20.12.0" +# +# Please note that multiple entries must be whitespace delimited. +# Please see `aiidalab install --help` for more information. +# ENV AIIDALAB_DEFAULT_APPS "aiidalab-widgets-base~=1.0" +ENV AIIDALAB_DEFAULT_APPS "" + +# Specify default factory reset (not set): +ENV AIIDALAB_FACTORY_RESET "" + +USER ${NB_USER} + +WORKDIR "/home/${NB_USER}" + +RUN mkdir -p /home/${NB_USER}/apps + +# Switch to NOTEBOOK_ARGS approach (see below) +# for newer jupyter docker stack versions. +RUN echo 'c.NotebookApp.default_url="/apps/apps/home/start.ipynb"' >> /etc/jupyter/jupyter_notebook_config.py +# ENV NOTEBOOK_ARGS \ + # "--NotebookApp.default_url='/apps/apps/home/start.ipynb'" \ + # "--ContentsManager.allow_hidden=True" diff --git a/my_init.d/factory_reset.sh b/stack/lab/before-notebook.d/factory_reset.sh similarity index 78% rename from my_init.d/factory_reset.sh rename to stack/lab/before-notebook.d/factory_reset.sh index 4f8049ff..50404249 100755 --- a/my_init.d/factory_reset.sh +++ b/stack/lab/before-notebook.d/factory_reset.sh @@ -9,9 +9,9 @@ export SHELL=/bin/bash # 0b010 2 - Remove all files and directories within the users home directory. # If the ~/AIIDALAB_FACTORY_RESET file exists, parse it (and remove it). -if [ -e "/home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET" ]; then - RESET_MODE="`cat /home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET`" - rm -f "/home/${SYSTEM_USER}/AIIDALAB_FACTORY_RESET" # Remove file after parsing. +if [ -e "/home/${NB_USER}/AIIDALAB_FACTORY_RESET" ]; then + RESET_MODE="`cat /home/${NB_USER}/AIIDALAB_FACTORY_RESET`" + rm -f "/home/${NB_USER}/AIIDALAB_FACTORY_RESET" # Remove file after parsing. fi # If the AIIDALAB_FACTORY_RESET environment variable is set, it takes preference (default mode=0): @@ -32,12 +32,12 @@ fi # mode & 001 -> delete ~/apps/ and ~/.local/ if (( (${RESET_MODE} & 0x1) == 0x1)); then echo "factory reset: Remove apps (~/apps/) and local software installation (~/.local/)." - rm -rf "/home/${SYSTEM_USER}/apps" - rm -rf "/home/${SYSTEM_USER}/.local" + rm -rf "/home/${NB_USER}/apps" + rm -rf "/home/${NB_USER}/.local" fi # mode & 010 -> delete all home directory contents if (( (${RESET_MODE} & 0x2) == 0x2)); then echo "factory reset: Remove user home directory contents." - find "/home/${SYSTEM_USER}/" -mindepth 1 -delete + find "/home/${NB_USER}/" -mindepth 1 -delete fi diff --git a/stack/lab/before-notebook.d/prepare-aiidalab.sh b/stack/lab/before-notebook.d/prepare-aiidalab.sh new file mode 100755 index 00000000..76f83cab --- /dev/null +++ b/stack/lab/before-notebook.d/prepare-aiidalab.sh @@ -0,0 +1,60 @@ +#!/bin/bash -e + +# Debugging. +set -x + +# Environment. +export SHELL=/bin/bash + +# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225 +if [ -L /home/${NB_USER}/${NB_USER} ]; then + rm /home/${NB_USER}/${NB_USER} +fi + +# Install the home app. +if [ ! -e /home/${NB_USER}/apps/home ]; then + echo "Install home app." + # The home app is installed in system space and linked to from user space. + # That ensures that users are not inadvertently running the wrong version of + # the home app for a given system environment, but still makes it possible to + # manually install a specific version of the home app in between upgrades, e.g., + # for development work, by simply replacing the link with a clone of the repository. + ln -s /opt/aiidalab-home /home/${NB_USER}/apps/home +elif [[ -d /home/${NB_USER}/apps/home && ! -L /home/${NB_USER}/apps/home ]]; then + # Backup an existing repository of the home app and replace with link to /opt/aiidalab-home. + # This mechanism preserves potential development work on a manually installed repository + # of the home app and also constitutes a migration path for existing aiidalab accounts, where + # the home app was installed directly into user space by default. + mv /home/${NB_USER}/apps/home /home/${NB_USER}/apps/.home~`date --iso-8601=seconds` \ + && ln -s /opt/aiidalab-home /home/${NB_USER}/apps/home || echo "WARNING: Unable to install home app." +fi + + +# Install default apps (see the Dockerfile for an explanation of the +# AIIDALAB_DEFAULT_APPS variable). +if [[ ${INITIAL_SETUP} == 1 ]]; then + + # Iterate over lines in AIIDALAB_DEFAULT_APPS variable. + for app in ${AIIDALAB_DEFAULT_APPS:-}; do + aiidalab install --yes "${app}" + done +fi + +# Clear user trash directory. +if [ -e /home/${NB_USER}/.trash ]; then + rm -rf /home/${NB_USER}/.trash/* +fi + +# Remove old apps_meta.sqlite requests cache files. +find -L /home/${NB_USER} -maxdepth 3 -name apps_meta.sqlite -writable -delete + +# Remove old temporary notebook files. +find -L /home/${NB_USER}/apps -maxdepth 2 -type f -name .*.ipynb -writable -delete + +# Uninstall aiidalab from user packages (if present). +# Would otherwise interfere with the system package. +USER_AIIDALAB_PACKAGE="$(python -c 'import site; print(site.USER_SITE)')/aiidalab" +if [ -e ${USER_AIIDALAB_PACKAGE} ]; then + echo "Uninstall local installation of aiidalab package." + pip uninstall --yes aiidalab +fi diff --git a/gears.svg b/stack/lab/gears.svg similarity index 100% rename from gears.svg rename to stack/lab/gears.svg diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..a5a4dd28 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,59 @@ +from pathlib import Path + +import pytest +import requests +from yaml import SafeLoader, load + +from requests.exceptions import ConnectionError + + +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code == 200: + return True + except ConnectionError: + return False + + +@pytest.fixture(scope="session") +def notebook_service(docker_ip, docker_services): + """Ensure that HTTP service is up and responsive.""" + port = docker_services.port_for("aiidalab", 8888) + url = f"http://{docker_ip}:{port}" + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive(url) + ) + return url + + +@pytest.fixture(scope="session") +def docker_compose(docker_services): + return docker_services._docker_compose + + +@pytest.fixture +def aiidalab_exec(docker_compose): + def execute(command, user=None, **kwargs): + if user: + command = f"exec -T --user={user} aiidalab {command}" + else: + command = f"exec -T aiidalab {command}" + return docker_compose.execute(command, **kwargs) + + return execute + + +@pytest.fixture +def nb_user(aiidalab_exec): + return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip() + + +@pytest.fixture(scope="session") +def _build_config(): + return load(Path("build.yml").read_text(), Loader=SafeLoader) + + +@pytest.fixture(scope="session") +def aiida_version(_build_config): + return _build_config["versions"]["aiida"] diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 120000 index 00000000..5c8318ef --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1 @@ +../docker-compose.yml \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..ade94cbc --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +docker-compose==1.29.2 +pytest==7.1.2 +pytest-docker==1.0.0 +requests==2.28.1 diff --git a/tests/test_aiidalab.py b/tests/test_aiidalab.py new file mode 100644 index 00000000..87d4be89 --- /dev/null +++ b/tests/test_aiidalab.py @@ -0,0 +1,62 @@ +import pytest +import requests +import json +from packaging.version import parse + + +def test_notebook_service_available(notebook_service): + response = requests.get(f"{notebook_service}/") + assert response.status_code == 200 + + +def test_pip_check(aiidalab_exec): + aiidalab_exec("pip check") + + +def test_aiidalab_available(aiidalab_exec, nb_user): + output = aiidalab_exec("aiidalab --version", user=nb_user).decode().strip().lower() + assert "aiidalab" in output + + +def test_create_conda_environment(aiidalab_exec, nb_user): + output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip() + assert "conda activate tmp" in output + + +def test_correct_aiida_version_installed(aiidalab_exec, aiida_version): + info = json.loads(aiidalab_exec("mamba list --json aiida-core").decode())[0] + assert info["name"] == "aiida-core" + assert parse(info["version"]) == parse(aiida_version) + + +@pytest.mark.parametrize("package_manager", ["mamba", "pip"]) +@pytest.mark.parametrize("incompatible_version", ["1.6.3"]) +def test_prevent_installation_of_incompatible_aiida_version( + aiidalab_exec, nb_user, aiida_version, package_manager, incompatible_version +): + assert parse(aiida_version) != parse(incompatible_version) + # Expected to succeed: + aiidalab_exec( + f"{package_manager} install aiida-core=={aiida_version}", user=nb_user + ) + with pytest.raises(Exception): + aiidalab_exec( + f"{package_manager} install aiida-core={incompatible_version}", user=nb_user + ) + + +@pytest.mark.parametrize("package_manager", ["mamba", "pip"]) +@pytest.mark.parametrize("incompatible_version", ["22.7.1"]) +def test_prevent_installation_of_incompatible_aiidalab_version( + aiidalab_exec, nb_user, package_manager, incompatible_version +): + with pytest.raises(Exception): + aiidalab_exec( + f"{package_manager} install aiidalab={incompatible_version}", user=nb_user + ) + + +def test_verdi_status(aiidalab_exec, nb_user): + output = aiidalab_exec("verdi status", user=nb_user).decode().strip() + assert "Connected to RabbitMQ" in output + assert "Daemon is running" in output