Base AiiDAlab image on jupyter/base-notebook image and support AiiDA 2.0

.github/workflows/build_and_test_on_pr.yml

@@ -8,7 +8,7 @@
 # 4. Copy id_rsa file from the docker container to local folder.
 # 5. Restart the container and check that the id_rsa file didn't change.
 
-name: build-and-test-image-from-pull-request
+name: build-and-test
 
 on: [pull_request]
 
@@ -17,50 +17,52 @@ jobs:
     build-and-test:
 
         runs-on: ubuntu-latest
-        timeout-minutes: 45
+        timeout-minutes: 15
+
+        services:
+            registry:
+                image: registry:2
+                ports:
+                    - 5000:5000
 
         steps:
             - uses: actions/checkout@v2
             - name: Set up QEMU
-              uses: docker/setup-qemu-action@v1
+              uses: docker/setup-qemu-action@v2
             - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v1
-            - name: Cache Docker layers
-              uses: actions/cache@v2
+              uses: docker/setup-buildx-action@v2
+              with:
+                  driver-opts: network=host
+            - name: Obtain Docker build args
+              id: meta_extra
+              run: |
+                  echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)"
+            - name: Build base (AiiDA) image
+              id: build_base_image
+              uses: docker/build-push-action@v3
+              with:
+                  context: stack/base
+                  tags: localhost:5000/aiidalab/base:latest
+                  build-args: |
+                      ${{ steps.meta_extra.outputs.build_args }}
+                  push: true
+            - name: Build lab image
+              id: build_lab_image
+              uses: docker/build-push-action@v3
               with:
-                  path: /tmp/.buildx-cache
-                  key: ${{ runner.os }}-buildx-${{ github.sha }}
-                  restore-keys: |
-                      ${{ runner.os }}-buildx-
-            - name: Build image locally
-              uses: docker/build-push-action@v2
+                  context: stack/lab
+                  tags: localhost:5000/aiidalab/lab:latest
+                  push: true
+                  build-args: |
+                      ${{ steps.meta_extra.outputs.build_args }}
+                      BASE_IMAGE=localhost:5000/aiidalab/base@${{ steps.build_base_image.outputs.digest }}
+            - uses: actions/setup-python@v4
               with:
-                  load: true
-                  push: false
-                  tags: aiidalab-docker-stack:latest
-                  cache-from: type=local,src=/tmp/.buildx-cache
-                  cache-to: type=local,dest=/tmp/.buildx-cache
-            - name: Start and test the container
-              id: test_run
+                  python-version: '3.10'
+                  cache: pip # caching pip dependencies
+            - run: pip install -r tests/requirements.txt
+            - name: Run tests
+              env:
+                  AIIDALAB_IMAGE: 'localhost:5000/aiidalab/lab@${{ steps.build_lab_image.outputs.digest }}'
               run: |
-                  mkdir tmp
-                  export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest`
-                  echo "::set-output name=docker_id_first_run::${DOCKERID}"
-                  docker exec --tty --user root $DOCKERID wait-for-services
-                  docker exec --tty --user aiida $DOCKERID wait-for-services
-                  docker exec --tty --user aiida $DOCKERID /bin/bash -l -c '/opt/conda/envs/pgsql/bin/pg_ctl -D /home/$SYSTEM_USER/.postgresql status' # Check that postgres is up.
-                  docker exec --tty --user root $DOCKERID /bin/bash -l -c '/opt/conda/envs/rmq/bin/rabbitmqctl status' # Check that rabbitmq is up.
-                  docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda create -y -n test_env python=3.8' # Check that one can create a new conda environment.
-                  docker exec --tty --user aiida $DOCKERID /bin/bash -l -c 'conda activate test_env' # Check that new environment works.
-                  sudo cp tmp/.ssh/id_rsa . # Copy id_rsa file from the mounted folder.
-                  docker stop $DOCKERID # Stop the container.
-                  export DOCKERID=`docker run -v $PWD/tmp:/home/aiida -d aiidalab-docker-stack:latest` # Start a new container using the same mounted folder.
-                  echo "::set-output name=docker_id_second_run::${DOCKERID}"
-                  docker exec --tty $DOCKERID wait-for-services
-                  sudo diff id_rsa tmp/.ssh/id_rsa # Check that the id_rsa file wasn't modified.
-            - name: Show the container log (first run).
-              if: always()
-              run: docker logs "${{ steps.test_run.outputs.docker_id_first_run }}"
-            - name: Show the container log (second run).
-              if: always()
-              run: docker logs "${{ steps.test_run.outputs.docker_id_second_run }}"
+                  pytest -v

.github/workflows/release_image.yml

@@ -16,31 +16,59 @@ jobs:
     build-docker-image:
 
         runs-on: ubuntu-latest
-        timeout-minutes: 45
+        timeout-minutes: 15
 
         steps:
             - uses: actions/checkout@v2
-            - name: Docker meta
-              id: meta
-              uses: docker/metadata-action@v4
-              with:
-                  images: ${{ github.repository }}
-                  tags: |
-                      type=ref,event=branch
-                      type=pep440,pattern={{version}}
+            - name: Install Conda environment from environment.yml
+              uses: mamba-org/provision-with-micromamba@v12
+            - name: Docker meta exta
+              id: meta_extra
+              run: |
+                  echo "::set-output name=tags::$(./build.py tags --github-actions)"
+                  echo "::set-output name=build_args::$(./build.py docker-build-args --github-actions)"
             - name: Set up QEMU
-              uses: docker/setup-qemu-action@v1
+              uses: docker/setup-qemu-action@v2
             - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v1
+              uses: docker/setup-buildx-action@v2
             - name: Login to DockerHub
               uses: docker/login-action@v1
               with:
                   username: ${{ secrets.DOCKER_USERNAME }}
                   password: ${{ secrets.DOCKER_PASSWORD }}
-            - name: Build and push
-              id: docker_build
-              uses: docker/build-push-action@v2
+            - name: Docker meta base
+              id: meta_base
+              uses: docker/metadata-action@v4
+              with:
+                  images: aiidalab/base
+                  tags: |
+                      type=ref,event=branch
+                      ${{ steps.meta_extra.outputs.tags }}
+            - name: Build and push base image
+              id: build_base_image
+              uses: docker/build-push-action@v3
+              with:
+                  context: stack/base
+                  push: true
+                  tags: ${{ steps.meta_base.outputs.tags }}
+                  platforms: linux/amd64, linux/arm64
+                  build-args: |
+                      ${{ steps.meta_extra.outputs.build_args }}
+            - name: Docker meta lab
+              id: meta_lab
+              uses: docker/metadata-action@v4
+              with:
+                  images: aiidalab/lab
+                  tags: |
+                      type=ref,event=branch
+                      ${{ steps.meta_extra.outputs.tags }}
+            - name: Build and push lab image
+              uses: docker/build-push-action@v3
               with:
+                  context: stack/lab
                   push: true
+                  tags: ${{ steps.meta_lab.outputs.tags }}
                   platforms: linux/amd64, linux/arm64
-                  tags: ${{ steps.meta.outputs.tags }}
+                  build-args: |
+                      ${{ steps.meta_extra.outputs.build_args }}
+                      BASE_IMAGE=aiidalab/base@${{ steps.build_base_image.outputs.digest }}

.gitignore

@@ -275,3 +275,4 @@ submit_test
 
 # Custom.
 Pipfile.lock
+.doit*

.pre-commit-config.yaml

@@ -16,8 +16,14 @@ repos:
       rev: 0.2.2
       hooks:
           - id: yamlfmt
+            args: [--preserve-quotes]
 
     - repo: https://github.com/sirosen/check-jsonschema
       rev: 0.17.0
       hooks:
           - id: check-github-workflows
+
+    - repo: https://github.com/psf/black
+      rev: 22.6.0
+      hooks:
+          - id: black

README.md

@@ -1,20 +1,32 @@
 # Docker Stack for AiiDAlab
 
-This repository contains the Dockerfile for the official AiiDAlab docker images.
+This repository contains the Dockerfiles for the official AiiDAlab docker images.
 
-Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/aiidalab-docker-stack with the following tags:
+Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/ with the following tags:
 
 - `latest` –  the latest tagged release.
 - `<version>` – a specific tagged release, example: `21.12.0`.
 - `master`/`develop` – the latest commit on the corresponding branches with the same name.
 
-## Get started
+## Build images locally
 
-### Local deployment
+To build the images locally, setup a build end testing environment with [conda](https://docs.conda.io/en/latest/miniconda.html) (or [mamba](https://mamba.readthedocs.io/en/latest/installation.html)):
 
-To run AiiDAlab on your own workstation or laptop you can either
-- run the image directly with: `docker run aiidalab-docker-stack -p 8888:8888`, or
-- _(recommended)_ use the `aiidalab-launch` tool which is a thin docker wrapper.
+```console
+conda env create -f environment.yml
+```
+
+Then activate the environment with
+```console
+conda activate aiidalab-docker-stack
+```
+
+To build the images, run `doit build`. =
+You can then run automated tests with `doit tests`.
+
+For local testing, you can start the images with `doit up`, however please refer to the next section for a production-ready local deployment of AiiDAlab with aiidalab-launch.
+
+## Run AiiDAlab in production
 
 The `aiidalab-launch` tool provides a convenient and robust method of both launching and managing one or multiple AiiDAlab instances on your computer.
 To use it, simply install it via pip