Release v2.4.1

.devcontainer/devcontainer.json

@@ -1,9 +1,16 @@
 {
     "dockerComposeFile": "docker-compose.yml",
-    "service": "aiida",
-    "workspaceFolder": "/home/aiida/aiida-core",
-    "postCreateCommand": "bash ./.devcontainer/post_create.sh",
-    "waitFor": "postCreateCommand",
+    "service": "daemon",
+    "workspaceFolder": "/workspaces/aiida-core",
+    "postCreateCommand": "/etc/init/aiida-prepare.sh",
+    "postStartCommand": "pip install -e /workspaces/aiida-core[tests,docs,rest,atomic_tools,pre-commit]",
+    "postAttachCommand": "verdi daemon start",
+    "waitFor": "postStartCommand",
+    "containerUser": "aiida",
+    "remoteUser": "aiida",
+    "remoteEnv": {
+        "HOME": "/home/aiida"
+    },
 	"customizations": {
         "vscode": {
             "extensions": ["ms-python.python", "eamodio.gitlens"]

.devcontainer/docker-compose.yml

@@ -1,54 +1,38 @@
+---
 version: '3.4'
 
 services:
 
-  rabbitmq:
-    image: rabbitmq:3.8.3-management
-    environment:
-        RABBITMQ_DEFAULT_USER: guest
-        RABBITMQ_DEFAULT_PASS: guest
-    ports:
-      - '5672:5672'
-      - '15672:15672'
+    database:
+        image: postgres:15
+        environment:
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: password
+            POSTGRES_HOST_AUTH_METHOD: trust
+        healthcheck:
+            test: [ "CMD-SHELL", "pg_isready"]
+            interval: 5s
+            timeout: 5s
+            retries: 10
 
-    healthcheck:
-      test: rabbitmq-diagnostics -q ping
-      interval: 30s
-      timeout: 30s
-      retries: 5
-    networks:
-      - aiida
+    messaging:
+        image: rabbitmq:3.8.14-management
+        environment:
+            RABBITMQ_DEFAULT_USER: guest
+            RABBITMQ_DEFAULT_PASS: guest
+        healthcheck:
+            test: rabbitmq-diagnostics check_port_connectivity
+            interval: 30s
+            timeout: 30s
+            retries: 10
 
-  postgres:
-    image: postgres:12
-    ports:
-      - '5432:5432'
-    networks:
-      - aiida
-    environment:
-      POSTGRES_HOST_AUTH_METHOD: trust
-
-  aiida:
-    #image: "aiidateam/aiida-core:main"
-    image: "aiida-core-dev"
-    build:
-      # need to add the parent directory to context to copy over new configure-aiida.sh
-      context: ..
-      dockerfile: .devcontainer/Dockerfile
-    user: aiida
-    environment:
-      DB_HOST: postgres
-      BROKER_HOST: rabbitmq
-
-    # no need for /sbin/my_init
-    entrypoint: tail -f /dev/null
-    volumes:
-      - ..:/home/aiida/aiida-core:cached
-    networks:
-      - aiida
-    depends_on:
-      - rabbitmq
-      - postgres
-
-networks:
-  aiida:
+    daemon:
+        image: ghcr.io/aiidateam/aiida-core-base:edge
+        user: aiida
+        entrypoint: tail -f /dev/null
+        environment:
+          SETUP_DEFAULT_AIIDA_PROFILE: 'true'
+          TZ: 'Europe/Zurich'
+        depends_on:
+            database:
+                condition: service_healthy

.docker/README.md

@@ -0,0 +1,21 @@
+# AiiDA docker stacks
+
+### Build images locally
+
+To build the images, run `docker buildx bake -f build.json -f docker-bake.hcl --load` (tested with *docker buildx* version v0.8.2).
+
+The build system will attempt to detect the local architecture and automatically build images for it (tested with amd64 and arm64).
+You can also specify a custom platform with the `--platform`, example: `docker buildx bake -f build.json -f docker-bake.hcl --set *.platform=linux/amd64 --load`.
+
+### Test the build images locally
+
+Run
+
+```bash
+TAG=newly-baked python -m pytest -s tests
+```
+
+### Trigger a build on ghcr.io and dockerhub
+
+Only the PR open to the organization repository will trigger a build on ghcr.io.
+Push to dockerhub is triggered when making a release on github.

.docker/aiida-core-base/Dockerfile

@@ -0,0 +1,176 @@
+# syntax=docker/dockerfile:1
+
+# Inspired by jupyter's docker-stacks-fundation image:
+# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile
+
+ARG BASE=ubuntu:22.04
+
+FROM $BASE
+
+LABEL maintainer="AiiDA Team <developers@aiida.net>"
+
+ARG SYSTEM_USER="aiida"
+ARG SYSTEM_UID="1000"
+ARG SYSTEM_GID="100"
+
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+ENV SYSTEM_USER="${SYSTEM_USER}"
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update --yes && \
+    # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as
+    #   the ubuntu base image is rebuilt too seldom sometimes (less than once a month)
+    apt-get upgrade --yes && \
+    apt-get install --yes --no-install-recommends \
+    # - bzip2 is necessary to extract the micromamba executable.
+    bzip2 \
+    # - xz-utils is necessary to extract the s6-overlay.
+    xz-utils \
+    ca-certificates \
+    locales \
+    sudo \
+    # development tools
+    git \
+    openssh-client \
+    vim \
+    # the gcc compiler need to build some python packages e.g. psutil and pymatgen
+    build-essential \
+    wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+
+# Install s6-overlay to handle startup and shutdown of services
+ARG S6_OVERLAY_VERSION=3.1.5.0
+RUN wget --progress=dot:giga -O /tmp/s6-overlay-noarch.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+    rm /tmp/s6-overlay-noarch.tar.xz
+
+RUN set -x && \
+    arch=$(uname -m) && \
+    wget --progress=dot:giga -O /tmp/s6-overlay-binary.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${arch}.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-binary.tar.xz && \
+    rm /tmp/s6-overlay-binary.tar.xz
+
+# Configure environment
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    SYSTEM_USER="${SYSTEM_USER}" \
+    SYSTEM_UID=${SYSTEM_UID} \
+    SYSTEM_GID=${SYSTEM_GID} \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH="${CONDA_DIR}/bin:${PATH}" \
+    HOME="/home/${SYSTEM_USER}"
+
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+# Enable prompt color in the skeleton .bashrc before creating the default SYSTEM_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    useradd -l -m -s /bin/bash -N -u "${SYSTEM_UID}" "${SYSTEM_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    chown "${SYSTEM_USER}:${SYSTEM_GID}" "${CONDA_DIR}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER ${SYSTEM_UID}
+
+# Pin python version here
+ARG PYTHON_VERSION
+
+# Download and install Micromamba, and initialize Conda prefix.
+#   <https://github.com/mamba-org/mamba#micromamba>
+#   Similar projects using Micromamba:
+#     - Micromamba-Docker: <https://github.com/mamba-org/micromamba-docker>
+#     - repo2docker: <https://github.com/jupyterhub/repo2docker>
+# Install Python, Mamba and jupyter_core
+# Cleanup temporary files and remove Micromamba
+# Correct permissions
+# Do all this in a single RUN command to avoid duplicating all of the
+# files across image layers when the permissions change
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" initial-condarc "${CONDA_DIR}/.condarc"
+WORKDIR /tmp
+RUN set -x && \
+    arch=$(uname -m) && \
+    if [ "${arch}" = "x86_64" ]; then \
+        # Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437>
+        arch="64"; \
+    fi && \
+    wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \
+        "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \
+    tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \
+    rm /tmp/micromamba.tar.bz2 && \
+    PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \
+    if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \
+    # Install the packages
+    ./micromamba install \
+        --root-prefix="${CONDA_DIR}" \
+        --prefix="${CONDA_DIR}" \
+        --yes \
+        "${PYTHON_SPECIFIER}" \
+        'mamba' && \
+    rm micromamba && \
+    # Pin major.minor version of python
+    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${SYSTEM_USER}"
+
+# Add ~/.local/bin to PATH where the dependencies get installed via pip
+# This require the package installed with `--user` flag in pip
+ENV PATH=${PATH}:/home/${NB_USER}/.local/bin
+
+# Switch to root to install AiiDA and set AiiDA as service
+# Install AiiDA from source code
+USER root
+COPY --from=src . /tmp/aiida-core
+RUN pip install /tmp/aiida-core --no-cache-dir && \
+    rm -rf /tmp/aiida-core
+
+# Enable verdi autocompletion.
+RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \
+     echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     fix-permissions "${CONDA_DIR}"
+
+# COPY AiiDA profile configuration for profile setup init script
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml"
+COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d
+COPY s6-assets/init /etc/init
+RUN mkdir /etc/init/run-before-daemon-start && \
+    mkdir /etc/init/run-after-daemon-start
+
+# Otherwise will stuck on oneshot services
+# https://github.com/just-containers/s6-overlay/issues/467
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
+
+# Switch back to USER aiida to avoid accidental container runs as root
+USER ${SYSTEM_UID}
+
+ENTRYPOINT ["/init"]
+
+WORKDIR "${HOME}"

.docker/aiida-core-base/fix-permissions

@@ -0,0 +1,35 @@
+#!/bin/bash
+# This is brought from jupyter docker-stacks:
+# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/fix-permissions
+# set permissions on a directory
+# after any installation, if a directory needs to be (human) user-writable,
+# run this script on it.
+# It will make everything in the directory owned by the group ${SYSTEM_GID}
+# and writable by that group.
+
+# uses find to avoid touching files that already have the right permissions,
+# which would cause massive image explosion
+
+# right permissions are:
+# group=${SYSEM_GID}
+# AND permissions include group rwX (directory-execute)
+# AND directories have setuid,setgid bits set
+
+set -e
+
+for d in "$@"; do
+    find "${d}" \
+        ! \( \
+            -group "${SYSTEM_GID}" \
+            -a -perm -g+rwX \
+        \) \
+        -exec chgrp "${SYSTEM_GID}" -- {} \+ \
+        -exec chmod g+rwX -- {} \+
+    # setuid, setgid *on directories only*
+    find "${d}" \
+        \( \
+            -type d \
+            -a ! -perm -6000 \
+        \) \
+        -exec chmod +6000 -- {} \+
+done

.docker/aiida-core-base/initial-condarc

@@ -0,0 +1,6 @@
+# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html
+
+auto_update_conda: false
+show_channel_urls: true
+channels:
+  - conda-forge

.docker/aiida-core-base/s6-assets/config-quick-setup.yaml

@@ -0,0 +1,15 @@
+---
+db_engine: postgresql_psycopg2
+db_backend: core.psql_dos
+db_host: database
+db_port: 5432
+su_db_username: postgres
+su_db_password: password
+su_db_name: template1
+db_name: aiida_db
+db_username: aiida
+db_password: password
+broker_host: messaging
+broker_port: 5672
+broker_username: guest
+broker_password: guest