Merge branch 'main' into fix/6524/force-kill

.docker/aiida-core-base/Dockerfile

@@ -137,7 +137,7 @@ RUN set -x && \
         mamba && \
     rm micromamba && \
     # Pin major.minor version of python
-    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    mamba list python | grep -oP 'python\s+\K[\d.]+' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${SYSTEM_USER}"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,7 +9,7 @@ assignees: ''
 
 <!-- Before raising an issue, it is suggested that you first check out the: -->
 
-- [ ] [AiiDA Troubleshooting Documentation](https://aiida.readthedocs.io/projects/aiida-core/en/latest/intro/troubleshooting.html)
+- [ ] [AiiDA Troubleshooting Documentation](https://aiida.readthedocs.io/projects/aiida-core/en/stable/installation/troubleshooting.html)
 - [ ] [AiiDA Discourse Forum](https://aiida.discourse.group/)
 
 ### Describe the bug

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,4 @@
 contact_links:
-- name: AiiDA Discussions
-  url: https://github.com/aiidateam/aiida-core/discussions
-  about: For aiida-core questions and discussion
-- name: AiiDA Users Forum
-  url: http://www.aiida.net/mailing-list/
+- name: AiiDA Discourse group
+  url: https://aiida.discourse.group
   about: For general questions and discussion

.github/workflows/benchmark.yml

@@ -56,7 +56,7 @@ jobs:
         python-version: '3.10'
 
     - name: Run benchmarks
-      run: pytest --benchmark-only --benchmark-json benchmark.json
+      run: pytest --db-backend psql --benchmark-only --benchmark-json benchmark.json tests/
 
     - name: Store benchmark result
       uses: aiidateam/github-action-benchmark@v3
@@ -73,4 +73,4 @@ jobs:
         alert-threshold: 200%
         comment-on-alert: true
         fail-on-alert: false
-        alert-comment-cc-users: '@chrisjsewell,@giovannipizzi'
+        alert-comment-cc-users: '@giovannipizzi,@agoscinski,@GeigerJ2,@khsrali,@unkcpz'

.github/workflows/ci-code.yml

@@ -104,7 +104,7 @@ jobs:
         AIIDA_WARN_v3: 1
       # Python 3.12 has a performance regression when running with code coverage
       # so run code coverage only for python 3.9.
-      run: pytest -v tests -m 'not nightly' ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
+      run: pytest --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
 
     - name: Upload coverage report
       if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core'
@@ -139,7 +139,7 @@ jobs:
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -m 'presto'
+      run: pytest -m 'presto' tests/
 
 
   verdi:

.github/workflows/extract-docker-image-names.sh

@@ -9,38 +9,35 @@ set -euo pipefail
 # The input to this script is a JSON string passed via BAKE_METADATA env variable
 # Here's example input (trimmed to relevant bits):
 # BAKE_METADATA: {
-#    "base": {
+#    "aiida-core-base": {
+#      # ...
 #      "containerimage.descriptor": {
 #        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
 #        "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
 #        "size": 6170,
 #      },
 #      "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
-#      "image.name": "ghcr.io/aiidalab/base"
-#    },
-#    "aiida-core-base": {
 #      "image.name": "ghcr.io/aiidateam/aiida-core-base"
-#       "containerimage.digest": "sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26",
-#       "...": ""
-#    }
-#    "aiida-core-with-services": {
-#      "image.name": "ghcr.io/aiidateam/aiida-core-with-services"
-#      "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
-#      "...": ""
-#    }
+#    },
 #    "aiida-core-dev": {
-#      "image.name": "ghcr.io/aiidateam/aiida-core-with-services"
 #      "containerimage.digest": "sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97",
+#      "...": ...
+#      "image.name": "ghcr.io/aiidateam/aiida-core-dev"
+#    },
+#    "aiida-core-with-services": {
 #      "...": ""
-#    }
+#      "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
+#      "image.name": "ghcr.io/aiidateam/aiida-core-with-services"
+#    },
+#    "some-other-key": ...
 #  }
 #
 # Example output (real output is on one line):
 #
 # images={
-#   "AIIDA_CORE_BASE_IMAGE": "ghcr.io/aiidateam/aiida-core-base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
-#   "AIIDA_CORE_WITH_SERVICES_IMAGE": "ghcr.io/aiidateam/aiida-core-with-services@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26",
-#   "AIIDA_CORE_DEV_IMAGE": "ghcr.io/aiidateam/aiida-core-dev@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
+#   "AIIDA_CORE_BASE_IMAGE": "ghcr.io/aiidateam/aiida-core-base@sha256:4c402a8bfd635650ad691674f8f29e7ddec5fa656fb425452067950415ee447f",
+#   "AIIDA_CORE_DEV_IMAGE": "ghcr.io/aiidateam/aiida-core-dev@sha256:f94c06e47f801e751f9829010b31532039b210aad2649d43205e16c08371b2ed",
+#   "AIIDA_CORE_WITH_SERVICES_IMAGE": "ghcr.io/aiidateam/aiida-core-with-services@sha256:bd8272f2a331af7eac3e83c44cc16d23b2e5f601a20ab4a865402659b758515e"
 # }
 #
 # This json output is later turned to environment variables using fromJson() GHA builtin
@@ -52,5 +49,7 @@ if [[ -z ${BAKE_METADATA-} ]];then
     exit 1
 fi
 
-images=$(echo "${BAKE_METADATA}" | jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries')
+images=$(echo "${BAKE_METADATA}" |
+jq -c 'to_entries | map(select(.key | startswith("aiida"))) | from_entries' | # filters out every key that does not start with aiida
+jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries')
 echo "images=$images"

.github/workflows/nightly.yml

@@ -104,19 +104,6 @@ jobs:
         rabbitmq-version: ['3.11', '3.12', '3.13']
 
     services:
-      postgres:
-        image: postgres:16
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:${{ matrix.rabbitmq-version }}-management
         ports:
@@ -132,9 +119,6 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install system dependencies
-      run: sudo apt update && sudo apt install postgresql
-
     - name: Setup SSH on localhost
       run: .github/workflows/setup_ssh.sh
 
@@ -145,7 +129,7 @@ jobs:
       id: tests
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -sv -m 'requires_rmq'
+      run: pytest -s --db-backend sqlite -m 'requires_rmq' tests/
 
     - name: Slack notification
       # Always run this step (otherwise it would be skipped if any of the previous steps fail) but only if the

.github/workflows/release.yml

@@ -54,19 +54,6 @@ jobs:
     timeout-minutes: 30
 
     services:
-      postgres:
-        image: postgres:10
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:3.8.14-management
         ports:
@@ -76,16 +63,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Install system dependencies
-      run: |
-        sudo apt update
-        sudo apt install postgresql graphviz
-
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
 
     - name: Run sub-set of test suite
-      run: pytest -sv -k 'requires_rmq'
+      run: pytest -s -m requires_rmq --db-backend=sqlite tests/
 
   publish:
 

.github/workflows/test-install.yml

@@ -229,7 +229,7 @@ jobs:
       env:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
-      run: pytest --verbose tests -m 'not nightly'
+      run: pytest --db-backend psql tests -m 'not nightly' tests/
 
     - name: Freeze test environment
       run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt
@@ -306,7 +306,7 @@ jobs:
       if: steps.check_reqs.outcome == 'Failure'  # only run if requirements/ are inconsistent
       id: create_update_requirements_pr
       continue-on-error: true
-      uses: peter-evans/create-pull-request@v6
+      uses: peter-evans/create-pull-request@v7
       with:
         branch: update-requirements
         commit-message: Automated update of requirements/ files.

.github/workflows/tests_nightly.sh

@@ -13,4 +13,4 @@ verdi -p test_aiida run ${SYSTEM_TESTS}/test_containerized_code.py
 bash ${SYSTEM_TESTS}/test_polish_workchains.sh
 verdi daemon stop
 
-AIIDA_TEST_PROFILE=test_aiida pytest -v tests -m 'nightly'
+AIIDA_TEST_PROFILE=test_aiida pytest --db-backend psql -m nightly tests/

.pre-commit-config.yaml

@@ -90,6 +90,7 @@ repos:
         .docker/.*|
         docs/.*|
         utils/.*|
+        tests/.*|
 
         src/aiida/calculations/arithmetic/add.py|
         src/aiida/calculations/diff_tutorial/calculations.py|
@@ -124,7 +125,6 @@ repos:
         src/aiida/engine/processes/ports.py|
         src/aiida/manage/configuration/__init__.py|
         src/aiida/manage/configuration/config.py|
-        src/aiida/manage/configuration/profile.py|
         src/aiida/manage/external/rmq/launcher.py|
         src/aiida/manage/tests/main.py|
         src/aiida/manage/tests/pytest_fixtures.py|
@@ -192,17 +192,6 @@ repos:
         src/aiida/transports/plugins/local.py|
         src/aiida/transports/plugins/ssh.py|
         src/aiida/workflows/arithmetic/multiply_add.py|
-
-        tests/conftest.py|
-        tests/repository/conftest.py|
-        tests/repository/test_repository.py|
-        tests/sphinxext/sources/workchain/conf.py|
-        tests/sphinxext/sources/workchain_broken/conf.py|
-        tests/storage/psql_dos/migrations/conftest.py|
-        tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py|
-        tests/test_calculation_node.py|
-        tests/test_nodes.py|
-
       )$
 
   - id: dm-generate-all

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v2.6.3 - 2024-11-6
+
+### Fixes
+- CLI: Fix exception for `verdi plugin list` (#6560) [[c3b10b7]](https://github.com/aiidateam/aiida-core/commit/c3b10b759a9cd062800ef120591d5c7fd0ae4ee7)
+- `DirectScheduler`: Ensure killing child processes (#6572) [[fddffca]](https://github.com/aiidateam/aiida-core/commit/fddffca67b4f7e3b76b19df7db8e1511c449d2d9)   
+- Engine: Fix state change broadcast before process node is updated  (#6580) [[867353c]](https://github.com/aiidateam/aiida-core/commit/867353c415c61d94a2427d5225dd5224a1b95fb9)
+
+### Devops
+- Docker: Replace sleep with `s6-notifyoncheck` (#6475) [[9579378b]](https://github.com/aiidateam/aiida-core/commit/9579378ba063237baa5b73380eb8e9f0a28529ee)
+- Fix failed docker CI using more reasoning grep regex to parse python version (#6581) [[332a4a91]](https://github.com/aiidateam/aiida-core/commit/332a4a915771afedcb144463b012558e4669e529)
+- DevOps: Fix json query in reading the docker names to filter out fields not starting with aiida (#6573) [[e1467edc]](https://github.com/aiidateam/aiida-core/commit/e1467edca902867e53605e0e60b67f8767bf8d3e)
+
+
 ## v2.6.2 - 2024-08-07
 
 ### Fixes
@@ -31,7 +44,7 @@
 
 ## v2.6.1 - 2024-07-01
 
-### Fixes:
+### Fixes
 - Fixtures: Make `pgtest` truly an optional dependency [[9fe8fd2e0]](https://github.com/aiidateam/aiida-core/commit/9fe8fd2e0b88e746ee2156eccb71b7adbab6b2c5)
 
 

docs/source/howto/data.rst

@@ -129,19 +129,24 @@ top-level process. Further, numbered subdirectories are created for each step of
 ``aiida.out`` of the ``ArithmeticAddCalculation`` are placed in ``inputs`` and ``outputs``. In addition, these also
 contain the submission script ``_aiidasubmit.sh``, as well as the scheduler stdout and stderr, ``_scheduler-stdout.txt``
 and ``_scheduler-stderr.txt``, respectively. Lastly, the source code of the ``multiply`` ``calcfunction`` presenting the
-first step of the workflow is contained in the ``source_file``.
-
-Upon having a closer look at the directory, we also find the hidden ``.aiida_node_metadata.yaml`` files, which are
-created for every ``ProcessNode`` and contain additional information about the ``Node``, the ``User``, and the
-``Computer``, as well as the ``.aiida`` subdirectory with machine-readable AiiDA-internal data in JSON format.
-
-Since child processes are explored recursively, arbitrarily complex, nested workflows can be dumped. As already seen
-above, the ``-p`` flag allows to specify a custom dumping path. If none is provided, it is automatically generated from
-the ``process_label`` (or ``process_type``) and the ``pk``. In addition, the command provides the ``-o`` flag to
-overwrite existing directories, the ``-f`` flag to dump all files for each ``CalculationNode`` of the workflow in a flat
-directory structure, and the ``--include-inputs/--exclude-inputs`` (``--include-outputs/--exclude-outputs``) flags to
-also dump additional node inputs (outputs) of each ``CalculationNode`` of the workflow into ``node_inputs``
-(``node_outputs``) subdirectories. For a full list of available options, call :code:`verdi process dump --help`.
+first step of the workflow is contained in the ``source_file``. Since child processes are explored recursively,
+arbitrarily complex, nested workflows can be dumped. Upon having a closer look at the directory, we also find the hidden
+``.aiida_node_metadata.yaml`` files, which are created for every ``ProcessNode`` and contain additional information
+about the ``Node``, the ``User``, and the ``Computer``, as well as the ``.aiida`` subdirectory with machine-readable
+AiiDA-internal data in JSON format.
+
+As already seen above, the ``-p`` flag allows to specify a custom dumping path. If none is provided, it is automatically
+generated from the ``process_label`` (or ``process_type``) and the ``pk``. In addition, the command provides the
+``-o/--overwrite`` flag to fully overwrite an existing dumping directory, as well as the ``--incremental`` flag, with
+which files are gradually added to an existing directory (this is the default behavior). By default, only sealed process
+nodes can be dumped, however, the behavior can be changed with the ``--dump-unsealed`` flag, which can be useful in
+conjunction with ``--incremental`` to gradually obtain data while a process is running. Furthermore, the ``-f/--flat``
+flag can be used to dump all files for each ``CalculationNode`` of the workflow in a flat directory structure, and the
+``--include-inputs/--exclude-inputs`` (``--include-outputs/--exclude-outputs``) flags are used to also dump additional
+node inputs (outputs) of each ``CalculationNode`` of the workflow into ``node_inputs`` (``node_outputs``)
+subdirectories.
+
+For a full list of available options, call :code:`verdi process dump --help`.
 
 .. _how-to:data:import:provenance:
 

docs/source/reference/command_line.rst

@@ -197,6 +197,7 @@ Below is a list with all available subcommands.
       check-load-time          Check for common indicators that slowdown `verdi`.
       check-undesired-imports  Check that verdi does not import python modules it shouldn't.
       launch-add               Launch an ``ArithmeticAddCalculation``.
+      launch-multiply-add      Launch a ``MultipylAddWorkChain``.
       rabbitmq                 Commands to interact with RabbitMQ.
       run-sql                  Run a raw SQL command on the profile database (only...
       validate-plugins         Validate all plugins by checking they can be loaded.

pyproject.toml

@@ -211,7 +211,7 @@ notebook = [
 ]
 pre-commit = [
   'aiida-core[atomic_tools,rest,tests,tui]',
-  'mypy~=1.10.0',
+  'mypy~=1.13.0',
   'packaging~=23.0',
   'pre-commit~=3.5',
   'sqlalchemy[mypy]~=2.0',
@@ -255,6 +255,7 @@ runaiida = 'aiida.cmdline.commands.cmd_run:run'
 verdi = 'aiida.cmdline.commands.cmd_verdi:verdi'
 
 [project.urls]
+Changelog = 'https://github.com/aiidateam/aiida-core/blob/main/CHANGELOG.md'
 Documentation = 'https://aiida.readthedocs.io'
 Home = 'http://www.aiida.net/'
 Source = 'https://github.com/aiidateam/aiida-core'
@@ -431,6 +432,13 @@ deps =
     py311: -rrequirements/requirements-py-3.11.txt
     py312: -rrequirements/requirements-py-3.12.txt
 
+[testenv:py{39,310,311,312}-presto]
+passenv =
+    PYTHONASYNCIODEBUG
+setenv =
+    AIIDA_WARN_v3 =
+commands = pytest -m 'presto' {posargs}
+
 [testenv:py{39,310,311,312}]
 passenv =
     PYTHONASYNCIODEBUG