Merge branch 'develop' into fix/5261/exposed-outputs-non-existing-nam…

…espace

.github/system_tests/test_polish_workchains.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -ev
+
+# Make sure the folder containing the workchains is in the python path before the daemon is started
+MODULE_POLISH="${GITHUB_WORKSPACE}/.molecule/default/files/polish"
+CLI_SCRIPT="${MODULE_POLISH}/cli.py"
+
+declare -a EXPRESSIONS=("1 -2 -1 4 -5 -5 * * * * +" "2 1 3 3 -1 + ^ ^ +" "3 -5 -1 -4 + * ^" "2 4 2 -4 * * +" "3 1 1 5 ^ ^ ^" "3 1 3 4 -4 2 * + + ^ ^")
+NUMBER_WORKCHAINS=5
+TIMEOUT=600
+CODE='add!'  # Note the exclamation point is necessary to force the value to be interpreted as LABEL type identifier
+
+# Get the absolute path for verdi
+VERDI=$(which verdi)
+
+if [ -n "$EXPRESSIONS" ]; then
+    for expression in "${EXPRESSIONS[@]}"; do
+        $VERDI -p test_${AIIDA_TEST_BACKEND} run "${CLI_SCRIPT}" -X $CODE -C -F -d -t $TIMEOUT "$expression"
+    done
+else
+    for i in $(seq 1 $NUMBER_WORKCHAINS); do
+        $VERDI -p test_${AIIDA_TEST_BACKEND} run "${CLI_SCRIPT}" -X $CODE -C -F -d -t $TIMEOUT
+    done
+fi

.github/workflows/benchmark.yml

@@ -50,9 +50,30 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.8'
+
+    - name: Upgrade pip
+      run: |
+        pip install --upgrade pip
+        pip --version
+
+    - name: Build pymatgen with compatible numpy
+      run: |
+        # This step is necessary because certain versions of `pymatgen` will not specify an explicit version of
+        # `numpy` in its build requirements, and so the latest version will be used. This causes problems,
+        # however, because this means that the compiled version of `pymatgen` can only be used with that version
+        # of `numpy` or higher, since `numpy` only guarantees forward compatibility of the ABI. If we want to
+        # run with an older version of `numpy`, we need to ensure that `pymatgen` is built with that same
+        # version. This we can accomplish by installing the desired version of `numpy` manually and then calling
+        # the install command for `pymatgen` with the `--no-build-isolation` flag. This flag will ensure that
+        # build dependencies are ignored and won't be installed (preventing the most recent version of `numpy`
+        # to be installed) and the build relies on those requirements already being present in the environment.
+        # We also need to install `wheel` because otherwise the `pymatgen` build will fail because `bdist_wheel`
+        # will not be available.
+        pip install numpy==1.21.4 wheel
+        pip install pymatgen==2022.0.16 --no-cache-dir --no-build-isolation
+
     - name: Install python dependencies
       run: |
-        python -m pip install --upgrade pip
         pip install -r requirements/requirements-py-3.8.txt
         pip install --no-deps -e .
         pip freeze

.github/workflows/check_release_tag.py

@@ -1,16 +1,31 @@
 # -*- coding: utf-8 -*-
 """Check that the GitHub release tag matches the package version."""
 import argparse
-import json
+import ast
+from pathlib import Path
+
+
+def get_version_from_module(content: str) -> str:
+    """Get the __version__ value from a module."""
+    # adapted from setuptools/config.py
+    try:
+        module = ast.parse(content)
+    except SyntaxError as exc:
+        raise IOError(f'Unable to parse module: {exc}')
+    try:
+        return next(
+            ast.literal_eval(statement.value) for statement in module.body if isinstance(statement, ast.Assign)
+            for target in statement.targets if isinstance(target, ast.Name) and target.id == '__version__'
+        )
+    except StopIteration:
+        raise IOError('Unable to find __version__ in module')
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('GITHUB_REF', help='The GITHUB_REF environmental variable')
-    parser.add_argument('SETUP_PATH', help='Path to the setup.json')
     args = parser.parse_args()
     assert args.GITHUB_REF.startswith('refs/tags/v'), f'GITHUB_REF should start with "refs/tags/v": {args.GITHUB_REF}'
     tag_version = args.GITHUB_REF[11:]
-    with open(args.SETUP_PATH, encoding='utf8') as handle:
-        data = json.load(handle)
-    pypi_version = data['version']
-    assert tag_version == pypi_version, f'The tag version {tag_version} != {pypi_version} specified in `setup.json`'
+    pypi_version = get_version_from_module(Path('aiida/__init__.py').read_text(encoding='utf-8'))
+    assert tag_version == pypi_version, f'The tag version {tag_version} != {pypi_version} specified in `pyproject.toml`'

.github/workflows/ci-code.yml

@@ -33,7 +33,7 @@ jobs:
       if: failure() && steps.check_reqs.outputs.error
       uses: peter-evans/commit-comment@v1
       with:
-        path: setup.json
+        path: pyproject.toml
         body: |
           ${{ steps.check_reqs.outputs.error }}
 
@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.8', '3.10']
         backend: ['django', 'sqlalchemy']
 
     services:
@@ -89,11 +89,26 @@ jobs:
         sudo apt install postgresql graphviz
 
     - name: Upgrade pip and setuptools
-      # It is crucial to update `setuptools` or the installation of `pymatgen` can break
       run: |
-        pip install --upgrade pip setuptools
+        pip install --upgrade pip
         pip --version
 
+    - name: Build pymatgen with compatible numpy
+      run: |
+        # This step is necessary because certain versions of `pymatgen` will not specify an explicit version of
+        # `numpy` in its build requirements, and so the latest version will be used. This causes problems,
+        # however, because this means that the compiled version of `pymatgen` can only be used with that version
+        # of `numpy` or higher, since `numpy` only guarantees forward compatibility of the ABI. If we want to
+        # run with an older version of `numpy`, we need to ensure that `pymatgen` is built with that same
+        # version. This we can accomplish by installing the desired version of `numpy` manually and then calling
+        # the install command for `pymatgen` with the `--no-build-isolation` flag. This flag will ensure that
+        # build dependencies are ignored and won't be installed (preventing the most recent version of `numpy`
+        # to be installed) and the build relies on those requirements already being present in the environment.
+        # We also need to install `wheel` because otherwise the `pymatgen` build will fail because `bdist_wheel`
+        # will not be available.
+        pip install numpy==1.21.4 wheel
+        pip install pymatgen==2022.0.16 --no-cache-dir --no-build-isolation
+
     - name: Install aiida-core
       run: |
         pip install --use-feature=2020-resolver -r requirements/requirements-py-${{ matrix.python-version }}.txt
@@ -114,10 +129,10 @@ jobs:
         .github/workflows/tests.sh
 
     - name: Upload coverage report
-      if: matrix.python-version == 3.7 && github.repository == 'aiidateam/aiida-core'
+      if: matrix.python-version == 3.8 && github.repository == 'aiidateam/aiida-core'
       uses: codecov/codecov-action@v1
       with:
-        name: aiida-pytests-py3.7-${{ matrix.backend }}
+        name: aiida-pytests-py3.8-${{ matrix.backend }}
         flags: ${{ matrix.backend }}
         file: ./coverage.xml
         fail_ci_if_error: false  # don't fail job, if coverage upload fails
@@ -130,7 +145,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.8', '3.10']
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/ci-style.yml

@@ -29,7 +29,9 @@ jobs:
 
     - name: Install python dependencies
       run: |
-        pip install -e .[all]
+        pip install --upgrade pip
+        pip install -r requirements/requirements-py-3.8.txt
+        pip install -e .[pre-commit]
         pip freeze
 
     - name: Run pre-commit

.github/workflows/docs-build.yml

@@ -22,7 +22,8 @@ jobs:
         python-version: '3.8'
     - name: Install python dependencies
       run: |
-        pip install -e .[docs,tests]
+        pip install --upgrade pip
+        pip install -e .[docs,tests,rest,atomic_tools]
     - name: Build HTML docs
       id: linkcheck
       run: |

.github/workflows/post-release.yml

@@ -23,15 +23,16 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: '3.8'
 
     - name: Install python dependencies
       run: |
+        pip install --upgrade pip
         pip install transifex-client sphinx-intl
-        pip install -e .[docs,tests]
+        pip install -e .[docs,tests,rest,atomic_tools]
 
     - name: Build pot files
       env:

.github/workflows/rabbitmq.yml

@@ -56,6 +56,22 @@ jobs:
                 pip install --upgrade pip
                 pip --version
 
+        -   name: Build pymatgen with compatible numpy
+            run: |
+                # This step is necessary because certain versions of `pymatgen` will not specify an explicit version of
+                # `numpy` in its build requirements, and so the latest version will be used. This causes problems,
+                # however, because this means that the compiled version of `pymatgen` can only be used with that version
+                # of `numpy` or higher, since `numpy` only guarantees forward compatibility of the ABI. If we want to
+                # run with an older version of `numpy`, we need to ensure that `pymatgen` is built with that same
+                # version. This we can accomplish by installing the desired version of `numpy` manually and then calling
+                # the install command for `pymatgen` with the `--no-build-isolation` flag. This flag will ensure that
+                # build dependencies are ignored and won't be installed (preventing the most recent version of `numpy`
+                # to be installed) and the build relies on those requirements already being present in the environment.
+                # We also need to install `wheel` because otherwise the `pymatgen` build will fail because `bdist_wheel`
+                # will not be available.
+                pip install numpy==1.21.4 wheel
+                pip install pymatgen==2022.0.16 --no-cache-dir --no-build-isolation
+
         -   name: Install aiida-core
             run: |
                 pip install -r requirements/requirements-py-3.8.txt

.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.8'
-    - run: python .github/workflows/check_release_tag.py $GITHUB_REF setup.json
+    - run: python .github/workflows/check_release_tag.py $GITHUB_REF
 
   pre-commit:
 
@@ -43,7 +43,11 @@ jobs:
         sudo apt update
         sudo apt install libkrb5-dev ruby ruby-dev
     - name: Install python dependencies
-      run: pip install -e .[all]
+      run: |
+        pip install --upgrade pip
+        pip install -r requirements/requirements-py-3.8.txt
+        pip install -e .[pre-commit]
+        pip freeze
     - name: Run pre-commit
       run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
 
@@ -82,9 +86,30 @@ jobs:
       run: |
         sudo apt update
         sudo apt install postgresql graphviz
+
+    - name: Upgrade pip
+      run: |
+        pip install --upgrade pip
+        pip --version
+
+    - name: Build pymatgen with compatible numpy
+      run: |
+        # This step is necessary because certain versions of `pymatgen` will not specify an explicit version of
+        # `numpy` in its build requirements, and so the latest version will be used. This causes problems,
+        # however, because this means that the compiled version of `pymatgen` can only be used with that version
+        # of `numpy` or higher, since `numpy` only guarantees forward compatibility of the ABI. If we want to
+        # run with an older version of `numpy`, we need to ensure that `pymatgen` is built with that same
+        # version. This we can accomplish by installing the desired version of `numpy` manually and then calling
+        # the install command for `pymatgen` with the `--no-build-isolation` flag. This flag will ensure that
+        # build dependencies are ignored and won't be installed (preventing the most recent version of `numpy`
+        # to be installed) and the build relies on those requirements already being present in the environment.
+        # We also need to install `wheel` because otherwise the `pymatgen` build will fail because `bdist_wheel`
+        # will not be available.
+        pip install numpy==1.21.4 wheel
+        pip install pymatgen==2022.0.16 --no-cache-dir --no-build-isolation
+
     - name: Install aiida-core
       run: |
-        pip install --upgrade pip setuptools
         pip install -r requirements/requirements-py-3.8.txt
         pip install --no-deps -e .
     - name: Run sub-set of test suite
@@ -105,12 +130,12 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.8'
-    - name: Build package
+    - name: install flit
       run: |
-        pip install wheel
-        python setup.py sdist bdist_wheel
-    - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.1.0
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_KEY }}
+        pip install flit~=3.4
+    - name: Build and publish
+      run: |
+        flit publish
+      env:
+        FLIT_USERNAME: __token__
+        FLIT_PASSWORD: ${{ secrets.PYPI_KEY }}

.github/workflows/setup.sh

@@ -19,18 +19,18 @@ sed -i "s|PLACEHOLDER_WORK_DIR|${GITHUB_WORKSPACE}|" "${CONFIG}/localhost.yaml"
 sed -i "s|PLACEHOLDER_REMOTE_ABS_PATH_DOUBLER|${CONFIG}/doubler.sh|" "${CONFIG}/doubler.yaml"
 sed -i "s|PLACEHOLDER_SSH_KEY|${HOME}/.ssh/slurm_rsa|" "${CONFIG}/slurm-ssh-config.yaml"
 
-verdi setup --config "${CONFIG}/profile.yaml"
+verdi setup --non-interactive --config "${CONFIG}/profile.yaml"
 
 # set up localhost computer
-verdi computer setup --config "${CONFIG}/localhost.yaml"
+verdi computer setup --non-interactive --config "${CONFIG}/localhost.yaml"
 verdi computer configure core.local localhost --config "${CONFIG}/localhost-config.yaml"
 verdi computer test localhost
-verdi code setup --config "${CONFIG}/doubler.yaml"
-verdi code setup --config "${CONFIG}/add.yaml"
+verdi code setup --non-interactive --config "${CONFIG}/doubler.yaml"
+verdi code setup --non-interactive --config "${CONFIG}/add.yaml"
 
 # set up slurm-ssh computer
-verdi computer setup --config "${CONFIG}/slurm-ssh.yaml"
-verdi computer configure core.ssh slurm-ssh --config "${CONFIG}/slurm-ssh-config.yaml" -n  # needs slurm container
+verdi computer setup --non-interactive --config "${CONFIG}/slurm-ssh.yaml"
+verdi computer configure core.ssh slurm-ssh --non-interactive --config "${CONFIG}/slurm-ssh-config.yaml" -n  # needs slurm container
 verdi computer test slurm-ssh --print-traceback
 
 verdi profile setdefault test_${AIIDA_TEST_BACKEND}