[devops] Run tests with sqlite backend

.github/actions/install-aiida-core/action.yml

@@ -28,7 +28,7 @@ runs:
   - name: Install uv installer
     run: curl --proto '=https' --tlsv1.2 -LsSf https://${{ env.UV_URL }} | sh
     env:
-      UV_VERSION: 0.1.44
+      UV_VERSION: 0.2.5
       UV_URL: github.com/astral-sh/uv/releases/download/$UV_VERSION/uv-installer.sh
     shell: bash
 

.github/workflows/ci-code.yml

@@ -1,4 +1,4 @@
-name: continuous-integration-code
+name: ci-code
 
 on:
   push:
@@ -13,6 +13,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
 
   check-requirements:
@@ -99,7 +102,9 @@ jobs:
       env:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
-      run: pytest --cov aiida --verbose tests -m 'not nightly'
+      # Python 3.12 has a performance regression when running with code coverage
+      # so run code coverage only for python 3.9.
+      run: pytest -v tests -m 'not nightly' ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
 
     - name: Upload coverage report
       if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core'
@@ -110,26 +115,58 @@ jobs:
         file: ./coverage.xml
         fail_ci_if_error: false  # don't fail job, if coverage upload fails
 
-  verdi:
 
+  tests-presto:
+
+    needs: [check-requirements]
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
 
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ['3.9', '3.12']
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install graphviz
+      run: sudo apt update && sudo apt install graphviz
+
+    - name: Install aiida-core
+      uses: ./.github/actions/install-aiida-core
+      with:
+        python-version: '3.11'
+
+    - name: Setup SSH on localhost
+      run: .github/workflows/setup_ssh.sh
+
+    - name: Run test suite
+      env:
+        AIIDA_WARN_v3: 0
+      run: pytest -m 'presto' --cov aiida
+
+    - name: Upload coverage report
+      if: github.repository == 'aiidateam/aiida-core'
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        name: aiida-pytests-presto
+        flags: presto
+        file: ./coverage.xml
+        fail_ci_if_error: false  # don't fail job, if coverage upload fails
+
+
+  verdi:
+
+    needs: [check-requirements]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
       with:
-        python-version: ${{ matrix.python-version }}
-        from-requirements: 'false'
+        python-version: '3.12'
 
-    - name: Run verdi
+    - name: Run verdi tests
       run: |
         verdi devel check-load-time
         verdi devel check-undesired-imports

.github/workflows/ci-style.yml

@@ -1,4 +1,4 @@
-name: continuous-integration-style
+name: ci-style
 
 on:
   push:

.github/workflows/nightly.yml

@@ -19,6 +19,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
 
   nightly-tests:
@@ -91,7 +94,7 @@ jobs:
   rabbitmq-tests:
 
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 10
 
     strategy:
       fail-fast: false
@@ -132,6 +135,12 @@ jobs:
     - name: Install system dependencies
       run: sudo apt update && sudo apt install postgresql
 
+    - name: Setup SSH on localhost
+      run: .github/workflows/setup_ssh.sh
+
+    - name: Suppress RabbitMQ version warning
+      run: verdi config set warnings.rabbitmq_version False
+
     - name: Run tests
       id: tests
       env:

.github/workflows/setup.sh

@@ -1,12 +1,8 @@
 #!/usr/bin/env bash
 set -ev
 
-ssh-keygen -q -t rsa -b 4096 -N "" -f "${HOME}/.ssh/id_rsa"
-ssh-keygen -y -f "${HOME}/.ssh/id_rsa" >> "${HOME}/.ssh/authorized_keys"
-ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts"
-
-# The permissions on the GitHub runner are 777 which will cause SSH to refuse the keys and cause authentication to fail
-chmod 755 "${HOME}"
+# Setup SSH on localhost
+${GITHUB_WORKSPACE}/.github/workflows/setup_ssh.sh
 
 # Replace the placeholders in configuration files with actual values
 CONFIG="${GITHUB_WORKSPACE}/.github/config"

.github/workflows/setup_ssh.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ev
+
+ssh-keygen -q -t rsa -b 4096 -N "" -f "${HOME}/.ssh/id_rsa"
+ssh-keygen -y -f "${HOME}/.ssh/id_rsa" >> "${HOME}/.ssh/authorized_keys"
+ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts"
+
+# The permissions on the GitHub runner are 777 which will cause SSH to refuse the keys and cause authentication to fail
+chmod 755 "${HOME}"

.github/workflows/verdi.sh

@@ -1,14 +1,13 @@
 #!/usr/bin/env bash
 
-# Test the loading time of `verdi`. This is and attempt to catch changes to the imports in `aiida.cmdline` that will
-# indirectly load the `aiida.orm` module which will trigger loading of the backend environment. This slows down `verdi`
-# significantly, making tab-completion unusable.
+# Test the loading time of `verdi`. This is an attempt to catch changes to the imports in `aiida.cmdline` that
+# would slow down `verdi` invocations and make tab-completion unusable.
 VERDI=`which verdi`
 
-# Typically, the loading time of `verdi` should be around ~0.2 seconds. When loading the database environment this
-# tends to go towards ~0.8 seconds. Since these timings are obviously machine and environment dependent, typically these
-# types of tests are fragile. But with a load limit of more than twice the ideal loading time, if exceeded, should give
-# a reasonably sure indication that the loading of `verdi` is unacceptably slowed down.
+# Typically, the loading time of `verdi` should be around ~0.2 seconds.
+# Typically these types of tests are fragile. But with a load limit of more than twice
+# the ideal loading time, if exceeded, should give a reasonably sure indication
+# that the loading of `verdi` is unacceptably slowed down.
 LOAD_LIMIT=0.4
 MAX_NUMBER_ATTEMPTS=5
 

docs/source/nitpick-exceptions

@@ -192,8 +192,11 @@ py:class _asyncio.Future
 
 py:class tqdm.std.tqdm
 
+py:class _pytest.tmpdir.TempPathFactory
+py:class pytest.tmpdir.TempPathFactory
 py:class pytest.TempPathFactory
 py:class PGTest
+py:class pgtest.pgtest.PGTest
 
 py:class IPython.core.magic.Magics
 
@@ -207,8 +210,6 @@ py:class flask_restful.Resource
 py:class flask.app.Flask
 py:class Flask
 
-py:class pytest.tmpdir.TempPathFactory
-
 py:class scoped_session
 py:class sqlalchemy.orm.decl_api.SqliteModel
 py:class sqlalchemy.orm.decl_api.Base

pyproject.toml

@@ -351,25 +351,33 @@ ignore_errors = true
 module = 'plumpy'
 
 [tool.pytest.ini_options]
-addopts = '--benchmark-skip --durations=50 --strict-config --strict-markers -ra --cov-report xml --cov-append '
+addopts = '--benchmark-skip --durations=50 --durations-min=1 --strict-config --strict-markers -ra --cov-report xml --cov-append '
 filterwarnings = [
+  'ignore:.*and will be removed in NumPy 2.0.*:DeprecationWarning:ase:',
+  'ignore:datetime.datetime.utcfromtimestamp:DeprecationWarning:pytz|dateutil|tqdm|aio_pika:',
+  'ignore:datetime.datetime.utcnow:DeprecationWarning:aio_pika|pytz|pgtest:',
   'ignore::DeprecationWarning:babel:',
   'ignore::DeprecationWarning:frozendict:',
   'ignore::DeprecationWarning:sqlalchemy:',
   'ignore::DeprecationWarning:yaml:',
   'ignore::DeprecationWarning:pymatgen:',
   'ignore::DeprecationWarning:jsonbackend:',
   'ignore::DeprecationWarning:pkg_resources:',
+  'ignore:ast.* is deprecated.*:DeprecationWarning:docstring_parser:',
+  'ignore::SyntaxWarning:CifFile:',
   'ignore::pytest.PytestCollectionWarning',
   'ignore:Creating AiiDA configuration folder.*:UserWarning',
   'ignore:Object of type .* not in session, .* operation along .* will not proceed:sqlalchemy.exc.SAWarning',
+  'ignore:Identity map already had an identity for .* inside of an event handler within the flush?:sqlalchemy.exc.SAWarning',
   'ignore:The `aiida.orm.nodes.data.upf` module is deprecated.*:aiida.common.warnings.AiidaDeprecationWarning',
   'ignore:The `Code` class is deprecated.*:aiida.common.warnings.AiidaDeprecationWarning',
   'default::ResourceWarning'
 ]
 markers = [
   'nightly: long running tests that should rarely be affected and so only run nightly',
   'requires_rmq: requires a connection (on port 5672) to RabbitMQ',
+  'requires_psql: requires a connection to PostgreSQL DB',
+  'presto: automatic marker meaning "not requires_rmq and not requires_psql"',
   'sphinx: set parameters for the sphinx `app` fixture'
 ]
 minversion = '7.0'

requirements/requirements-py-3.10.txt

@@ -199,7 +199,6 @@ tomli==2.0.1
 tornado==6.3.2
 tqdm==4.65.0
 traitlets==5.9.0
-trogon==0.5.0
 typing-extensions==4.6.3
 tzdata==2023.3
 uc-micro-py==1.0.2

requirements/requirements-py-3.11.txt

@@ -197,7 +197,6 @@ tinycss2==1.2.1
 tornado==6.3.2
 tqdm==4.65.0
 traitlets==5.9.0
-trogon==0.5.0
 typing-extensions==4.6.3
 tzdata==2023.3
 uc-micro-py==1.0.2

requirements/requirements-py-3.9.txt

@@ -201,7 +201,6 @@ tomli==2.0.1
 tornado==6.3.2
 tqdm==4.65.0
 traitlets==5.9.0
-trogon==0.5.0
 typing-extensions==4.6.3
 tzdata==2023.3
 uc-micro-py==1.0.2

src/aiida/storage/psql_dos/migrations/utils/integrity.py

@@ -161,7 +161,7 @@
     :param reason_message: a human readable message detailing the reason of the integrity violation
     :param action_message: an optional human readable message detailing a performed action, if any
     """
-    from datetime import datetime
+    from datetime import datetime, timezone
     from tempfile import NamedTemporaryFile
 
     from tabulate import tabulate
@@ -183,7 +183,7 @@
             )
         )
 
-        handle.write(f'# {datetime.utcnow().isoformat()}\n')
+        handle.write(f'# {datetime.datetime.now(timezone.utc).isoformat()}\n')
         handle.write(f'# Violation reason: {reason_message}\n')
         handle.write(f'# Performed action: {action_message}\n')
         handle.write('\n')

src/aiida/tools/pytest_fixtures/__init__.py

@@ -25,7 +25,7 @@
     aiida_localhost,
     ssh_key,
 )
-from .storage import config_psql_dos, postgres_cluster
+from .storage import config_psql_dos, config_sqlite_dos, postgres_cluster
 
 __all__ = (
     'aiida_code_installed',
@@ -44,6 +44,7 @@
     'aiida_profile_tmp',
     'aiida_profile',
     'config_psql_dos',
+    'config_sqlite_dos',
     'daemon_client',
     'entry_points',
     'postgres_cluster',