Update dependency management workflow

.github/CODEOWNERS

@@ -0,0 +1,10 @@
+# All files related to dependency management are owned by the
+# currently active dependency manager (DM) to trigger an automatic review
+# request from the DM upon changes. Please see AEP-002 for details:
+# https://github.com/aiidateam/AEP/tree/master/002_dependency_management
+setup.*                         @aiidateam/dependency-manager
+environment.yml                 @aiidateam/dependency-manager
+requirements*.txt               @aiidateam/dependency-manager
+pyproject.toml                  @aiidateam/dependency-manager
+utils/dependency_management.py  @aiidateam/dependency-manager
+.github/workflows/dm.yml        @aiidateam/dependency-manager

.github/workflows/ci.yml

@@ -1,28 +1,9 @@
-name: aiida-core
+name: continuous-integration
 
 on: [push, pull_request]
 
 jobs:
 
-  conda:
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-
-    - name: Conda install
-      env:
-        PYTHON_VERSION: 3.7
-      run:
-        .github/workflows/conda.sh
-
   docs:
 
     runs-on: ubuntu-latest
@@ -91,8 +72,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.8]
         backend: ['django', 'sqlalchemy']
+        python-version: [3.5, 3.8]
 
     steps:
     - uses: actions/checkout@master
@@ -119,12 +100,21 @@ jobs:
         sudo apt install postgresql-10 rabbitmq-server graphviz
         sudo systemctl status rabbitmq-server.service
 
-    - name: Install python dependencies
+    - name: Upgrade pip
       run: |
         pip install --upgrade pip
-        pip install numpy==1.17.4
-        pip install -e .[atomic_tools,docs,notebook,rest,testing]
+        pip --version
+
+    - name: upgrade setuptools [py35]
+      if: matrix.python-version == 3.5
+      run: pip install -I setuptools==38.2.0  # Minimally required version for Python 3.5.
+
+    - name: Install aiida-core
+      run: |
+        pip install -r requirements/requirements-py-${{ matrix.python-version }}.txt
+        pip install --no-deps -e .
         reentry scan
+        pip freeze
 
     - name: Setup environment
       env:

.github/workflows/test-install.yml

@@ -0,0 +1,153 @@
+name: test-install
+
+on:
+  push:
+    branch:
+      - master
+      - develop
+      - release/*
+      - dm/*
+    paths:
+      - 'setup.*'
+      - 'environment.yml'
+      - 'requirements*.txt'
+      - 'pyproject.toml'
+      - '.github/workflows/test-install.yml'
+  schedule:
+    - cron: '30 02 * * *'  # nightly build
+
+jobs:
+
+  validate-dependency-specification:
+    # Note: The specification is also validated by the pre-commit hook.
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+
+    - name: Install dm-script dependencies
+      run: pip install click~=7.0 pyyaml~=5.1 toml
+
+    - name: Validate
+      run: python ./utils/dependency_management.py validate-all
+
+  install-with-pip:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+
+    - name: Pip install
+      run: |
+        python -m pip install -e .
+        python -m pip freeze
+
+    - name: Test importing aiida
+      run:
+        python -c "import aiida"
+
+  install-with-conda:
+
+    runs-on: ubuntu-latest
+    name: install-with-conda
+
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Setup Conda
+      uses: s-weigand/setup-conda@v1
+      with:
+        update-conda: true
+        python-version: 3.7
+    - run: conda --version
+    - run: python --version
+    - run: which python
+
+    - name: Create conda environment
+      run: |
+        conda env create -f environment.yml -n test-environment
+        source activate test-environment
+        python -m pip install --no-deps -e .
+
+    - name: Test importing aiida
+      run: |
+        source activate test-environment
+        python -c "import aiida"
+
+  tests:
+
+    needs: [install-with-pip, install-with-conda]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+        backend: ['django', 'sqlalchemy']
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: CasperWA/postgresql-action@v1.2
+      with:
+        postgresql version: '10'
+        postgresql db: test_${{ matrix.backend }}
+        postgresql user: postgres
+        postgresql password: ''
+        postgresql auth: trust
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system dependencies
+      run: |
+        wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
+        echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
+        echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
+        sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list
+        sudo apt update
+        sudo apt install postgresql-10 rabbitmq-server graphviz
+        sudo systemctl status rabbitmq-server.service
+
+    - run: pip install --upgrade pip
+
+    - name: upgrade setuptools [py35]
+      if: matrix.python-version == 3.5
+      run: pip install -I setuptools==38.2.0
+
+    - name: Install aiida-core
+      run: |
+        pip install -e .[atomic_tools,docs,notebook,rest,testing]
+        reentry scan
+
+    - run: pip freeze
+
+    - name: Setup AiiDA environment
+      env:
+        AIIDA_TEST_BACKEND: ${{ matrix.backend }}
+      run:
+        .github/workflows/setup.sh
+
+    - name: Run test suite
+      env:
+        AIIDA_TEST_BACKEND: ${{ matrix.backend }}
+      run:
+        .github/workflows/tests.sh

.github/workflows/update-requirements.yml

@@ -0,0 +1,104 @@
+name: update-requirements
+
+on:
+  push:
+    branch:
+      - release/*
+    paths:
+      - 'setup.json'
+      - '.github/workflows/update-requirements.yml'
+
+jobs:
+
+  tests:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ['django', 'sqlalchemy']
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v1
+    - uses: CasperWA/postgresql-action@v1.2
+      with:
+        postgresql version: '10'
+        postgresql db: test_${{ matrix.backend }}
+        postgresql user: postgres
+        postgresql password: ''
+        postgresql auth: trust
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system dependencies
+      run: |
+        wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
+        echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
+        echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
+        sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list
+        sudo apt update
+        sudo apt install postgresql-10 rabbitmq-server graphviz
+        sudo systemctl status rabbitmq-server.service
+
+    - run: pip install --upgrade pip
+
+    - name: upgrade setuptools [py35]
+      if: matrix.python-version == 3.5
+      run: pip install -I setuptools==38.2.0
+
+    - name: Install aiida-core
+      run: |
+        pip install -e .[atomic_tools,docs,notebook,rest,testing]
+        reentry scan
+
+    - name: Setup environment
+      env:
+        AIIDA_TEST_BACKEND: ${{ matrix.backend }}
+      run:
+        .github/workflows/setup.sh
+
+    - name: Run test suite
+      env:
+        AIIDA_TEST_BACKEND: ${{ matrix.backend }}
+      run:
+        .github/workflows/tests.sh
+
+    - name: Freeze test environment
+      run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt
+
+    # Add python-version specific requirements file to the requirements.txt artifact.
+    # Will be used in the next job to create a PR in case they are different from the current version.
+    - uses: actions/upload-artifact@v1
+      if: matrix.backend == 'django'  # The requirements are identical between backends.
+      with:
+        name: requirements.txt
+        path: requirements-py-${{ matrix.python-version }}.txt
+
+  update-requirements:
+
+    needs: tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download requirements.txt files
+        uses: actions/download-artifact@v1
+        with:
+          name: requirements.txt
+          path: requirements
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v2
+        with:
+          commit-message: "Update requirements.txt"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Update requirements.txt"
+          team-reviewers: dependency-manager
+          branch: "dm/update-requirements.txt"

.pre-commit-config.yaml

@@ -111,38 +111,39 @@
     entry: prospector
 
   - id: rtd-requirements
-    name: Requirements for RTD
-    entry: python ./docs/update_req_for_rtd.py --pre-commit
+    name: Validate docs/requirements_for_rtd.txt
+    entry: python ./utils/dependency_management.py validate-rtd-reqs
     language: system
     files: >-
       (?x)^(
         setup.json|
         setup.py|
+        utils/dependency_management.py|
         docs/requirements_for_rtd.txt|
-        docs/update_req_for_rtd.py|
       )$
     pass_filenames: false
 
   - id: pyproject
-    name: Validating pyproject.toml
-    entry: python ./utils/validate_consistency.py toml
+    name: Validate pyproject.toml
+    entry: python ./utils/dependency_management.py validate-pyproject-toml
     language: system
     files: >-
       (?x)^(
         setup.json|
         setup.py|
-        utils/validate_consistency.py|
+        utils/dependency_management.py|
+        pyproject.toml
       )$
     pass_filenames: false
-  - id: conda
-    name: Validating environment.yml
-    entry: python ./utils/validate_consistency.py conda
+  - id: dependencies
+    name: Validate environment.yml
+    entry: python ./utils/dependency_management.py validate-environment-yml
     language: system
     files: >-
       (?x)^(
         setup.json|
         setup.py|
-        utils/validate_consistency.py|
+        utils/dependency_management.py|
         environment.yml|
       )$
     pass_filenames: false