push-pr_workflow.yml

name: Python CI

on: [push, pull_request]

jobs:
  Changelog:
    name: CHANGELOG.md updated
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'

    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        fetch-depth: 0 # Checkout the whole history, in case the target is way far behind

    - name: Check if target branch has been merged
      run: |
        if git merge-base --is-ancestor ${{ github.event.pull_request.base.sha }} ${{ github.sha }}; then
          echo "Target branch has been merged into the source branch."
        else
          echo "Target branch has not been merged into the source branch. Please merge in target first."
          exit 1
        fi

    - name: Check that CHANGELOG has been updated
      run: |
        # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution.
        if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '^CHANGELOG.md$'; then
          echo "Thanks for helping keep our CHANGELOG up-to-date!"
        else
          echo "Please update the CHANGELOG.md file with notes on your contribution."
          exit 1
        fi

  Lint:
    runs-on: ubuntu-latest
    env:
      MAX_LINE_LENGTH: 127
      MAX_COMPLEXITY: 15

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.x'

    - name: Check cache
      uses: actions/cache@v2
      with:
        path: ~/.cache/pip
        key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi
        pip3 install --upgrade -r requirements/dev.txt

    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --max-complexity=$MAX_COMPLEXITY --statistics --max-line-length=$MAX_LINE_LENGTH

    - name: Lint with isort
      run: |
        python3 -m isort --check --line-length $MAX_LINE_LENGTH merlin
        python3 -m isort --check --line-length $MAX_LINE_LENGTH tests
        python3 -m isort --check --line-length $MAX_LINE_LENGTH *.py

    - name: Lint with Black
      run: |
        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 merlin
        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 tests
        python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 *.py

    - name: Lint with PyLint
      run: |
        python3 -m pylint merlin --rcfile=setup.cfg --exit-zero
        python3 -m pylint tests --rcfile=setup.cfg --exit-zero

  Common-setup:
    runs-on: ubuntu-latest
    env: 
      GO_VERSION: 1.18.1
      SINGULARITY_VERSION: 3.9.9
      OS: linux
      ARCH: amd64

    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache Python dependencies
      uses: actions/cache@v2
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
        pip3 install -r requirements/dev.txt

    - name: Cache Singularity build artifacts
      uses: actions/cache@v2
      with:
        path: |
          ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION
          /usr/local/go
        key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install singularity
      run: |
        sudo apt-get update && sudo apt-get install -y \
          build-essential \
          libssl-dev \
          uuid-dev \
          libgpgme11-dev \
          squashfs-tools \
          libseccomp-dev \
          pkg-config
        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
        rm go$GO_VERSION.$OS-$ARCH.tar.gz
        export PATH=$PATH:/usr/local/go/bin
        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
        cd singularity-ce-$SINGULARITY_VERSION
        ./mconfig && \
          make -C ./builddir && \
          sudo make -C ./builddir install

    - name: Install merlin to run unit tests
      run: |
        pip3 install -e .
        merlin config

    - name: Install CLI task dependencies generated from the 'feature demo' workflow
      run: |
        merlin example feature_demo
        pip3 install -r feature_demo/requirements.txt

  Local-test-suite:
    needs: Common-setup
    runs-on: ubuntu-latest
    env: 
      GO_VERSION: 1.18.1
      SINGULARITY_VERSION: 3.9.9
      OS: linux
      ARCH: amd64

    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache Python dependencies
      uses: actions/cache@v2
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
        pip3 install -r requirements/dev.txt

    - name: Cache Singularity build artifacts
      uses: actions/cache@v2
      with:
        path: |
          ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION
          /usr/local/go
        key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install singularity
      run: |
        sudo apt-get update && sudo apt-get install -y \
          build-essential \
          libssl-dev \
          uuid-dev \
          libgpgme11-dev \
          squashfs-tools \
          libseccomp-dev \
          pkg-config
        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
        rm go$GO_VERSION.$OS-$ARCH.tar.gz
        export PATH=$PATH:/usr/local/go/bin
        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
        cd singularity-ce-$SINGULARITY_VERSION
        ./mconfig && \
          make -C ./builddir && \
          sudo make -C ./builddir install

    - name: Install merlin to run unit tests
      run: |
        pip3 install -e .
        merlin config

    - name: Install CLI task dependencies generated from the 'feature demo' workflow
      run: |
        merlin example feature_demo
        pip3 install -r feature_demo/requirements.txt

    - name: Run pytest over unit test suite
      run: |
        python3 -m pytest -v --order-scope=module tests/unit/

    - name: Run integration test suite for local tests
      run: |
        python3 tests/integration/run_tests.py --verbose --local

  # TODO:
  # - create common-setup job again but with correct caches
  # - make local/integration tests depend on common-setup and check caches (see if this works)
  # - see if local tests even need singularity
  #    - likely will for now but won't once server tests are ported to pytest
  Integration-tests:
    needs: Common-setup
    runs-on: ubuntu-latest
    env: 
      GO_VERSION: 1.18.1
      SINGULARITY_VERSION: 3.9.9
      OS: linux
      ARCH: amd64

    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache Python dependencies
      uses: actions/cache@v2
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
        pip3 install -r requirements/dev.txt

    - name: Install merlin
      run: |
        pip3 install -e .
        merlin config

    - name: Cache Singularity build artifacts
      uses: actions/cache@v2
      with:
        path: |
          ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION
          /usr/local/go
        key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install singularity
      run: |
        sudo apt-get update && sudo apt-get install -y \
          build-essential \
          libssl-dev \
          uuid-dev \
          libgpgme11-dev \
          squashfs-tools \
          libseccomp-dev \
          pkg-config
        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
        rm go$GO_VERSION.$OS-$ARCH.tar.gz
        export PATH=$PATH:/usr/local/go/bin
        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
        cd singularity-ce-$SINGULARITY_VERSION
        ./mconfig && \
          make -C ./builddir && \
          sudo make -C ./builddir install

    - name: Install CLI task dependencies generated from the 'feature demo' workflow
      run: |
        merlin example feature_demo
        pip3 install -r feature_demo/requirements.txt

    # TODO remove the --ignore statement once those tests are fixed
    - name: Run integration test suite for distributed tests
      run: |
        pytest --ignore tests/integration/test_celeryadapter.py tests/integration/

  Distributed-test-suite:
    runs-on: ubuntu-latest
    services:
      # rabbitmq:
      #   image: rabbitmq:latest
      #   ports:
      #     - 5672:5672
      #   options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5
      # Label used to access the service container
      redis:
        # Docker Hub image
        image: redis
        # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379

    strategy:
      matrix:
        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Check cache
      uses: actions/cache@v2
      with:
        path: ${{ env.pythonLocation }}
        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}

    - name: Install dependencies
      run: |
        python3 -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
        pip3 install -r requirements/dev.txt

    - name: Install merlin and setup redis as the broker
      run: |
        pip3 install -e .
        merlin config --broker redis

    - name: Install CLI task dependencies generated from the 'feature demo' workflow
      run: |
        merlin example feature_demo
        pip3 install -r feature_demo/requirements.txt

    - name: Run integration test suite for distributed tests
      env:
        REDIS_HOST: redis
        REDIS_PORT: 6379
      run: |
        python3 tests/integration/run_tests.py --verbose --distributed

    # - name: Setup rabbitmq config
    #   run: |
    #     merlin config --test rabbitmq

    # - name: Run integration test suite for rabbitmq
    #   env:
    #     AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }}
    #     RABBITMQ_USER: Jimmy_Space
    #     RABBITMQ_PASS: Alexander_Rules
    #   ports:
    #     - ${{ job.services.rabbitmq.ports['5672'] }}
    #   run: |
    #     python3 tests/integration/run_tests.py --verbose --ids 31 32