Testing/manager #589
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python CI | |
| on: [push, pull_request] | |
| jobs: | |
| Changelog: | |
| name: CHANGELOG.md updated | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Checkout the whole history, in case the target is way far behind | |
| - name: Check if target branch has been merged | |
| run: | | |
| if git merge-base --is-ancestor ${{ github.event.pull_request.base.sha }} ${{ github.sha }}; then | |
| echo "Target branch has been merged into the source branch." | |
| else | |
| echo "Target branch has not been merged into the source branch. Please merge in target first." | |
| exit 1 | |
| fi | |
| - name: Check that CHANGELOG has been updated | |
| run: | | |
| # If this step fails, this means you haven't updated the CHANGELOG.md file with notes on your contribution. | |
| if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '^CHANGELOG.md$'; then | |
| echo "Thanks for helping keep our CHANGELOG up-to-date!" | |
| else | |
| echo "Please update the CHANGELOG.md file with notes on your contribution." | |
| exit 1 | |
| fi | |
| Lint: | |
| runs-on: ubuntu-latest | |
| env: | |
| MAX_LINE_LENGTH: 127 | |
| MAX_COMPLEXITY: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Check cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi | |
| pip3 install --upgrade -r requirements/dev.txt | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --max-complexity=$MAX_COMPLEXITY --statistics --max-line-length=$MAX_LINE_LENGTH | |
| - name: Lint with isort | |
| run: | | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH merlin | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH tests | |
| python3 -m isort --check --line-length $MAX_LINE_LENGTH *.py | |
| - name: Lint with Black | |
| run: | | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 merlin | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 tests | |
| python3 -m black --check --line-length $MAX_LINE_LENGTH --target-version py38 *.py | |
| - name: Lint with PyLint | |
| run: | | |
| python3 -m pylint merlin --rcfile=setup.cfg --exit-zero | |
| python3 -m pylint tests --rcfile=setup.cfg --exit-zero | |
| Local-test-suite: | |
| runs-on: ubuntu-latest | |
| env: | |
| GO_VERSION: 1.18.1 | |
| SINGULARITY_VERSION: 3.9.9 | |
| OS: linux | |
| ARCH: amd64 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip3 install -r requirements/dev.txt | |
| - name: Install singularity | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| uuid-dev \ | |
| libgpgme11-dev \ | |
| squashfs-tools \ | |
| libseccomp-dev \ | |
| pkg-config | |
| wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz | |
| sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz | |
| rm go$GO_VERSION.$OS-$ARCH.tar.gz | |
| export PATH=$PATH:/usr/local/go/bin | |
| wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| cd singularity-ce-$SINGULARITY_VERSION | |
| ./mconfig && \ | |
| make -C ./builddir && \ | |
| sudo make -C ./builddir install | |
| - name: Install merlin to run unit tests | |
| run: | | |
| pip3 install -e . | |
| merlin config | |
| - name: Install CLI task dependencies generated from the 'feature demo' workflow | |
| run: | | |
| merlin example feature_demo | |
| pip3 install -r feature_demo/requirements.txt | |
| - name: Run integration test suite for local tests | |
| run: | | |
| python3 tests/integration/run_tests.py --verbose --local | |
| Unit-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| GO_VERSION: 1.18.1 | |
| SINGULARITY_VERSION: 3.9.9 | |
| OS: linux | |
| ARCH: amd64 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip3 install -r requirements/dev.txt | |
| - name: Install singularity | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| uuid-dev \ | |
| libgpgme11-dev \ | |
| squashfs-tools \ | |
| libseccomp-dev \ | |
| pkg-config | |
| wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz | |
| sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz | |
| rm go$GO_VERSION.$OS-$ARCH.tar.gz | |
| export PATH=$PATH:/usr/local/go/bin | |
| wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| cd singularity-ce-$SINGULARITY_VERSION | |
| ./mconfig && \ | |
| make -C ./builddir && \ | |
| sudo make -C ./builddir install | |
| - name: Install merlin to run unit tests | |
| run: | | |
| pip3 install -e . | |
| merlin config | |
| - name: Install CLI task dependencies generated from the 'feature demo' workflow | |
| run: | | |
| merlin example feature_demo | |
| pip3 install -r feature_demo/requirements.txt | |
| - name: Run pytest over unit test suite | |
| run: | | |
| python3 -m pytest -v --order-scope=module tests/unit/ | |
| Integration-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| GO_VERSION: 1.18.1 | |
| SINGULARITY_VERSION: 3.9.9 | |
| OS: linux | |
| ARCH: amd64 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Check cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip3 install -r requirements/dev.txt | |
| - name: Install merlin | |
| run: | | |
| pip3 install -e . | |
| merlin config | |
| - name: Install singularity | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y \ | |
| build-essential \ | |
| libssl-dev \ | |
| uuid-dev \ | |
| libgpgme11-dev \ | |
| squashfs-tools \ | |
| libseccomp-dev \ | |
| pkg-config | |
| wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz | |
| sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz | |
| rm go$GO_VERSION.$OS-$ARCH.tar.gz | |
| export PATH=$PATH:/usr/local/go/bin | |
| wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz | |
| cd singularity-ce-$SINGULARITY_VERSION | |
| ./mconfig && \ | |
| make -C ./builddir && \ | |
| sudo make -C ./builddir install | |
| - name: Install CLI task dependencies generated from the 'feature demo' workflow | |
| run: | | |
| merlin example feature_demo | |
| pip3 install -r feature_demo/requirements.txt | |
| # TODO remove the --ignore statement once those tests are fixed | |
| - name: Run integration test suite for distributed tests | |
| run: | | |
| python3 -m pytest -v --ignore tests/integration/test_celeryadapter.py tests/integration/ |