Refactored CI (#2122)

* Refactored CI

* Added test-integration.txt for integration tests

* Removed .tox from "make clean"

* Fixed pypy

* Cleanup

* Added "--reruns 2" to integration tests

.github/workflows/linter.yml

@@ -0,0 +1,40 @@
+name: Linter
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  pre-commit:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    steps:
+
+      - name: Checkout branch
+        uses: actions/checkout@v4
+
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+
+  lint:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+      - name: Check out code from GitHub
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: useblacksmith/setup-python@v6
+        with:
+            python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: pip install --upgrade pip wheel tox tox-docker
+      - name: Run flake8
+        run: tox -v -e flake8 -- -v
+      - name: Run pydocstyle
+        run: tox -v -e pydocstyle -- -v
+      - name: Run apicheck
+        run: tox -v -e apicheck -- -v
+      - name: Run mypy
+        run: tox -v -e mypy -- -v

.github/workflows/python-package.yml

@@ -0,0 +1,120 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Kombu
+
+on:
+  push:
+    branches: [ 'main']
+    paths:
+        - '**.py'
+        - '**.txt'
+        - '.github/workflows/python-package.yml'
+        - '**.toml'
+        - "tox.ini"
+  pull_request:
+    branches: [ 'main' ]
+    paths:
+        - '**.py'
+        - '**.txt'
+        - '**.toml'
+        - '.github/workflows/python-package.yml'
+        - "tox.ini"
+  workflow_dispatch:
+
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  Unit:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        os: ["blacksmith-4vcpu-ubuntu-2204"]
+
+    steps:
+    - name: Install apt packages
+      if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu')
+      run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: useblacksmith/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: 'pip'
+        cache-dependency-path: '**/setup.py'
+
+    - name: Install tox
+      run: python -m pip install --upgrade pip wheel tox tox-docker
+    - name: >
+        Run tox for
+        "${{ matrix.python-version }}-unit"
+      timeout-minutes: 5
+      run: >
+          tox --verbose --verbose -e
+          "${{ matrix.python-version }}-unit" -vv
+
+    - uses: codecov/codecov-action@v4
+      with:
+        flags: unittests # optional
+        fail_ci_if_error: false # optional (default = false)
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true # optional (default = false)
+
+  Integration:
+      needs:
+      - Unit
+      if: needs.Unit.result == 'success'
+
+      runs-on: blacksmith-4vcpu-ubuntu-2204
+      strategy:
+          fail-fast: false
+          matrix:
+              python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+              toxenv: [
+                  'py-amqp',
+                  'py-redis',
+                  'py-mongodb',
+                  'py-kafka'
+              ]
+              experimental: [false]
+              include:
+                - python-version: pypy3.10
+                  toxenv: 'py-amqp'
+                  experimental: true
+                - python-version: pypy3.10
+                  toxenv: 'py-redis'
+                  experimental: true
+                - python-version: pypy3.10
+                  toxenv: 'py-mongodb'
+                  experimental: true
+                # - python-version: pypy3.10
+                #   toxenv: 'py-kafka'
+                #   experimental: true
+
+      steps:
+          -   name: Install apt packages
+              run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+
+          -   uses: actions/checkout@v4
+          -   name: Set up Python ${{ matrix.python-version }}
+              uses: useblacksmith/setup-python@v6
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  allow-prereleases: true
+                  cache: 'pip'
+                  cache-dependency-path: '**/setup.py'
+          -   name: Install tox
+              run: python -m pip install --upgrade pip wheel tox tox-docker
+          -   name: >
+                  Run tox for
+                  "${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}"
+              timeout-minutes: 30
+              run: >
+                  tox --verbose --verbose -e
+                  "${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}" -vv

.gitignore

@@ -36,3 +36,6 @@ venv/
 env
 .eggs
 .python-version
+.coverage.*
+control/
+.env

Makefile

@@ -128,7 +128,7 @@ clean-pyc:
 removepyc: clean-pyc
 
 clean-build:
-	rm -rf build/ dist/ .eggs/ *.egg-info/ .tox/ .coverage cover/
+	rm -rf build/ dist/ .eggs/ *.egg-info/ .coverage cover/
 
 clean-git:
 	$(GIT) clean -xdn

requirements/test-integration.txt

@@ -0,0 +1,2 @@
+pytest-xdist==3.6.1
+pytest-rerunfailures>=11.1.2

requirements/test.txt

@@ -3,3 +3,5 @@ Pyro4==4.82
 pytest-freezer==0.4.8
 pytest-sugar==1.0.0
 pytest==8.3.3
+pre-commit>=3.5.0,<3.6.0; python_version < '3.9'
+pre-commit>=3.8.0; python_version >= '3.9'

tox.ini

@@ -1,13 +1,14 @@
 [tox]
 envlist =
-    {pypy3.9,3.8,3.9,3.10,3.11,3.12}-unit
-    {pypy3.9,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
-    {pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-redis
-    {pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-mongodb
-    {3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-unit
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-redis
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-mongodb
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
     flake8
     apicheck
     pydocstyle
+    mypy
 
 requires =
     tox-docker<=4.1
@@ -17,9 +18,10 @@ requires =
 python =
     3.8: py38
     3.9: py39
-    3.10: py310, mypy
+    3.10: py310
     3.11: py311
     3.12: py312
+    pypy3: pypy3
 
 [testenv]
 sitepackages = False
@@ -28,28 +30,29 @@ passenv =
   DISTUTILS_USE_SDK
 deps=
     -r{toxinidir}/requirements/dev.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
     apicheck,3.8-linux,3.9-linux,3.10-linux,3.11-linux,3.12-linux: -r{toxinidir}/requirements/extras/confluentkafka.txt
     apicheck,linkcheck: -r{toxinidir}/requirements/docs.txt
     flake8,pydocstyle,mypy: -r{toxinidir}/requirements/pkgutils.txt
+    integration: -r{toxinidir}/requirements/test-integration.txt
 
 commands =
-    unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail {posargs}
-    integration-py-amqp: pytest -xv -E py-amqp t/integration {posargs:-n2}
-    integration-redis: pytest -xv -E redis t/integration {posargs:-n2}
-    integration-mongodb: pytest -xv -E mongodb t/integration {posargs:-n2}
-    integration-kafka: pytest -xv -E kafka t/integration {posargs:-n2}
+    unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail --cov-report term {posargs}
+    integration-py-amqp: pytest -xv -E py-amqp t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-redis: pytest -xv -E redis t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-mongodb: pytest -xv -E mongodb t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-kafka: pytest -xv -E kafka t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
 
 basepython =
-    pypy3.9: pypy3.9
-    pypy3.8: pypy3.8
+    pypy3: pypy3
     3.8: python3.8
     3.9: python3.9
-    3.10,apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.10
+    3.10: python3.10
     3.11: python3.11
     3.12: python3.12
+    apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.12
 
 install_command = python -m pip --disable-pip-version-check install {opts} {packages}
 
@@ -135,4 +138,30 @@ commands =
     pydocstyle {toxinidir}/kombu
 
 [testenv:mypy]
-commands = python -m mypy --config-file setup.cfg
+commands = python -m mypy --config-file setup.cfg
+
+[testenv:lint]
+allowlist_externals = pre-commit
+commands =
+    pre-commit {posargs:run --all-files --show-diff-on-failure}
+
+[testenv:clean]
+deps = cleanpy
+allowlist_externals = make
+commands =
+    python -m cleanpy .
+    make clean
+
+[testenv:parallel]
+description = Run all integration tests in parallel
+allowlist_externals = tox
+setenv =
+    PYTHONUNBUFFERED = 1
+    PYTHONDONTWRITEBYTECODE = 1
+commands =
+    tox -e \
+    3.12-linux-integration-py-amqp,\
+    3.12-linux-integration-redis,\
+    3.12-linux-integration-mongodb,\
+    3.12-linux-integration-kafka \
+    -p -o -- --exitfirst -n auto --dist=loadscope {posargs}