From 3448887f787ff6bfe8cce0516360c6ef1aa85b5f Mon Sep 17 00:00:00 2001
From: Tomer Nosrati <tomer.nosrati@gmail.com>
Date: Wed, 25 Sep 2024 21:10:28 +0300
Subject: [PATCH] Refactored CI (#2122)

* Refactored CI

* Added test-integration.txt for integration tests

* Removed .tox from "make clean"

* Fixed pypy

* Cleanup

* Added "--reruns 2" to integration tests
---
 .github/workflows/ci.yaml            |  86 -------------------
 .github/workflows/linter.yml         |  40 +++++++++
 .github/workflows/python-package.yml | 120 +++++++++++++++++++++++++++
 .gitignore                           |   3 +
 Makefile                             |   2 +-
 requirements/test-integration.txt    |   2 +
 requirements/test.txt                |   2 +
 tox.ini                              |  65 +++++++++++----
 8 files changed, 215 insertions(+), 105 deletions(-)
 delete mode 100644 .github/workflows/ci.yaml
 create mode 100644 .github/workflows/linter.yml
 create mode 100644 .github/workflows/python-package.yml
 create mode 100644 requirements/test-integration.txt

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
deleted file mode 100644
index c4d4e3065..000000000
--- a/.github/workflows/ci.yaml
+++ /dev/null
@@ -1,86 +0,0 @@
-name: CI
-on: [pull_request, push]
-jobs:
-  #################### Unittests ####################
-  unittest:
-    runs-on: blacksmith-4vcpu-ubuntu-2204
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-    steps:
-      - name: Install system packages
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - name: Check out code from GitHub
-        uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        id: python
-        uses: useblacksmith/setup-python@v6
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install --upgrade pip wheel tox tox-docker
-      - name: Run unittest
-        run: tox -v -e ${{ matrix.python-version }}-linux-unit -- -v
-  #################### Integration tests ####################
-  integration:
-    needs: [unittest]
-    runs-on: blacksmith-4vcpu-ubuntu-2204
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [3.8,3.9,"3.10","3.11","3.12"]
-        experimental: [false]
-        include:
-          - python-version: pypy3.9
-            experimental: true
-    steps:
-      - name: Install system packages
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - name: Check out code from GitHub
-        uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        id: python
-        uses: useblacksmith/setup-python@v6
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install --upgrade pip wheel tox tox-docker
-      # Tox fails if a Python versions contains a hyphen, this changes "pypy-3.9" to "pypy3.9".
-      - name: Run AMQP integration tests
-        run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-amqp -- -v
-      - name: Run redis integration tests
-        run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-redis -- -v
-      - name: Run MongoDB integration tests
-        run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-mongodb -- -v
-      - name: Run kafka integration tests
-        if: ${{ matrix.python-version != 'pypy3.9'}}
-        run: tox -v -e ${{ matrix.python-version }}-linux-integration-py-kafka -- -v
-
-  #################### Linters and checkers ####################
-  lint:
-    needs: [unittest, integration]
-    runs-on: blacksmith-4vcpu-ubuntu-2204
-    strategy:
-      matrix:
-        python-version: ["3.10"]
-    steps:
-      - name: Install system packages
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - name: Check out code from GitHub
-        uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        id: python
-        uses: useblacksmith/setup-python@v6
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install --upgrade pip wheel tox tox-docker
-      - name: Run flake8
-        run: tox -v -e flake8 -- -v
-      - name: Run pydocstyle
-        run: tox -v -e pydocstyle -- -v
-      - name: Run apicheck
-        run: tox -v -e apicheck -- -v
-      - name: Run mypy
-        run: tox -v -e mypy -- -v
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
new file mode 100644
index 000000000..09f498086
--- /dev/null
+++ b/.github/workflows/linter.yml
@@ -0,0 +1,40 @@
+name: Linter
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  pre-commit:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    steps:
+
+      - name: Checkout branch
+        uses: actions/checkout@v4
+
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+
+  lint:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+      - name: Check out code from GitHub
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: useblacksmith/setup-python@v6
+        with:
+            python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: pip install --upgrade pip wheel tox tox-docker
+      - name: Run flake8
+        run: tox -v -e flake8 -- -v
+      - name: Run pydocstyle
+        run: tox -v -e pydocstyle -- -v
+      - name: Run apicheck
+        run: tox -v -e apicheck -- -v
+      - name: Run mypy
+        run: tox -v -e mypy -- -v
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
new file mode 100644
index 000000000..c2e2c86ca
--- /dev/null
+++ b/.github/workflows/python-package.yml
@@ -0,0 +1,120 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Kombu
+
+on:
+  push:
+    branches: [ 'main']
+    paths:
+        - '**.py'
+        - '**.txt'
+        - '.github/workflows/python-package.yml'
+        - '**.toml'
+        - "tox.ini"
+  pull_request:
+    branches: [ 'main' ]
+    paths:
+        - '**.py'
+        - '**.txt'
+        - '**.toml'
+        - '.github/workflows/python-package.yml'
+        - "tox.ini"
+  workflow_dispatch:
+
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  Unit:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        os: ["blacksmith-4vcpu-ubuntu-2204"]
+
+    steps:
+    - name: Install apt packages
+      if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu')
+      run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: useblacksmith/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: 'pip'
+        cache-dependency-path: '**/setup.py'
+
+    - name: Install tox
+      run: python -m pip install --upgrade pip wheel tox tox-docker
+    - name: >
+        Run tox for
+        "${{ matrix.python-version }}-unit"
+      timeout-minutes: 5
+      run: >
+          tox --verbose --verbose -e
+          "${{ matrix.python-version }}-unit" -vv
+
+    - uses: codecov/codecov-action@v4
+      with:
+        flags: unittests # optional
+        fail_ci_if_error: false # optional (default = false)
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true # optional (default = false)
+
+  Integration:
+      needs:
+      - Unit
+      if: needs.Unit.result == 'success'
+
+      runs-on: blacksmith-4vcpu-ubuntu-2204
+      strategy:
+          fail-fast: false
+          matrix:
+              python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+              toxenv: [
+                  'py-amqp',
+                  'py-redis',
+                  'py-mongodb',
+                  'py-kafka'
+              ]
+              experimental: [false]
+              include:
+                - python-version: pypy3.10
+                  toxenv: 'py-amqp'
+                  experimental: true
+                - python-version: pypy3.10
+                  toxenv: 'py-redis'
+                  experimental: true
+                - python-version: pypy3.10
+                  toxenv: 'py-mongodb'
+                  experimental: true
+                # - python-version: pypy3.10
+                #   toxenv: 'py-kafka'
+                #   experimental: true
+
+      steps:
+          -   name: Install apt packages
+              run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+
+          -   uses: actions/checkout@v4
+          -   name: Set up Python ${{ matrix.python-version }}
+              uses: useblacksmith/setup-python@v6
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  allow-prereleases: true
+                  cache: 'pip'
+                  cache-dependency-path: '**/setup.py'
+          -   name: Install tox
+              run: python -m pip install --upgrade pip wheel tox tox-docker
+          -   name: >
+                  Run tox for
+                  "${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}"
+              timeout-minutes: 30
+              run: >
+                  tox --verbose --verbose -e
+                  "${{ matrix.python-version }}-linux-integration-${{ matrix.toxenv }}" -vv
diff --git a/.gitignore b/.gitignore
index 606902625..e03003a6b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,6 @@ venv/
 env
 .eggs
 .python-version
+.coverage.*
+control/
+.env
diff --git a/Makefile b/Makefile
index fbf715d79..93982c3b8 100644
--- a/Makefile
+++ b/Makefile
@@ -128,7 +128,7 @@ clean-pyc:
 removepyc: clean-pyc
 
 clean-build:
-	rm -rf build/ dist/ .eggs/ *.egg-info/ .tox/ .coverage cover/
+	rm -rf build/ dist/ .eggs/ *.egg-info/ .coverage cover/
 
 clean-git:
 	$(GIT) clean -xdn
diff --git a/requirements/test-integration.txt b/requirements/test-integration.txt
new file mode 100644
index 000000000..99a615e1d
--- /dev/null
+++ b/requirements/test-integration.txt
@@ -0,0 +1,2 @@
+pytest-xdist==3.6.1
+pytest-rerunfailures>=11.1.2
diff --git a/requirements/test.txt b/requirements/test.txt
index a6dbfbe28..8aba86caa 100644
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -3,3 +3,5 @@ Pyro4==4.82
 pytest-freezer==0.4.8
 pytest-sugar==1.0.0
 pytest==8.3.3
+pre-commit>=3.5.0,<3.6.0; python_version < '3.9'
+pre-commit>=3.8.0; python_version >= '3.9'
diff --git a/tox.ini b/tox.ini
index 3f48d5b6a..a380920e7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,13 +1,14 @@
 [tox]
 envlist =
-    {pypy3.9,3.8,3.9,3.10,3.11,3.12}-unit
-    {pypy3.9,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
-    {pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-redis
-    {pypy3.9,3.8,3.9,3.10,3.11}-linux-integration-mongodb
-    {3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-unit
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-py-amqp
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-redis
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-mongodb
+    {pypy3.10,3.8,3.9,3.10,3.11,3.12}-linux-integration-kafka
     flake8
     apicheck
     pydocstyle
+    mypy
 
 requires =
     tox-docker<=4.1
@@ -17,9 +18,10 @@ requires =
 python =
     3.8: py38
     3.9: py39
-    3.10: py310, mypy
+    3.10: py310
     3.11: py311
     3.12: py312
+    pypy3: pypy3
 
 [testenv]
 sitepackages = False
@@ -28,28 +30,29 @@ passenv =
   DISTUTILS_USE_SDK
 deps=
     -r{toxinidir}/requirements/dev.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
-    apicheck,pypy3.9,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/default.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test.txt
+    apicheck,pypy3.10,3.8,3.9,3.10,3.11,3.12: -r{toxinidir}/requirements/test-ci.txt
     apicheck,3.8-linux,3.9-linux,3.10-linux,3.11-linux,3.12-linux: -r{toxinidir}/requirements/extras/confluentkafka.txt
     apicheck,linkcheck: -r{toxinidir}/requirements/docs.txt
     flake8,pydocstyle,mypy: -r{toxinidir}/requirements/pkgutils.txt
+    integration: -r{toxinidir}/requirements/test-integration.txt
 
 commands =
-    unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail {posargs}
-    integration-py-amqp: pytest -xv -E py-amqp t/integration {posargs:-n2}
-    integration-redis: pytest -xv -E redis t/integration {posargs:-n2}
-    integration-mongodb: pytest -xv -E mongodb t/integration {posargs:-n2}
-    integration-kafka: pytest -xv -E kafka t/integration {posargs:-n2}
+    unit: python -bb -m pytest -rxs -xv --cov=kombu --cov-report=xml --no-cov-on-fail --cov-report term {posargs}
+    integration-py-amqp: pytest -xv -E py-amqp t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-redis: pytest -xv -E redis t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-mongodb: pytest -xv -E mongodb t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
+    integration-kafka: pytest -xv -E kafka t/integration -n auto --reruns 2 --reruns-delay 1 {posargs}
 
 basepython =
-    pypy3.9: pypy3.9
-    pypy3.8: pypy3.8
+    pypy3: pypy3
     3.8: python3.8
     3.9: python3.9
-    3.10,apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.10
+    3.10: python3.10
     3.11: python3.11
     3.12: python3.12
+    apicheck,pydocstyle,flake8,linkcheck,cov,mypy: python3.12
 
 install_command = python -m pip --disable-pip-version-check install {opts} {packages}
 
@@ -135,4 +138,30 @@ commands =
     pydocstyle {toxinidir}/kombu
 
 [testenv:mypy]
-commands = python -m mypy --config-file setup.cfg
\ No newline at end of file
+commands = python -m mypy --config-file setup.cfg
+
+[testenv:lint]
+allowlist_externals = pre-commit
+commands =
+    pre-commit {posargs:run --all-files --show-diff-on-failure}
+
+[testenv:clean]
+deps = cleanpy
+allowlist_externals = make
+commands =
+    python -m cleanpy .
+    make clean
+
+[testenv:parallel]
+description = Run all integration tests in parallel
+allowlist_externals = tox
+setenv =
+    PYTHONUNBUFFERED = 1
+    PYTHONDONTWRITEBYTECODE = 1
+commands =
+    tox -e \
+    3.12-linux-integration-py-amqp,\
+    3.12-linux-integration-redis,\
+    3.12-linux-integration-mongodb,\
+    3.12-linux-integration-kafka \
+    -p -o -- --exitfirst -n auto --dist=loadscope {posargs}