Auto-retry tests that fail randomly (#1678)

.config/requirements-test.txt

@@ -8,6 +8,7 @@ pre-commit
 pytest-env
 pytest-mock
 pytest-plus # implements PYTEST_REQPASS
+pytest-rerunfailures
 pytest-subtests
 pytest-xdist
 

.vscode/settings.json

@@ -10,7 +10,7 @@
   "[python]": {
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-      "source.fixAll": true
+      "source.fixAll": "explicit"
     }
   },
   "prettier.enable": false

requirements.txt

@@ -1,8 +1,8 @@
 #
 # This file is autogenerated by pip-compile with Python 3.9
-# To update, run:
+# by the following command:
 #
-#    pip-compile --extra=docs --extra=test --no-annotate --output-file=requirements.txt --resolver=backtracking --strip-extras --unsafe-package=resolvelib --unsafe-package=ruamel-yaml-clib pyproject.toml
+#    pip-compile --all-extras --no-annotate --output-file=requirements.txt --strip-extras pyproject.toml
 #
 ansible-builder==3.0.0
 ansible-compat==4.1.10
@@ -96,6 +96,7 @@ pytest==7.4.4
 pytest-env==1.1.3
 pytest-mock==3.12.0
 pytest-plus==0.6.1
+pytest-rerunfailures==13.0
 pytest-subtests==0.11.0
 pytest-xdist==3.5.0
 python-daemon==3.0.1

tests/integration/actions/images/test_direct_interactive_noee.py

@@ -26,6 +26,7 @@
 steps = add_indices(initial_steps + base_steps)
 
 
+@pytest.mark.flaky(reruns=4)
 @pytest.mark.parametrize("step", steps, ids=step_id_padded)
 class Test(BaseClass):
     """Run the tests for images from CLI, interactive, without an EE."""

tests/integration/actions/lint/base.py

@@ -43,6 +43,7 @@ def fixture_tmux_session(request: pytest.FixtureRequest):
         with TmuxSession(**params) as tmux_session:
             yield tmux_session
 
+    @pytest.mark.flaky(reruns=2)
     def test(self, request: pytest.FixtureRequest, tmux_session: TmuxSession, step: UiTestStep):
         """Run the tests for lint, mode and ``ee`` set in child class.
 

tests/integration/actions/run_unicode/base.py

@@ -59,6 +59,7 @@ def fixture_tmux_session(request: pytest.FixtureRequest):
         with TmuxSession(**params) as tmux_session:
             yield tmux_session
 
+    @pytest.mark.flaky(reruns=2)
     def test(
         self,
         request: pytest.FixtureRequest,

tests/unit/logger/test_time_zone.py

@@ -54,6 +54,7 @@ def args(self, log_file: Path) -> list[str]:
 )
 
 
+@pytest.mark.flaky(reruns=2)
 @pytest.mark.parametrize("data", test_data, ids=id_func)
 def test_tz_support(
     data: Scenario,

tox.ini

@@ -78,7 +78,7 @@ setenv =
   # without his upgrade would likely not do anything
   PIP_CONSTRAINT = /dev/null
 commands =
-  pip-compile -q --upgrade --no-annotate --extra test --extra docs --output-file=requirements.txt --strip-extras setup.cfg
+  pip-compile -q --upgrade --no-annotate --all-extras --output-file=requirements.txt --strip-extras pyproject.toml
 
 [testenv:report]
 description = Produce coverage report