bpo-46523: fix tests rerun when setUp[Class|Module] fails

Lib/test/libregrtest/main.py

@@ -28,6 +28,14 @@
 # Must be smaller than buildbot "1200 seconds without output" limit.
 EXIT_TIMEOUT = 120.0
 
+# bpo-46523: When rerunning tests, we might need to rerun the whole
+# class or module suite if some its life-cycle hooks fail.
+_TEST_LIFECYCLE_HOOKS = frozenset((
+    'setUp', 'tearDown',
+    'setUpClass', 'tearDownClass',
+    'setUpModule', 'tearDownModule',
+))
+
 
 class Regrtest:
     """Execute a test suite.
@@ -322,8 +330,12 @@ def rerun_failed_tests(self):
 
             errors = result.errors or []
             failures = result.failures or []
-            error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
-            failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
+            error_names = [
+                self.normalize_test_name(test_full_name, is_error=True)
+                for (test_full_name, *_) in errors]
+            failure_names = [
+                self.normalize_test_name(test_full_name)
+                for (test_full_name, *_) in failures]
             self.ns.verbose = True
             orig_match_tests = self.ns.match_tests
             if errors or failures:
@@ -349,6 +361,16 @@ def rerun_failed_tests(self):
 
         self.display_result()
 
+    def normalize_test_name(self, test_full_name, is_error=False):
+        short_name = test_full_name.split(" ")[0]
+        if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
+            # This means that we have a failure in a life-cycle hook,
+            # we need to rerun the whole module or class suite.
+            lpar = test_full_name.index('(')
+            rpar = test_full_name.index(')')
+            return test_full_name[lpar + 1: rpar].split('.')[-1]
+        return short_name
+
     def display_result(self):
         # If running the test suite for PGO then no one cares about results.
         if self.ns.pgo:

Lib/test/support/__init__.py

@@ -1089,7 +1089,7 @@ def _run_suite(suite):
     if junit_xml_list is not None:
         junit_xml_list.append(result.get_xml_element())
 
-    if not result.testsRun and not result.skipped:
+    if not result.testsRun and not result.skipped and not result.errors:
         raise TestDidNotRun
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures: