From 8c8339943b1d8d950350828e2533dca08ff7f2e0 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Thu, 18 Jul 2024 09:56:28 +0300 Subject: [PATCH 1/2] fix: Disable auto-unsupported for single server/client tests It doesn't make sense to automatically claim a test as unsupported if we're only testing a single implementation. --- interop.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/interop.py b/interop.py index 41fd699f..fd8f8779 100644 --- a/interop.py +++ b/interop.py @@ -184,7 +184,9 @@ def _postprocess_results(self): questionable = [TestResult.FAILED, TestResult.UNSUPPORTED] for c in set(clients) - set(self._no_auto_unsupported): for t in self._tests: - if all(self.test_results[s][c][t] in questionable for s in servers): + if len(servers) > 1 and all( + self.test_results[s][c][t] in questionable for s in servers + ): print( f'Client {c} failed test "{t.name()}" against all servers, ' + 'marking the entire test as "unsupported"' @@ -194,7 +196,9 @@ def _postprocess_results(self): # If a server failed a test against all clients, make the test unsupported for the server for s in set(servers) - set(self._no_auto_unsupported): for t in self._tests: - if all(self.test_results[s][c][t] in questionable for c in clients): + if len(clients) > 1 and all( + self.test_results[s][c][t] in questionable for c in clients + ): print( f'Server {s} failed test "{t.name()}" against all clients, ' + 'marking the entire test as "unsupported"' From d31fa5415d22272fdd261fe77e8a301983e9ab1f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Sun, 3 Nov 2024 13:27:47 +0000 Subject: [PATCH 2/2] Change logic --- interop.py | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/interop.py b/interop.py index fd8f8779..d6b27cea 100644 --- a/interop.py +++ b/interop.py @@ -180,31 +180,29 @@ def _check_impl_is_compliant(self, name: str) -> bool: def _postprocess_results(self): clients = list(set(client for client, _ in self._client_server_pairs)) servers = list(set(server for _, server in self._client_server_pairs)) - # If a client failed a test against all servers, make the test unsupported for the client questionable = [TestResult.FAILED, TestResult.UNSUPPORTED] - for c in set(clients) - set(self._no_auto_unsupported): - for t in self._tests: - if len(servers) > 1 and all( - self.test_results[s][c][t] in questionable for s in servers - ): - print( - f'Client {c} failed test "{t.name()}" against all servers, ' - + 'marking the entire test as "unsupported"' - ) - for s in servers: - self.test_results[s][c][t] = TestResult.UNSUPPORTED + # If a client failed a test against all servers, make the test unsupported for the client + if len(servers) > 1: + for c in set(clients) - set(self._no_auto_unsupported): + for t in self._tests: + if all(self.test_results[s][c][t] in questionable for s in servers): + print( + f"Client {c} failed or did not support test {t.name()} " + + 'against all servers, marking the entire test as "unsupported"' + ) + for s in servers: + self.test_results[s][c][t] = TestResult.UNSUPPORTED # If a server failed a test against all clients, make the test unsupported for the server - for s in set(servers) - set(self._no_auto_unsupported): - for t in self._tests: - if len(clients) > 1 and all( - self.test_results[s][c][t] in questionable for c in clients - ): - print( - f'Server {s} failed test "{t.name()}" against all clients, ' - + 'marking the entire test as "unsupported"' - ) - for c in clients: - self.test_results[s][c][t] = TestResult.UNSUPPORTED + if len(clients) > 1: + for s in set(servers) - set(self._no_auto_unsupported): + for t in self._tests: + if all(self.test_results[s][c][t] in questionable for c in clients): + print( + f"Server {s} failed or did not support test {t.name()} " + + 'against all clients, marking the entire test as "unsupported"' + ) + for c in clients: + self.test_results[s][c][t] = TestResult.UNSUPPORTED def _print_results(self): """print the interop table"""