From b21ea60b74aeee8b98b38e9836cf63e4db8b7316 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Fri, 7 Jul 2023 23:09:47 -0700 Subject: [PATCH 1/2] Treat 'PRECONDITION_FAILED' as 'PASS' for interop scoring purposes 'PRECONDITION_FAILED' means that the condition tested by `assert_implements_optional` is `false`. For instance, a test might test multiple times the same feature with different video/audio codecs that are optional. If the codec is not supported, 'PRECONDITION_FAILED' would be returned as status. https://web-platform-tests.org/writing-tests/testharness-api.html#optional-features This is different from the API not being supported, in that case, `assert_implements_optional` would fail with an exception (since it couldn't evaluate the condition), and the status would be 'ERROR'. --- interop-scoring/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interop-scoring/main.js b/interop-scoring/main.js index 5651e8beb..752c75a68 100644 --- a/interop-scoring/main.js +++ b/interop-scoring/main.js @@ -310,12 +310,12 @@ function scoreRuns(runs, allTestsSet) { } subtestTotal = results['subtests'].length; for (const subtest of results['subtests']) { - if (subtest['status'] == 'PASS') { + if (subtest['status'] == 'PASS' || subtest['status'] == 'PRECONDITION_FAILED') { subtestPasses += 1; } } } else { - if (results['status'] == 'PASS') { + if (results['status'] == 'PASS' || subtest['status'] == 'PRECONDITION_FAILED') { subtestPasses = 1; } } From 64b12238c03b5b2b0ff7c47bb304ab6ec488a017 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Fri, 7 Jul 2023 23:12:11 -0700 Subject: [PATCH 2/2] Fix typo --- interop-scoring/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interop-scoring/main.js b/interop-scoring/main.js index 752c75a68..b9cb94b65 100644 --- a/interop-scoring/main.js +++ b/interop-scoring/main.js @@ -315,7 +315,7 @@ function scoreRuns(runs, allTestsSet) { } } } else { - if (results['status'] == 'PASS' || subtest['status'] == 'PRECONDITION_FAILED') { + if (results['status'] == 'PASS' || results['status'] == 'PRECONDITION_FAILED') { subtestPasses = 1; } }