From 3a1fc4ec28af79b03a8e92f4a4e3df9f63d1c11d Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 6 Feb 2025 21:07:08 -0500 Subject: [PATCH] NoJira: Support approved regression for new tests --- hack/gen-resolved-issue.py | 13 ++++++++----- pkg/api/componentreadiness/component_report.go | 11 +++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/hack/gen-resolved-issue.py b/hack/gen-resolved-issue.py index 4f99b2303..323503c3d 100755 --- a/hack/gen-resolved-issue.py +++ b/hack/gen-resolved-issue.py @@ -822,12 +822,15 @@ def triage_regressions(regressed_tests, triaged_incidents, issue_url, currrent_c if args.intentional_regressions: #get the data from json_data + # default to 95% pass rate if not if "base_stats" not in json_data or "sample_stats" not in json_data: - continue - - intentionalRegression["PreviousSuccesses"] = json_data["base_stats"]["success_count"] - intentionalRegression["PreviousFailures"] = json_data["base_stats"]["failure_count"] - intentionalRegression["PreviousFlakes"] = json_data["base_stats"]["flake_count"] + intentionalRegression["PreviousSuccesses"] = 95 + intentionalRegression["PreviousFailures"] = 5 + intentionalRegression["PreviousFlakes"] = 0 + else: + intentionalRegression["PreviousSuccesses"] = json_data["base_stats"]["success_count"] + intentionalRegression["PreviousFailures"] = json_data["base_stats"]["failure_count"] + intentionalRegression["PreviousFlakes"] = json_data["base_stats"]["flake_count"] intentionalRegression["RegressedSuccesses"] = json_data["sample_stats"]["success_count"] intentionalRegression["RegressedFailures"] = json_data["sample_stats"]["failure_count"] diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index 2ea7598cc..fb397d35f 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -1549,6 +1549,12 @@ func (c *componentReportGenerator) assessComponentStatus( if baseTotal == 0 && c.RequestAdvancedOptions.PassRateRequiredNewTests > 0 { // If we have no base stats, fall back to a raw pass rate comparison for new or improperly renamed tests: + //swap out sample with approvedRegression if we have it + if approvedRegression != nil { + sampleFailure = approvedRegression.PreviousFailures + sampleFlake = approvedRegression.PreviousFlakes + sampleSuccess = approvedRegression.PreviousSuccesses + } testStats := c.buildPassRateTestStats(sampleSuccess, sampleFailure, sampleFlake, float64(c.RequestAdvancedOptions.PassRateRequiredNewTests)) // If a new test reports no regression, and we're not using pass rate mode for all tests, we alter @@ -1559,6 +1565,11 @@ func (c *componentReportGenerator) assessComponentStatus( return testStats } else if c.RequestAdvancedOptions.PassRateRequiredAllTests > 0 { // If requested, switch to pass rate only testing to see what does not meet the criteria: + if approvedRegression != nil { + sampleFailure = approvedRegression.PreviousFailures + sampleFlake = approvedRegression.PreviousFlakes + sampleSuccess = approvedRegression.PreviousSuccesses + } testStats := c.buildPassRateTestStats(sampleSuccess, sampleFailure, sampleFlake, float64(c.RequestAdvancedOptions.PassRateRequiredAllTests)) // include base stats even though we didn't do fishers exact here, this is helpful