diff --git a/tools/flakeguard/cmd/aggregate.go b/tools/flakeguard/cmd/aggregate.go new file mode 100644 index 000000000..45e0b4ad0 --- /dev/null +++ b/tools/flakeguard/cmd/aggregate.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports" + "github.com/spf13/cobra" +) + +var AggregateAllCmd = &cobra.Command{ + Use: "aggregate-all", + Short: "Aggregate all test results and output them to a file", + Run: func(cmd *cobra.Command, args []string) { + resultsFolderPath, _ := cmd.Flags().GetString("results-path") + outputPath, _ := cmd.Flags().GetString("output-json") + + // Aggregate all test results + allResults, err := reports.AggregateTestResults(resultsFolderPath) + if err != nil { + log.Fatalf("Error aggregating results: %v", err) + } + + // Output all results to JSON file + if outputPath != "" && len(allResults) > 0 { + if err := saveResults(outputPath, allResults); err != nil { + log.Fatalf("Error writing aggregated results to file: %v", err) + } + fmt.Printf("Aggregated test results saved to %s\n", outputPath) + } else { + fmt.Println("No test results found.") + } + }, +} + +func init() { + AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files") + AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format") +} diff --git a/tools/flakeguard/cmd/aggregate_failed.go b/tools/flakeguard/cmd/aggregate_failed.go new file mode 100644 index 000000000..0f3c00c58 --- /dev/null +++ b/tools/flakeguard/cmd/aggregate_failed.go @@ -0,0 +1,60 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "os" + + "github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports" + "github.com/spf13/cobra" +) + +var AggregateFailedCmd = &cobra.Command{ + Use: "aggregate-failed", + Short: "Aggregate all test results, then filter and output only failed tests based on a threshold", + Run: func(cmd *cobra.Command, args []string) { + resultsFolderPath, _ := cmd.Flags().GetString("results-path") + outputPath, _ := cmd.Flags().GetString("output-json") + threshold, _ := cmd.Flags().GetFloat64("threshold") + + // Aggregate all test results + allResults, err := reports.AggregateTestResults(resultsFolderPath) + if err != nil { + log.Fatalf("Error aggregating results: %v", err) + } + + // Filter to only include failed tests based on threshold + var failedResults []reports.TestResult + for _, result := range allResults { + if result.PassRatio < threshold && !result.Skipped { + failedResults = append(failedResults, result) + } + } + + // Output failed results to JSON file + if outputPath != "" && len(failedResults) > 0 { + if err := saveResults(outputPath, failedResults); err != nil { + log.Fatalf("Error writing failed results to file: %v", err) + } + fmt.Printf("Filtered failed test results saved to %s\n", outputPath) + } else { + fmt.Println("No failed tests found based on the specified threshold.") + } + }, +} + +func init() { + AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files") + AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format") + AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed") +} + +// Helper function to save results to JSON file +func saveResults(filePath string, results []reports.TestResult) error { + data, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("error marshaling results: %v", err) + } + return os.WriteFile(filePath, data, 0644) +} diff --git a/tools/flakeguard/cmd/run.go b/tools/flakeguard/cmd/run.go index a2bcd2032..a0d9bb5d7 100644 --- a/tools/flakeguard/cmd/run.go +++ b/tools/flakeguard/cmd/run.go @@ -1,10 +1,12 @@ package cmd import ( + "bytes" "encoding/json" "fmt" "log" "os" + "os/exec" "github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports" "github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner" @@ -23,6 +25,12 @@ var RunTestsCmd = &cobra.Command{ outputPath, _ := cmd.Flags().GetString("output-json") threshold, _ := cmd.Flags().GetFloat64("threshold") skipTests, _ := cmd.Flags().GetStringSlice("skip-tests") + printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests") + + // Check if project dependencies are correctly set up + if err := checkDependencies(projectPath); err != nil { + log.Fatalf("Error: %v", err) + } var testPackages []string if testPackagesJson != "" { @@ -36,15 +44,16 @@ var RunTestsCmd = &cobra.Command{ } runner := runner.Runner{ - ProjectPath: projectPath, - Verbose: true, - RunCount: runCount, - UseRace: useRace, - FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0 - SkipTests: skipTests, + ProjectPath: projectPath, + Verbose: true, + RunCount: runCount, + UseRace: useRace, + FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0 + SkipTests: skipTests, + SelectedTestPackages: testPackages, } - testResults, err := runner.RunTests(testPackages) + testResults, err := runner.RunTests() if err != nil { fmt.Printf("Error running tests: %v\n", err) os.Exit(1) @@ -54,7 +63,7 @@ var RunTestsCmd = &cobra.Command{ failedTests := reports.FilterFailedTests(testResults, threshold) skippedTests := reports.FilterSkippedTests(testResults) - if len(failedTests) > 0 { + if len(failedTests) > 0 && printFailedTests { fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", threshold) fmt.Printf("%d failed tests:\n", len(failedTests)) reports.PrintTests(failedTests, os.Stdout) @@ -87,10 +96,27 @@ func init() { RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects") RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages") RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run") + RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json") RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests") RunTestsCmd.Flags().Bool("race", false, "Enable the race detector") RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure") RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format") RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky") RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running") + RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console") +} + +func checkDependencies(projectPath string) error { + cmd := exec.Command("go", "mod", "tidy") + cmd.Dir = projectPath + + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + + if err := cmd.Run(); err != nil { + return fmt.Errorf("dependency check failed: %v\n%s\nPlease run 'go mod tidy' to fix missing or unused dependencies", err, out.String()) + } + + return nil } diff --git a/tools/flakeguard/main.go b/tools/flakeguard/main.go index 33df0f6b6..ac4444489 100644 --- a/tools/flakeguard/main.go +++ b/tools/flakeguard/main.go @@ -28,6 +28,8 @@ func init() { rootCmd.AddCommand(cmd.FindTestsCmd) rootCmd.AddCommand(cmd.RunTestsCmd) + rootCmd.AddCommand(cmd.AggregateAllCmd) + rootCmd.AddCommand(cmd.AggregateFailedCmd) } func main() { diff --git a/tools/flakeguard/reports/reports.go b/tools/flakeguard/reports/reports.go index 8cf479abb..656bb3ed8 100644 --- a/tools/flakeguard/reports/reports.go +++ b/tools/flakeguard/reports/reports.go @@ -1,19 +1,24 @@ package reports import ( + "encoding/json" "fmt" "io" + "os" + "path/filepath" + "sort" "strings" ) type TestResult struct { - TestName string - TestPackage string - PassRatio float64 - Skipped bool // Indicates if the test was skipped - Runs int - Outputs []string // Stores outputs for a test - Durations []float64 // Stores elapsed time in seconds for each run of the test + TestName string + TestPackage string + PassRatio float64 // Pass ratio in decimal format like 0.5 + PassRatioPercentage string // Pass ratio in percentage format like "50%" + Skipped bool // Indicates if the test was skipped + Runs int + Outputs []string // Stores outputs for a test + Durations []float64 // Stores elapsed time in seconds for each run of the test } // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold. @@ -49,6 +54,71 @@ func FilterSkippedTests(results []TestResult) []TestResult { return skippedTests } +// AggregateTestResults aggregates all JSON test results. +func AggregateTestResults(folderPath string) ([]TestResult, error) { + // Map to hold unique tests based on their TestName and TestPackage + testMap := make(map[string]TestResult) + + err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(path) == ".json" { + // Read file content + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + // Parse JSON data into TestResult slice + var results []TestResult + if jsonErr := json.Unmarshal(data, &results); jsonErr != nil { + return jsonErr + } + // Process each result + for _, result := range results { + // Unique key for each test based on TestName and TestPackage + key := result.TestName + "|" + result.TestPackage + if existingResult, found := testMap[key]; found { + // Aggregate runs, durations, and outputs + totalRuns := existingResult.Runs + result.Runs + existingResult.Durations = append(existingResult.Durations, result.Durations...) + existingResult.Outputs = append(existingResult.Outputs, result.Outputs...) + + // Calculate total successful runs for correct pass ratio calculation + successfulRuns := existingResult.PassRatio*float64(existingResult.Runs) + result.PassRatio*float64(result.Runs) + existingResult.Runs = totalRuns + existingResult.PassRatio = successfulRuns / float64(totalRuns) + existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped + + // Update the map with the aggregated result + testMap[key] = existingResult + } else { + // Add new entry to the map + testMap[key] = result + } + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("error reading files: %v", err) + } + + // Convert map to slice of TestResult and set PassRatioPercentage + aggregatedResults := make([]TestResult, 0, len(testMap)) + for _, result := range testMap { + result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100) + aggregatedResults = append(aggregatedResults, result) + } + + // Sort by PassRatio in ascending order + sort.Slice(aggregatedResults, func(i, j int) bool { + return aggregatedResults[i].PassRatio < aggregatedResults[j].PassRatio + }) + + return aggregatedResults, nil +} + // PrintTests prints tests in a pretty format func PrintTests(tests []TestResult, w io.Writer) { for i, test := range tests { diff --git a/tools/flakeguard/reports/reports_test.go b/tools/flakeguard/reports/reports_test.go index 646d9f413..496d568e2 100644 --- a/tools/flakeguard/reports/reports_test.go +++ b/tools/flakeguard/reports/reports_test.go @@ -2,6 +2,11 @@ package reports import ( "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" "strings" "testing" ) @@ -109,3 +114,219 @@ func TestPrintTests(t *testing.T) { } } } + +// Sorts TestResult slice by TestName and TestPackage for consistent comparison +func sortTestResults(results []TestResult) { + sort.Slice(results, func(i, j int) bool { + if results[i].TestName == results[j].TestName { + return results[i].TestPackage < results[j].TestPackage + } + return results[i].TestName < results[j].TestName + }) +} + +// Helper function to write a JSON file for testing +func writeTempJSONFile(t *testing.T, dir string, filename string, data interface{}) string { + filePath := filepath.Join(dir, filename) + fileData, err := json.Marshal(data) + if err != nil { + t.Fatalf("Failed to marshal JSON: %v", err) + } + if err := os.WriteFile(filePath, fileData, 0644); err != nil { + t.Fatalf("Failed to write JSON file: %v", err) + } + return filePath +} + +func TestAggregateTestResults(t *testing.T) { + // Create a temporary directory for test JSON files + tempDir, err := os.MkdirTemp("", "aggregatetestresults") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Test cases + testCases := []struct { + description string + inputFiles []interface{} + expectedOutput []TestResult + }{ + { + description: "Unique test results without aggregation", + inputFiles: []interface{}{ + []TestResult{ + { + TestName: "TestA", + TestPackage: "pkgA", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: false, + Runs: 2, + Durations: []float64{0.01, 0.02}, + Outputs: []string{"Output1", "Output2"}, + }, + }, + []TestResult{ + { + TestName: "TestB", + TestPackage: "pkgB", + PassRatio: 0.5, + PassRatioPercentage: "50%", + Skipped: false, + Runs: 4, + Durations: []float64{0.05, 0.05, 0.05, 0.05}, + Outputs: []string{"Output3", "Output4", "Output5", "Output6"}, + }, + }, + }, + expectedOutput: []TestResult{ + { + TestName: "TestA", + TestPackage: "pkgA", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: false, + Runs: 2, + Durations: []float64{0.01, 0.02}, + Outputs: []string{"Output1", "Output2"}, + }, + { + TestName: "TestB", + TestPackage: "pkgB", + PassRatio: 0.5, + PassRatioPercentage: "50%", + Skipped: false, + Runs: 4, + Durations: []float64{0.05, 0.05, 0.05, 0.05}, + Outputs: []string{"Output3", "Output4", "Output5", "Output6"}, + }, + }, + }, + { + description: "Duplicate test results with aggregation", + inputFiles: []interface{}{ + []TestResult{ + { + TestName: "TestC", + TestPackage: "pkgC", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: false, + Runs: 2, + Durations: []float64{0.1, 0.1}, + Outputs: []string{"Output7", "Output8"}, + }, + }, + []TestResult{ + { + TestName: "TestC", + TestPackage: "pkgC", + PassRatio: 0.5, + PassRatioPercentage: "50%", + Skipped: false, + Runs: 2, + Durations: []float64{0.2, 0.2}, + Outputs: []string{"Output9", "Output10"}, + }, + }, + }, + expectedOutput: []TestResult{ + { + TestName: "TestC", + TestPackage: "pkgC", + PassRatio: 0.75, // Calculated as (2*1 + 2*0.5) / 4 + PassRatioPercentage: "75%", + Skipped: false, + Runs: 4, + Durations: []float64{0.1, 0.1, 0.2, 0.2}, + Outputs: []string{"Output7", "Output8", "Output9", "Output10"}, + }, + }, + }, + { + description: "All Skipped test results", + inputFiles: []interface{}{ + []TestResult{ + { + TestName: "TestD", + TestPackage: "pkgD", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: true, + Runs: 3, + Durations: []float64{0.1, 0.2, 0.1}, + Outputs: []string{"Output11", "Output12", "Output13"}, + }, + }, + []TestResult{ + { + TestName: "TestD", + TestPackage: "pkgD", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: true, + Runs: 2, + Durations: []float64{0.15, 0.15}, + Outputs: []string{"Output14", "Output15"}, + }, + }, + }, + expectedOutput: []TestResult{ + { + TestName: "TestD", + TestPackage: "pkgD", + PassRatio: 1, + PassRatioPercentage: "100%", + Skipped: true, // Should remain true as all runs are skipped + Runs: 5, + Durations: []float64{0.1, 0.2, 0.1, 0.15, 0.15}, + Outputs: []string{"Output11", "Output12", "Output13", "Output14", "Output15"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + // Write input files to the temporary directory + for i, inputData := range tc.inputFiles { + writeTempJSONFile(t, tempDir, fmt.Sprintf("input%d.json", i), inputData) + } + + // Run AggregateTestResults + result, err := AggregateTestResults(tempDir) + if err != nil { + t.Fatalf("AggregateTestResults failed: %v", err) + } + + // Sort both result and expectedOutput for consistent comparison + sortTestResults(result) + sortTestResults(tc.expectedOutput) + + // Compare the result with the expected output + if len(result) != len(tc.expectedOutput) { + t.Fatalf("Expected %d results, got %d", len(tc.expectedOutput), len(result)) + } + + for i, expected := range tc.expectedOutput { + got := result[i] + if got.TestName != expected.TestName || got.TestPackage != expected.TestPackage || got.Runs != expected.Runs || got.Skipped != expected.Skipped { + t.Errorf("Result %d - expected %+v, got %+v", i, expected, got) + } + if got.PassRatio != expected.PassRatio { + t.Errorf("Result %d - expected PassRatio %f, got %f", i, expected.PassRatio, got.PassRatio) + } + if got.PassRatioPercentage != expected.PassRatioPercentage { + t.Errorf("Result %d - expected PassRatioPercentage %s, got %s", i, expected.PassRatioPercentage, got.PassRatioPercentage) + } + if len(got.Durations) != len(expected.Durations) { + t.Errorf("Result %d - expected %d durations, got %d", i, len(expected.Durations), len(got.Durations)) + } + if len(got.Outputs) != len(expected.Outputs) { + t.Errorf("Result %d - expected %d outputs, got %d", i, len(expected.Outputs), len(got.Outputs)) + } + } + }) + } +} diff --git a/tools/flakeguard/runner/runner.go b/tools/flakeguard/runner/runner.go index 6b64331f5..0842bc38f 100644 --- a/tools/flakeguard/runner/runner.go +++ b/tools/flakeguard/runner/runner.go @@ -2,11 +2,11 @@ package runner import ( "bufio" - "bytes" "encoding/json" "errors" "fmt" "log" + "os" "os/exec" "strings" @@ -14,88 +14,94 @@ import ( ) type Runner struct { - ProjectPath string // Path to the Go project directory. - Verbose bool // If true, provides detailed logging. - RunCount int // Number of times to run the tests. - UseRace bool // Enable race detector. - FailFast bool // Stop on first test failure. - SkipTests []string // Test names to exclude. + ProjectPath string // Path to the Go project directory. + Verbose bool // If true, provides detailed logging. + RunCount int // Number of times to run the tests. + UseRace bool // Enable race detector. + FailFast bool // Stop on first test failure. + SkipTests []string // Test names to exclude. + SelectedTestPackages []string // Explicitly selected packages to run. } // RunTests executes the tests for each provided package and aggregates all results. // It returns all test results and any error encountered during testing. -func (r *Runner) RunTests(packages []string) ([]reports.TestResult, error) { - var jsonOutputs [][]byte - - for _, p := range packages { +func (r *Runner) RunTests() ([]reports.TestResult, error) { + var jsonFilePaths []string + for _, p := range r.SelectedTestPackages { for i := 0; i < r.RunCount; i++ { - jsonOutput, passed, err := r.runTestPackage(p) + jsonFilePath, passed, err := r.runTests(p) if err != nil { return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err) } - jsonOutputs = append(jsonOutputs, jsonOutput) + jsonFilePaths = append(jsonFilePaths, jsonFilePath) if !passed && r.FailFast { break } } } - return parseTestResults(jsonOutputs) + return parseTestResults(jsonFilePaths) } type exitCoder interface { ExitCode() int } -// runTestPackage executes the test command for a single test package. -// It returns the command output, a boolean indicating success, and any error encountered. -func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) { - args := []string{"test", "-json", "-count=1"} // Enable JSON output +// runTests runs the tests for a given package and returns the path to the output file. +func (r *Runner) runTests(packageName string) (string, bool, error) { + args := []string{"test", packageName, "-json", "-count=1"} // Enable JSON output if r.UseRace { args = append(args, "-race") } - - // Construct regex pattern from ExcludedTests slice if len(r.SkipTests) > 0 { skipPattern := strings.Join(r.SkipTests, "|") args = append(args, fmt.Sprintf("-skip=%s", skipPattern)) } - args = append(args, testPackage) - if r.Verbose { log.Printf("Running command: go %s\n", strings.Join(args, " ")) } + + // Create a temporary file to store the output + tmpFile, err := os.CreateTemp("", "test-output-*.json") + if err != nil { + return "", false, fmt.Errorf("failed to create temp file: %w", err) + } + defer tmpFile.Close() + + // Run the command with output directed to the file cmd := exec.Command("go", args...) cmd.Dir = r.ProjectPath + cmd.Stdout = tmpFile + cmd.Stderr = tmpFile - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - - // Run the command - err := cmd.Run() + err = cmd.Run() if err != nil { var exErr exitCoder // Check if the error is due to a non-zero exit code if errors.As(err, &exErr) && exErr.ExitCode() == 0 { - return nil, false, fmt.Errorf("test command failed at %s: %w", testPackage, err) + return "", false, fmt.Errorf("test command failed at %s: %w", packageName, err) } - return out.Bytes(), false, nil // Test failed + return tmpFile.Name(), false, nil // Test failed } - return out.Bytes(), true, nil // Test succeeded + return tmpFile.Name(), true, nil // Test succeeded } -// parseTestResults analyzes multiple JSON outputs from 'go test -json' commands to determine test results. -// It accepts a slice of []byte where each []byte represents a separate JSON output from a test run. -// This function aggregates results across multiple test runs, summing runs and passes for each test. -func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { +// parseTestResults reads the test output files and returns the parsed test results. +func parseTestResults(filePaths []string) ([]reports.TestResult, error) { testDetails := make(map[string]*reports.TestResult) // Holds run, pass counts, and other details for each test - // Process each data set - for _, data := range datas { - scanner := bufio.NewScanner(bytes.NewReader(data)) + // Process each file + for _, filePath := range filePaths { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open test output file: %w", err) + } + defer os.Remove(filePath) // Clean up file after parsing + defer file.Close() + + scanner := bufio.NewScanner(file) for scanner.Scan() { var entry struct { Action string `json:"Action"` @@ -130,11 +136,13 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { result.Runs++ case "pass": result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs) + result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100) result.Durations = append(result.Durations, entry.Elapsed) case "output": result.Outputs = append(result.Outputs, entry.Output) case "fail": result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs) + result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100) result.Durations = append(result.Durations, entry.Elapsed) case "skip": result.Skipped = true @@ -144,7 +152,7 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading standard input: %w", err) + return nil, fmt.Errorf("reading test output file: %w", err) } }