flakeguard: Add cmds to aggregate test results and improve memory eff…

…iciency by save test outputs to files (#1336)

tools/flakeguard/cmd/aggregate.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/spf13/cobra"
+)
+
+var AggregateAllCmd = &cobra.Command{
+	Use:   "aggregate-all",
+	Short: "Aggregate all test results and output them to a file",
+	Run: func(cmd *cobra.Command, args []string) {
+		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
+		outputPath, _ := cmd.Flags().GetString("output-json")
+
+		// Aggregate all test results
+		allResults, err := reports.AggregateTestResults(resultsFolderPath)
+		if err != nil {
+			log.Fatalf("Error aggregating results: %v", err)
+		}
+
+		// Output all results to JSON file
+		if outputPath != "" && len(allResults) > 0 {
+			if err := saveResults(outputPath, allResults); err != nil {
+				log.Fatalf("Error writing aggregated results to file: %v", err)
+			}
+			fmt.Printf("Aggregated test results saved to %s\n", outputPath)
+		} else {
+			fmt.Println("No test results found.")
+		}
+	},
+}
+
+func init() {
+	AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
+	AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format")
+}

tools/flakeguard/cmd/aggregate_failed.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/spf13/cobra"
+)
+
+var AggregateFailedCmd = &cobra.Command{
+	Use:   "aggregate-failed",
+	Short: "Aggregate all test results, then filter and output only failed tests based on a threshold",
+	Run: func(cmd *cobra.Command, args []string) {
+		resultsFolderPath, _ := cmd.Flags().GetString("results-path")
+		outputPath, _ := cmd.Flags().GetString("output-json")
+		threshold, _ := cmd.Flags().GetFloat64("threshold")
+
+		// Aggregate all test results
+		allResults, err := reports.AggregateTestResults(resultsFolderPath)
+		if err != nil {
+			log.Fatalf("Error aggregating results: %v", err)
+		}
+
+		// Filter to only include failed tests based on threshold
+		var failedResults []reports.TestResult
+		for _, result := range allResults {
+			if result.PassRatio < threshold && !result.Skipped {
+				failedResults = append(failedResults, result)
+			}
+		}
+
+		// Output failed results to JSON file
+		if outputPath != "" && len(failedResults) > 0 {
+			if err := saveResults(outputPath, failedResults); err != nil {
+				log.Fatalf("Error writing failed results to file: %v", err)
+			}
+			fmt.Printf("Filtered failed test results saved to %s\n", outputPath)
+		} else {
+			fmt.Println("No failed tests found based on the specified threshold.")
+		}
+	},
+}
+
+func init() {
+	AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
+	AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
+	AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
+}
+
+// Helper function to save results to JSON file
+func saveResults(filePath string, results []reports.TestResult) error {
+	data, err := json.MarshalIndent(results, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling results: %v", err)
+	}
+	return os.WriteFile(filePath, data, 0644)
+}

tools/flakeguard/cmd/run.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner"
@@ -23,6 +25,12 @@ var RunTestsCmd = &cobra.Command{
 		outputPath, _ := cmd.Flags().GetString("output-json")
 		threshold, _ := cmd.Flags().GetFloat64("threshold")
 		skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
+		printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests")
+
+		// Check if project dependencies are correctly set up
+		if err := checkDependencies(projectPath); err != nil {
+			log.Fatalf("Error: %v", err)
+		}
 
 		var testPackages []string
 		if testPackagesJson != "" {
@@ -36,15 +44,16 @@ var RunTestsCmd = &cobra.Command{
 		}
 
 		runner := runner.Runner{
-			ProjectPath: projectPath,
-			Verbose:     true,
-			RunCount:    runCount,
-			UseRace:     useRace,
-			FailFast:    threshold == 1.0, // Fail test on first test run if threshold is 1.0
-			SkipTests:   skipTests,
+			ProjectPath:          projectPath,
+			Verbose:              true,
+			RunCount:             runCount,
+			UseRace:              useRace,
+			FailFast:             threshold == 1.0, // Fail test on first test run if threshold is 1.0
+			SkipTests:            skipTests,
+			SelectedTestPackages: testPackages,
 		}
 
-		testResults, err := runner.RunTests(testPackages)
+		testResults, err := runner.RunTests()
 		if err != nil {
 			fmt.Printf("Error running tests: %v\n", err)
 			os.Exit(1)
@@ -54,7 +63,7 @@ var RunTestsCmd = &cobra.Command{
 		failedTests := reports.FilterFailedTests(testResults, threshold)
 		skippedTests := reports.FilterSkippedTests(testResults)
 
-		if len(failedTests) > 0 {
+		if len(failedTests) > 0 && printFailedTests {
 			fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", threshold)
 			fmt.Printf("%d failed tests:\n", len(failedTests))
 			reports.PrintTests(failedTests, os.Stdout)
@@ -87,10 +96,27 @@ func init() {
 	RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
 	RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
 	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
+	RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
 	RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
 	RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
 	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
 	RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
 	RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
 	RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
+	RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
+}
+
+func checkDependencies(projectPath string) error {
+	cmd := exec.Command("go", "mod", "tidy")
+	cmd.Dir = projectPath
+
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("dependency check failed: %v\n%s\nPlease run 'go mod tidy' to fix missing or unused dependencies", err, out.String())
+	}
+
+	return nil
 }

tools/flakeguard/main.go

@@ -28,6 +28,8 @@ func init() {
 
 	rootCmd.AddCommand(cmd.FindTestsCmd)
 	rootCmd.AddCommand(cmd.RunTestsCmd)
+	rootCmd.AddCommand(cmd.AggregateAllCmd)
+	rootCmd.AddCommand(cmd.AggregateFailedCmd)
 }
 
 func main() {

tools/flakeguard/reports/reports.go

@@ -1,19 +1,24 @@
 package reports
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
+	"sort"
 	"strings"
 )
 
 type TestResult struct {
-	TestName    string
-	TestPackage string
-	PassRatio   float64
-	Skipped     bool // Indicates if the test was skipped
-	Runs        int
-	Outputs     []string  // Stores outputs for a test
-	Durations   []float64 // Stores elapsed time in seconds for each run of the test
+	TestName            string
+	TestPackage         string
+	PassRatio           float64 // Pass ratio in decimal format like 0.5
+	PassRatioPercentage string  // Pass ratio in percentage format like "50%"
+	Skipped             bool    // Indicates if the test was skipped
+	Runs                int
+	Outputs             []string  // Stores outputs for a test
+	Durations           []float64 // Stores elapsed time in seconds for each run of the test
 }
 
 // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
@@ -49,6 +54,71 @@ func FilterSkippedTests(results []TestResult) []TestResult {
 	return skippedTests
 }
 
+// AggregateTestResults aggregates all JSON test results.
+func AggregateTestResults(folderPath string) ([]TestResult, error) {
+	// Map to hold unique tests based on their TestName and TestPackage
+	testMap := make(map[string]TestResult)
+
+	err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && filepath.Ext(path) == ".json" {
+			// Read file content
+			data, readErr := os.ReadFile(path)
+			if readErr != nil {
+				return readErr
+			}
+			// Parse JSON data into TestResult slice
+			var results []TestResult
+			if jsonErr := json.Unmarshal(data, &results); jsonErr != nil {
+				return jsonErr
+			}
+			// Process each result
+			for _, result := range results {
+				// Unique key for each test based on TestName and TestPackage
+				key := result.TestName + "|" + result.TestPackage
+				if existingResult, found := testMap[key]; found {
+					// Aggregate runs, durations, and outputs
+					totalRuns := existingResult.Runs + result.Runs
+					existingResult.Durations = append(existingResult.Durations, result.Durations...)
+					existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
+
+					// Calculate total successful runs for correct pass ratio calculation
+					successfulRuns := existingResult.PassRatio*float64(existingResult.Runs) + result.PassRatio*float64(result.Runs)
+					existingResult.Runs = totalRuns
+					existingResult.PassRatio = successfulRuns / float64(totalRuns)
+					existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
+
+					// Update the map with the aggregated result
+					testMap[key] = existingResult
+				} else {
+					// Add new entry to the map
+					testMap[key] = result
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error reading files: %v", err)
+	}
+
+	// Convert map to slice of TestResult and set PassRatioPercentage
+	aggregatedResults := make([]TestResult, 0, len(testMap))
+	for _, result := range testMap {
+		result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
+		aggregatedResults = append(aggregatedResults, result)
+	}
+
+	// Sort by PassRatio in ascending order
+	sort.Slice(aggregatedResults, func(i, j int) bool {
+		return aggregatedResults[i].PassRatio < aggregatedResults[j].PassRatio
+	})
+
+	return aggregatedResults, nil
+}
+
 // PrintTests prints tests in a pretty format
 func PrintTests(tests []TestResult, w io.Writer) {
 	for i, test := range tests {