chore(regression): improve coverage with testing tag matrix

.github/workflows/regression.yml

@@ -13,11 +13,28 @@ on:
       - "LICENSE"
 
 jobs:
+  # Generate matrix of tags for all permutations of the tests
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.generate.outputs.tags }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate tag combinations
+        id: generate
+        run: |
+          go run mage.go tagsmatrix > tags.json
+          echo "::set-output name=tags::$(cat tags.json)"
+        shell: bash
   test:
+    needs: generate-matrix
     strategy:
       matrix:
         go-version: [1.22.x, 1.23.x]
         os: [ubuntu-latest]
+        build-flag: ${{ fromJson(needs.generate-matrix.outputs.tags) }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
@@ -26,36 +43,31 @@ jobs:
         uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version: ${{ matrix.go-version }}
-          cache: true
+          cache: true  
       - name: Tests and coverage
-        run: go run mage.go coverage
+        run: |
+          go run mage.go coverage ${{ matrix.build-flag }}
       - name: "Codecov: General"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
-          files: build/coverage.txt
+          files: build/${{ matrix.build-flag }}.txt
           flags: default
       - name: "Codecov: Examples"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
-          files: build/coverage-examples.txt
+          files: build/${{ matrix.build-flag }}-examples.txt
           flags: examples
       - name: "Codecov: FTW"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
-          files: build/coverage-ftw.txt
+          files: build/${{ matrix.build-flag }}-ftw.txt
           flags: ftw
-      - name: "Codecov: FTW Multiphase tag"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
-        if: ${{ matrix.go-version == '1.22.x' }}
-        with:
-          files: build/coverage-ftw-multiphase.txt
-          flags: ftw-multiphase
       - name: "Codecov: Tinygo"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
-          files: build/coverage-tinygo.txt
+          files: build/${{ matrix.build-flag }}-tinygo.txt
           flags: tinygo

magefile.go

@@ -7,12 +7,14 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -134,39 +136,38 @@ func Test() error {
 	return nil
 }
 
+func buildTagsFlags(tags string) string {
+	if tags == "" {
+		return ""
+	}
+	return fmt.Sprintf("%q", tags)
+}
+
 // Coverage runs tests with coverage and race detector enabled.
-func Coverage() error {
+func Coverage(tags string) error {
+	tags = buildTagsFlags(tags)
 	if err := os.MkdirAll("build", 0755); err != nil {
 		return err
 	}
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
-		return err
+	fmt.Println("Running tests with coverage")
+	fmt.Println("Tags:", tags)
+	tagsCmd := ""
+	if tags != "" {
+		tagsCmd = "-tags=" + tags
 	}
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-examples.txt", "-covermode=atomic", "-coverpkg=./...", "./examples/http-server"); err != nil {
+	if err := sh.RunV("go", "test", "-race", tagsCmd, fmt.Sprintf("-coverprofile=build/%s-coverage.txt", tags), "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
 		return err
 	}
-	if err := sh.RunV("go", "test", "-coverprofile=build/coverage-ftw.txt", "-covermode=atomic", "-coverpkg=./...", "./testing/coreruleset"); err != nil {
+	// Execute http-server tests with coverage
+	if err := sh.RunV("go", "test", "-race", tagsCmd, fmt.Sprintf("-coverprofile=build/%s-coverage-examples.txt", tags), "-covermode=atomic", "-coverpkg=./...", "./examples/http-server"); err != nil {
 		return err
 	}
-	// Execute coverage tests with multiphase evaluation enabled
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-multiphase.txt", "-covermode=atomic", "-coverpkg=./...", "-tags=coraza.rule.multiphase_evaluation", "./..."); err != nil {
-		return err
-	}
-	// Executes http-server tests with multiphase evaluation enabled
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-examples.txt", "-covermode=atomic", "-tags=coraza.rule.multiphase_evaluation", "-coverpkg=./...", "./examples/http-server"); err != nil {
-		return err
-	}
-	// Execute FTW tests with multiphase evaluation enabled as well
-	if err := sh.RunV("go", "test", "-coverprofile=build/coverage-ftw-multiphase.txt", "-covermode=atomic", "-coverpkg=./...", "-tags=coraza.rule.multiphase_evaluation", "./testing/coreruleset"); err != nil {
-		return err
-	}
-	// This is not actually running tests with tinygo, but with the tag that includes its code so we can calculate coverage
-	// for it.
-	if err := sh.RunV("go", "test", "-race", "-tags=tinygo", "-coverprofile=build/coverage-tinygo.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
+	// Execute FTW tests with coverage as well
+	if err := sh.RunV("go", "test", tagsCmd, fmt.Sprintf("-coverprofile=build/%s-coverage-ftw.txt", tags), "-covermode=atomic", "-coverpkg=./...", "./testing/coreruleset"); err != nil {
 		return err
 	}
 
-	return sh.RunV("go", "tool", "cover", "-html=build/coverage.txt", "-o", "build/coverage.html")
+	return sh.RunV("go", "tool", "cover", fmt.Sprintf("-html=build/%s-coverage.txt", tags), "-o", fmt.Sprintf("build/%s-coverage.html", tags))
 }
 
 // Fuzz runs fuzz tests
@@ -228,3 +229,43 @@ func Precommit() error {
 func Check() {
 	mg.SerialDeps(Lint, Test)
 }
+
+// combinations generates all possible combinations of build tags
+func combinations(tags []string) []string {
+	var result []string
+	n := len(tags)
+	for i := 0; i < (1 << n); i++ {
+		var combo []string
+		for j := 0; j < n; j++ {
+			if i&(1<<j) != 0 {
+				combo = append(combo, tags[j])
+			}
+		}
+		if len(combo) > 0 {
+			result = append(result, strings.Join(combo, ","))
+		} else {
+			result = append(result, "")
+		}
+	}
+	return result
+}
+
+// Generates a JSON output to stdout which contains all permutations of build tags for the project.
+func TagsMatrix() error {
+	tags := []string{
+		"coraza.rule.case_sensitive_args_keys",
+		"memoize_builders",
+		"coraza.rule.multiphase_valuation",
+		"tinygo",
+	}
+	combos := combinations(tags)
+
+	jsonData, err := json.Marshal(combos)
+	if err != nil {
+		fmt.Println("Error generating JSON:", err)
+		return nil
+	}
+
+	fmt.Println(string(jsonData))
+	return nil
+}