chore(regression): improve coverage with testing tag matrix

.github/workflows/regression.yml

@@ -11,13 +11,31 @@ on:
     paths-ignore:
       - "**/*.md"
       - "LICENSE"
-
+  schedule:
+    - cron: "0 0 * * *"
 jobs:
+  # Generate matrix of tags for all permutations of the tests
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.generate.outputs.tags }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate tag combinations
+        id: generate
+        run: |
+          go run mage.go tagsmatrix > tags.json
+          echo "::set-output name=tags::$(cat tags.json)"
+        shell: bash
   test:
+    needs: generate-matrix
     strategy:
       matrix:
         go-version: [1.22.x, 1.23.x]
         os: [ubuntu-latest]
+        build-flag: ${{ fromJson(needs.generate-matrix.outputs.tags) }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout code
@@ -26,36 +44,33 @@ jobs:
         uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5
         with:
           go-version: ${{ matrix.go-version }}
-          cache: true
+          cache: true  
       - name: Tests and coverage
-        run: go run mage.go coverage
+        run: |
+          export BUILD_TAGS=${{ matrix.build-flag }}
+          go run mage.go coverage
       - name: "Codecov: General"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage.txt
-          flags: default
+          flags: default+${{ matrix.build-flag }}
       - name: "Codecov: Examples"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage-examples.txt
-          flags: examples
+          flags: examples+${{ matrix.build-flag }}
       - name: "Codecov: FTW"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         if: ${{ matrix.go-version == '1.22.x' }}
         with:
           files: build/coverage-ftw.txt
-          flags: ftw
-      - name: "Codecov: FTW Multiphase tag"
-        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
-        if: ${{ matrix.go-version == '1.22.x' }}
-        with:
-          files: build/coverage-ftw-multiphase.txt
-          flags: ftw-multiphase
+          flags: ftw+${{ matrix.build-flag }}
       - name: "Codecov: Tinygo"
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
-        if: ${{ matrix.go-version == '1.22.x' }}
+        # only if coverage-tinygo.txt exists
+        if: ${{ matrix.go-version == '1.22.x' && hashFiles('build/coverage-tinygo.txt') != '' }}
         with:
           files: build/coverage-tinygo.txt
-          flags: tinygo
+          flags: tinygo+${{ matrix.build-flag }}

internal/corazawaf/transaction_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/internal/collections"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
+	"github.com/corazawaf/coraza/v3/internal/environment"
 	utils "github.com/corazawaf/coraza/v3/internal/strings"
 	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
@@ -1731,6 +1732,9 @@ func TestForceRequestBodyOverride(t *testing.T) {
 }
 
 func TestCloseFails(t *testing.T) {
+	if !environment.HasAccessToFS {
+		t.Skip("skipping test as it requires access to filesystem")
+	}
 	waf := NewWAF()
 	tx := waf.NewTransaction()
 	col := tx.Variables().FilesTmpNames().(*collections.Map)

internal/seclang/directives_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/corazawaf/coraza/v3/internal/corazawaf"
+	"github.com/corazawaf/coraza/v3/internal/environment"
 	"github.com/corazawaf/coraza/v3/types"
 )
 
@@ -165,11 +166,6 @@ func TestDirectives(t *testing.T) {
 			{"", expectErrorOnDirective},
 			{"1000", func(w *corazawaf.WAF) bool { return w.UploadFileLimit == 1000 }},
 		},
-		"SecUploadDir": {
-			{"", expectErrorOnDirective},
-			{"/tmp-non-existing", expectErrorOnDirective},
-			{os.TempDir(), func(w *corazawaf.WAF) bool { return w.UploadDir == os.TempDir() }},
-		},
 		"SecSensorId": {
 			{"", expectErrorOnDirective},
 			{"test", func(w *corazawaf.WAF) bool { return w.SensorID == "test" }},
@@ -315,6 +311,13 @@ func TestDirectives(t *testing.T) {
 			{"1000", func(waf *corazawaf.WAF) bool { return waf.ArgumentLimit == 1000 }},
 		},
 	}
+	if environment.HasAccessToFS {
+		directiveCases["SecUploadDir"] = []directiveCase{
+			{"", expectErrorOnDirective},
+			{"/tmp-non-existing", expectErrorOnDirective},
+			{os.TempDir(), func(w *corazawaf.WAF) bool { return w.UploadDir == os.TempDir() }},
+		}
+	}
 
 	for name, dCases := range directiveCases {
 		t.Run(name, func(t *testing.T) {

magefile.go

@@ -7,12 +7,15 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -134,36 +137,51 @@ func Test() error {
 	return nil
 }
 
+func buildTagsFlags(tags string) string {
+	if tags == "" {
+		return ""
+	}
+	// we remove all non alphanumeric _,-
+	rx := regexp.MustCompile("^[\\w_,\\.]+$")
+	if !rx.MatchString(tags) {
+		panic("Invalid build tags")
+	}
+	return tags
+}
+
 // Coverage runs tests with coverage and race detector enabled.
+// Usage: mage coverage [buildTags]
 func Coverage() error {
+	buildTags := os.Getenv("BUILD_TAGS")
+	tags := buildTagsFlags(buildTags)
 	if err := os.MkdirAll("build", 0755); err != nil {
 		return err
 	}
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
-		return err
-	}
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-examples.txt", "-covermode=atomic", "-coverpkg=./...", "./examples/http-server"); err != nil {
-		return err
+	fmt.Println("Running tests with coverage")
+	fmt.Println("Tags:", tags)
+	tagsCmd := ""
+	if tags != "" {
+		tagsCmd = "-tags=" + tags
 	}
-	if err := sh.RunV("go", "test", "-coverprofile=build/coverage-ftw.txt", "-covermode=atomic", "-coverpkg=./...", "./testing/coreruleset"); err != nil {
+	if err := sh.RunV("go", "test", "-race", tagsCmd, "-coverprofile=build/coverage.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
 		return err
 	}
-	// Execute coverage tests with multiphase evaluation enabled
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-multiphase.txt", "-covermode=atomic", "-coverpkg=./...", "-tags=coraza.rule.multiphase_evaluation", "./..."); err != nil {
+	// Execute http-server tests with coverage
+	if err := sh.RunV("go", "test", "-race", tagsCmd, "-coverprofile=build/coverage-examples.txt", "-covermode=atomic", "-coverpkg=./...", "./examples/http-server"); err != nil {
 		return err
 	}
-	// Executes http-server tests with multiphase evaluation enabled
-	if err := sh.RunV("go", "test", "-race", "-coverprofile=build/coverage-examples.txt", "-covermode=atomic", "-tags=coraza.rule.multiphase_evaluation", "-coverpkg=./...", "./examples/http-server"); err != nil {
+	// Execute FTW tests with coverage as well
+	if err := sh.RunV("go", "test", tagsCmd, "-coverprofile=build/coverage-ftw.txt", "-covermode=atomic", "-coverpkg=./...", "./testing/coreruleset"); err != nil {
 		return err
 	}
-	// Execute FTW tests with multiphase evaluation enabled as well
-	if err := sh.RunV("go", "test", "-coverprofile=build/coverage-ftw-multiphase.txt", "-covermode=atomic", "-coverpkg=./...", "-tags=coraza.rule.multiphase_evaluation", "./testing/coreruleset"); err != nil {
-		return err
-	}
-	// This is not actually running tests with tinygo, but with the tag that includes its code so we can calculate coverage
-	// for it.
-	if err := sh.RunV("go", "test", "-race", "-tags=tinygo", "-coverprofile=build/coverage-tinygo.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
-		return err
+	// we run tinygo tag only if memoize_builders is is not enabled
+	if !strings.Contains(tags, "memoize_builders") {
+		if tagsCmd != "" {
+			tagsCmd += ",tinygo"
+		}
+		if err := sh.RunV("go", "test", "-race", tagsCmd, "-coverprofile=build/coverage-tinygo.txt", "-covermode=atomic", "-coverpkg=./...", "./..."); err != nil {
+			return err
+		}
 	}
 
 	return sh.RunV("go", "tool", "cover", "-html=build/coverage.txt", "-o", "build/coverage.html")
@@ -228,3 +246,43 @@ func Precommit() error {
 func Check() {
 	mg.SerialDeps(Lint, Test)
 }
+
+// combinations generates all possible combinations of build tags
+func combinations(tags []string) []string {
+	var result []string
+	n := len(tags)
+	for i := 0; i < (1 << n); i++ {
+		var combo []string
+		for j := 0; j < n; j++ {
+			if i&(1<<j) != 0 {
+				combo = append(combo, tags[j])
+			}
+		}
+		if len(combo) > 0 {
+			result = append(result, strings.Join(combo, ","))
+		} else {
+			result = append(result, "")
+		}
+	}
+	return result
+}
+
+// Generates a JSON output to stdout which contains all permutations of build tags for the project.
+func TagsMatrix() error {
+	tags := []string{
+		"coraza.rule.case_sensitive_args_keys",
+		"memoize_builders",
+		"coraza.rule.multiphase_valuation",
+		"no_fs_access",
+	}
+	combos := combinations(tags)
+
+	jsonData, err := json.Marshal(combos)
+	if err != nil {
+		fmt.Println("Error generating JSON:", err)
+		return nil
+	}
+
+	fmt.Println(string(jsonData))
+	return nil
+}