adding pre-checks based on regex to fail the tests

.github/workflows/pr-e2e.yml

@@ -92,6 +92,22 @@ jobs:
         run: |
           gh pr checkout ${{ needs.triage.outputs.pr_num }}
 
+
+      - name: Run pre-regex checks
+        continue-on-error: false
+        id: test-regex
+        env:
+          COMMENT_BODY: ${{ github.event.comment.body }}
+        run: |
+          MESSAGE="$COMMENT_BODY"
+          REGEX='/run-e2e (.+)'
+          if [[ "$MESSAGE" =~ $REGEX ]]
+          then
+            export E2E_TEST_REGEX="$(echo ${BASH_REMATCH[1]} | head -1)"
+          fi
+          echo "${{ needs.triage.outputs.pr_num }}"
+          make e2e-regex
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -155,14 +171,7 @@ jobs:
           GCP_RUN_IDENTITY_TESTS: true
           E2E_IMAGE_TAG: ${{ needs.triage.outputs.image_tag }}
           TEST_CLUSTER_NAME: keda-e2e-cluster-pr
-          COMMENT_BODY: ${{ github.event.comment.body }}
         run: |
-          MESSAGE="$COMMENT_BODY"
-          REGEX='/run-e2e (.+)'
-          if [[ "$MESSAGE" =~ $REGEX ]]
-          then
-            export E2E_TEST_REGEX="$(echo ${BASH_REMATCH[1]} | head -1)"
-          fi
           echo "${{ needs.triage.outputs.pr_num }}"
           make e2e-test
 

Makefile

@@ -98,14 +98,18 @@ scale-node-pool: az-login ## Scale nodepool.
 		--resource-group $(TF_AZURE_RESOURCE_GROUP) \
 		--node-count $(NODE_POOL_SIZE)
 
+.PHONY: e2e-regex
+e2e-regex:
+	go run -tags e2e-regex ./tests/run-regex.go
+
 .PHONY: e2e-test
-e2e-test: get-cluster-context ## Run e2e tests against Azure cluster.
+e2e-test: get-cluster-context e2e-regex ## Run e2e tests against Azure cluster.
 	TERMINFO=/etc/terminfo
 	TERM=linux
 	go run -tags e2e ./tests/run-all.go
 
 .PHONY: e2e-test-local
-e2e-test-local: ## Run e2e tests against Kubernetes cluster configured in ~/.kube/config.
+e2e-test-local: e2e-regex ## Run e2e tests against Kubernetes cluster configured in ~/.kube/config.
 	go run -tags e2e ./tests/run-all.go
 
 .PHONY: e2e-test-clean-crds

tests/run-all.go

@@ -7,15 +7,12 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"math/rand"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"sync"
-	"time"
 
 	"golang.org/x/sync/semaphore"
 	"k8s.io/client-go/kubernetes"
@@ -41,11 +38,6 @@ type TestResult struct {
 func main() {
 	ctx := context.Background()
 
-	e2eRegex := os.Getenv("E2E_TEST_REGEX")
-	if e2eRegex == "" {
-		e2eRegex = ".*_test.go"
-	}
-
 	//
 	// Install KEDA
 	//
@@ -56,16 +48,8 @@ func main() {
 		os.Exit(1)
 	}
 
-	//
-	// Detect test cases
-	//
-	regularTestFiles := getRegularTestFiles(e2eRegex)
-	sequentialTestFiles := getSequentialTestFiles(e2eRegex)
-	if len(regularTestFiles) == 0 && len(sequentialTestFiles) == 0 {
-		uninstallKeda(ctx)
-		fmt.Printf("No test has been executed, please review your regex: '%s'\n", e2eRegex)
-		os.Exit(1)
-	}
+	regularTestFiles := strings.Split(os.Getenv("REGULAR_TEST_FILES"), ",")
+	sequentialTestFiles := strings.Split(os.Getenv("SEQUENTIAL_TEST_FILES"), ",")
 
 	//
 	// Execute regular tests
@@ -122,63 +106,6 @@ func executeTest(ctx context.Context, file string, timeout string, tries int) Te
 	return result
 }
 
-func getRegularTestFiles(e2eRegex string) []string {
-	// We exclude utils and sequential folders and helper files
-	filter := func(path string, file string) bool {
-		return !strings.HasPrefix(path, "tests") || // we need this condition to skip non e2e test from execution
-			strings.Contains(path, "utils") ||
-			strings.Contains(path, "sequential") ||
-			!strings.HasSuffix(file, "_test.go")
-	}
-	return getTestFiles(e2eRegex, filter)
-}
-
-func getSequentialTestFiles(e2eRegex string) []string {
-	filter := func(path string, file string) bool {
-		return !strings.HasPrefix(path, "tests") || // we need this condition to skip non e2e test from execution
-			!strings.Contains(path, "sequential") ||
-			!strings.HasSuffix(file, "_test.go")
-	}
-	return getTestFiles(e2eRegex, filter)
-}
-
-func getTestFiles(e2eRegex string, filter func(path string, file string) bool) []string {
-	testFiles := []string{}
-	regex, err := regexp.Compile(e2eRegex)
-
-	if err != nil {
-		return testFiles
-	}
-
-	err = filepath.Walk(".",
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			// We exclude utils and sequential folders and helper files
-			if filter(path, info.Name()) {
-				return nil
-			}
-			if regex.MatchString(path) {
-				testFiles = append(testFiles, path)
-			}
-
-			return nil
-		})
-
-	if err != nil {
-		return []string{}
-	}
-
-	// We randomize the executions
-	rand.New(rand.NewSource(time.Now().UnixNano()))
-	rand.Shuffle(len(testFiles), func(i, j int) {
-		testFiles[i], testFiles[j] = testFiles[j], testFiles[i]
-	})
-
-	return testFiles
-}
-
 func executeRegularTests(ctx context.Context, testCases []string) []TestResult {
 	sem := semaphore.NewWeighted(int64(concurrentTests))
 	mutex := &sync.RWMutex{}

tests/run-regex.go

@@ -0,0 +1,90 @@
+// +build e2e-regex
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var (
+	concurrentTests        = 15
+	regularTestsTimeout    = "20m"
+	regularTestsRetries    = 3
+	sequentialTestsTimeout = "20m"
+	sequentialTestsRetries = 1
+)
+
+func main() {
+	e2eRegex := os.Getenv("E2E_TEST_REGEX")
+	if e2eRegex == "" {
+		e2eRegex = ".*_test.go"
+	}
+
+	//
+	// Detect test cases
+	//
+	regularTestFiles := getRegularTestFiles(e2eRegex)
+	sequentialTestFiles := getSequentialTestFiles(e2eRegex)
+	if len(regularTestFiles) == 0 && len(sequentialTestFiles) == 0 {
+		fmt.Printf("No test will be executed, please review your regex: '%s'\n", e2eRegex)
+		os.Exit(1)
+	}
+
+	// Store the test files in environment variables
+	os.Setenv("REGULAR_TEST_FILES", strings.Join(regularTestFiles, ","))
+	os.Setenv("SEQUENTIAL_TEST_FILES", strings.Join(sequentialTestFiles, ","))
+}
+
+func getRegularTestFiles(e2eRegex string) []string {
+	filter := func(path string, file string) bool {
+		return !strings.HasPrefix(path, "tests") || // we need this condition to skip non e2e test from execution
+			strings.Contains(path, "utils") ||
+			strings.Contains(path, "sequential") ||
+			!strings.HasSuffix(file, "_test.go")
+	}
+	return getTestFiles(e2eRegex, filter)
+}
+
+func getSequentialTestFiles(e2eRegex string) []string {
+	filter := func(path string, file string) bool {
+		return !strings.HasPrefix(path, "tests") || // we need this condition to skip non e2e test from execution
+			!strings.Contains(path, "sequential") ||
+			!strings.HasSuffix(file, "_test.go")
+	}
+	return getTestFiles(e2eRegex, filter)
+}
+
+func getTestFiles(e2eRegex string, filter func(path string, file string) bool) []string {
+	testFiles := []string{}
+	regex, err := regexp.Compile(e2eRegex)
+
+	if err != nil {
+		return testFiles
+	}
+
+	err = filepath.Walk(".",
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			// We exclude utils and sequential folders and helper files
+			if filter(path, info.Name()) {
+				return nil
+			}
+			if regex.MatchString(path) {
+				testFiles = append(testFiles, path)
+			}
+
+			return nil
+		})
+
+	if err != nil {
+		return []string{}
+	}
+
+	return testFiles
+}