Running all the tests on two clusters and only full-rr on 3 clusters (#…

…769)

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

.github/workflows/terratest-more-clusters.yaml

@@ -0,0 +1,67 @@
+name: Terratest for n clusters
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - '**.svg'
+      - '**.drawio'
+      - '.spelling'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - '**.md'
+      - '**.svg'
+      - '**.drawio'
+      - '.spelling'
+
+jobs:
+  terratest-n-clusters:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.9"
+
+      - name: Build artifacts
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist --skip-publish --skip-validate --snapshot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create 1st k3s Cluster
+        uses: AbsaOSS/k3d-action@v1.5.0
+        with:
+          cluster-name: "test-gslb1"
+          args: -c k3d/test-gslb1.yaml
+
+      - name: Create 2nd k3s Cluster
+        uses: AbsaOSS/k3d-action@v1.5.0
+        with:
+          cluster-name: "test-gslb2"
+          args: -c k3d/test-gslb2.yaml
+
+      - name: Create 3rd k3s Cluster
+        uses: AbsaOSS/k3d-action@v1.5.0
+        with:
+          cluster-name: "test-gslb3"
+          args: -c k3d/test-gslb3.yaml
+
+      - name: K8GB deployment
+        run: |
+          make deploy-test-version list-running-pods CLUSTERS_NUMBER=3
+          echo "Cluster 1 (eu):"
+          kubectl get no -owide --context=k3d-test-gslb1
+          echo "Cluster 2 (us):"
+          kubectl get no -owide --context=k3d-test-gslb2
+          echo "Cluster 3 (cz):"
+          kubectl get no -owide --context=k3d-test-gslb3
+
+      - name: Terratest
+        continue-on-error: true
+        run: make terratest

.github/workflows/terratest.yaml

@@ -46,21 +46,13 @@ jobs:
           cluster-name: "test-gslb2"
           args: -c k3d/test-gslb2.yaml
 
-      - name: Create 3rd k3s Cluster
-        uses: AbsaOSS/k3d-action@v1.5.0
-        with:
-          cluster-name: "test-gslb3"
-          args: -c k3d/test-gslb3.yaml
-
       - name: K8GB deployment
         run: |
-          make deploy-test-version list-running-pods CLUSTERS_NUMBER=3
+          make deploy-test-version list-running-pods
           echo "Cluster 1 (eu):"
           kubectl get no -owide --context=k3d-test-gslb1
           echo "Cluster 2 (us):"
           kubectl get no -owide --context=k3d-test-gslb2
-          echo "Cluster 3 (cz):"
-          kubectl get no -owide --context=k3d-test-gslb3
 
       - name: Terratest
         run: make terratest

.licignore

@@ -11,3 +11,6 @@
 /api/**
 main.go
 gslb_controller.go
+
+# Build tags in tests...
+/terratest/test/**

Makefile

@@ -416,12 +416,12 @@ test-failover:
 .PHONY: terratest
 terratest: # Run terratest suite
 	@$(eval RUNNING_CLUSTERS := $(shell k3d cluster list --no-headers | grep $(CLUSTER_NAME) -c))
-	@if [ "$(RUNNING_CLUSTERS)" != 3 ] ; then \
-		echo "$(RED)Make sure you run the tests against 3 running clusters$(NC)" ;\
-		echo "$(RED)Currently $(RUNNING_CLUSTERS) running clusters were discovered$(NC)" ;\
-		exit 1 ;\
+	@$(eval TEST_TAGS := $(shell [ $(RUNNING_CLUSTERS) == 2 ] && echo all || echo rr_multicluster))
+	@if [ "$(RUNNING_CLUSTERS)" -lt 2 ] ; then \
+		echo "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\
+		exit 1;\
 	fi
-	cd terratest/test/ && go mod download && go test -v -timeout 15m -parallel=12
+	cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 15m -parallel=12 --tags=$(TEST_TAGS)
 
 .PHONY: website
 website:

terratest/test/init.go

@@ -31,6 +31,7 @@ func init() {
 	p1, _ := env.GetEnvAsIntOrFallback("DNS_SERVER1_PORT", 5053)
 	p2, _ := env.GetEnvAsIntOrFallback("DNS_SERVER2_PORT", 5054)
 	p3, _ := env.GetEnvAsIntOrFallback("DNS_SERVER3_PORT", 5055)
+	clNum, _ :=  env.GetEnvAsIntOrFallback("CLUSTERS_NUMBER", 2)
 	settings = utils.TestSettings{
 		DNSZone:         env.GetEnvAsStringOrFallback("GSLB_DOMAIN", "cloud.example.com"),
 		PrimaryGeoTag:   env.GetEnvAsStringOrFallback("PRIMARY_GEO_TAG", "eu"),
@@ -45,5 +46,6 @@ func init() {
 		Cluster2:        env.GetEnvAsStringOrFallback("K8GB_CLUSTER2", "k3d-test-gslb2"),
 		Cluster3:        env.GetEnvAsStringOrFallback("K8GB_CLUSTER3", "k3d-test-gslb3"),
 		PodinfoImage:    env.GetEnvAsStringOrFallback("PODINFO_IMAGE_REPO", "ghcr.io/stefanprodan/podinfo"),
+		ClustersNumber:  clNum,
 	}
 }

terratest/test/k8gb_abstract_full_roundrobin_test.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2021 The k8gb Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
+*/
+package test
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/require"
+	"k8gbterratest/utils"
+	"testing"
+)
+
+func abstractTestFullRoundRobin(t *testing.T, n int) {
+	if n < 2 || n > 8 {
+		t.Logf("Use value of n that represents the number of clusters from interval [2,8]")
+		t.FailNow()
+	}
+	t.Logf(fmt.Sprintf("Running TestFullRoundRobin for %d clusters", n))
+	tags := []string{"eu", "us", "cz", "af", "ru", "ap", "uk", "ca"}
+	var instances []*utils.Instance
+
+	t.Parallel()
+	const host = "roundrobin-test.cloud.example.com"
+	const gslbPath = "../examples/roundrobin2.yaml"
+
+	// start all the test apps on all the clusters
+	for i := 0; i < n; i+=1 {
+		instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053 + i).
+			WithGslb(gslbPath, host).
+			WithTestApp(tags[i]).
+			Start()
+		require.NoError(t, er)
+		instances = append(instances, instance)
+		defer instance.Kill()
+	}
+	var err error
+	t.Run(fmt.Sprintf("round-robin on %d concurrent clusters with podinfo running", n), func(t *testing.T) {
+		for _, ins  := range instances {
+			err = ins.WaitForAppIsRunning()
+			require.NoError(t, err)
+		}
+	})
+
+	// at the beginning, it should contain all the targets
+	var workingTargets []string
+	for _, ins  := range instances {
+		workingTargets = append(workingTargets, ins.GetLocalTargets()...)
+	}
+	t.Run(fmt.Sprintf("all %d clusters should be interconnected", n), func(t *testing.T) {
+		allShouldExpectTheseTargets(t, instances, workingTargets)
+	})
+
+	// kill the apps on clusters one by one and expect less and less targets to be available
+	for i, instance := range instances {
+		t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i + 1, tags[i]), func(t *testing.T) {
+			workingTargets = distinct(subtract(workingTargets, instance.GetLocalTargets()))
+			t.Logf("New expected targets: %v", workingTargets)
+			instance.StopTestApp()
+			allShouldExpectTheseTargets(t, instances, workingTargets)
+		})
+	}
+
+	// start the test apps again on each cluster and check if the targets start appearing
+	for i, instance := range instances {
+		t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i + 1, tags[i]), func(t *testing.T) {
+			instance.StartTestApp()
+			workingTargets = distinct(append(workingTargets, instance.GetLocalTargets()...))
+			t.Logf("New expected targets: %v", workingTargets)
+			allShouldExpectTheseTargets(t, instances, workingTargets)
+		})
+	}
+}
+
+func allShouldExpectTheseTargets(t *testing.T, instances []*utils.Instance, workingTargets []string) {
+	for _, instance := range instances {
+		err := instance.WaitForExpected(workingTargets)
+		require.NoError(t, err)
+	}
+}
+
+// helper function for subtracting slice from slice (in O(n))
+func subtract(from, what []string) (diff []string) {
+	aux := make(map[string]bool, len(what))
+	for _, w := range what {
+		aux[w] = true
+	}
+	for _, f := range from {
+		if _, found := aux[f]; !found {
+			diff = append(diff, f)
+		}
+	}
+	return
+}
+
+// helper function for de-duplicating elements in slice (in O(n))
+func distinct(s []string) (uniq []string) {
+	aux := make(map[string]bool)
+	for _, item := range s {
+		if _, found := aux[item]; !found {
+			aux[item] = true
+			uniq = append(uniq, item)
+		}
+	}
+	return
+}

terratest/test/k8gb_basic_app_test.go

@@ -1,3 +1,5 @@
+// +build basic all
+
 /*
 Copyright 2021 The k8gb Contributors.
 

terratest/test/k8gb_failover_playground_test.go

@@ -1,3 +1,5 @@
+// +build failover all
+
 /*
 Copyright 2021 The k8gb Contributors.
 

terratest/test/k8gb_full_failover_test.go

@@ -1,3 +1,5 @@
+// +build failover all
+
 /*
 Copyright 2021 The k8gb Contributors.
 

terratest/test/k8gb_full_roundrobin_test.go

@@ -1,3 +1,5 @@
+// +build rr rr_multicluster all
+
 /*
 Copyright 2021 The k8gb Contributors.
 
@@ -18,110 +20,9 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
 package test
 
 import (
-	"k8gbterratest/utils"
 	"testing"
-
-	"github.com/stretchr/testify/require"
 )
 
 func TestFullRoundRobin(t *testing.T) {
-	t.Parallel()
-	const host = "roundrobin-test.cloud.example.com"
-	const gslbPath = "../examples/roundrobin2.yaml"
-
-	instanceEU, err := utils.NewWorkflow(t, settings.Cluster1, settings.Port1).
-		WithGslb(gslbPath, host).
-		WithTestApp("eu").
-		Start()
-	require.NoError(t, err)
-	defer instanceEU.Kill()
-	instanceUS, err := utils.NewWorkflow(t, settings.Cluster2, settings.Port2).
-		WithGslb(gslbPath, host).
-		WithTestApp("us").
-		Start()
-	require.NoError(t, err)
-	defer instanceUS.Kill()
-	instanceCZ, err := utils.NewWorkflow(t, settings.Cluster3, settings.Port3).
-		WithGslb(gslbPath, host).
-		WithTestApp("cz").
-		Start()
-	require.NoError(t, err)
-	defer instanceCZ.Kill()
-
-	t.Run("round-robin on three concurrent clusters with podinfo running", func(t *testing.T) {
-		err = instanceEU.WaitForAppIsRunning()
-		require.NoError(t, err)
-		err = instanceUS.WaitForAppIsRunning()
-		require.NoError(t, err)
-		err = instanceCZ.WaitForAppIsRunning()
-		require.NoError(t, err)
-	})
-
-	euLocalTargets := instanceEU.GetLocalTargets()
-	usLocalTargets := instanceUS.GetLocalTargets()
-	czLocalTargets := instanceCZ.GetLocalTargets()
-	euAndCZTargets := append(euLocalTargets, czLocalTargets...)
-	czAndUSTargets := append(czLocalTargets, usLocalTargets...)
-	expectedIPs := append(euAndCZTargets, usLocalTargets...)
-
-	t.Run("kill podinfo on the second (us) cluster", func(t *testing.T) {
-		instanceUS.StopTestApp()
-		err = instanceEU.WaitForExpected(euAndCZTargets)
-		require.NoError(t, err)
-		err = instanceUS.WaitForExpected(euAndCZTargets)
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected(euAndCZTargets)
-		require.NoError(t, err)
-	})
-
-	t.Run("kill podinfo on the third (cz) cluster", func(t *testing.T) {
-		instanceCZ.StopTestApp()
-		err = instanceUS.WaitForExpected(euLocalTargets)
-		require.NoError(t, err)
-		err = instanceEU.WaitForExpected(euLocalTargets)
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected(euLocalTargets)
-		require.NoError(t, err)
-	})
-
-	t.Run("kill podinfo on the first (eu) cluster", func(t *testing.T) {
-		instanceEU.StopTestApp()
-		err = instanceUS.WaitForExpected([]string{})
-		require.NoError(t, err)
-		err = instanceEU.WaitForExpected([]string{})
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected([]string{})
-		require.NoError(t, err)
-	})
-
-	t.Run("start podinfo on the third (cz) cluster", func(t *testing.T) {
-		instanceCZ.StartTestApp()
-		err = instanceEU.WaitForExpected(czLocalTargets)
-		require.NoError(t, err)
-		err = instanceUS.WaitForExpected(czLocalTargets)
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected(czLocalTargets)
-		require.NoError(t, err)
-	})
-
-	t.Run("start podinfo on the second (us) cluster", func(t *testing.T) {
-		instanceUS.StartTestApp()
-		err = instanceEU.WaitForExpected(czAndUSTargets)
-		require.NoError(t, err)
-		err = instanceUS.WaitForExpected(czAndUSTargets)
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected(czAndUSTargets)
-		require.NoError(t, err)
-	})
-
-	t.Run("start podinfo on the first (eu) cluster", func(t *testing.T) {
-		// start app in the both clusters
-		instanceEU.StartTestApp()
-		err = instanceEU.WaitForExpected(expectedIPs)
-		require.NoError(t, err)
-		err = instanceUS.WaitForExpected(expectedIPs)
-		require.NoError(t, err)
-		err = instanceCZ.WaitForExpected(expectedIPs)
-		require.NoError(t, err)
-	})
+	abstractTestFullRoundRobin(t, settings.ClustersNumber)
 }

terratest/test/k8gb_ingress_annotation_failover_test.go

@@ -1,3 +1,5 @@
+// +build failover all
+
 /*
 Copyright 2021 The k8gb Contributors.
 

terratest/test/k8gb_ingress_annotation_rr_test.go

@@ -1,3 +1,5 @@
+// +build rr all
+
 /*
 Copyright 2021 The k8gb Contributors.