Failover HTTP test

related to #533

I enhanced terratest extensions and utils with a function that can run a command in busybox and a function that can read json output from test app.

Thanks to this I can automate tests from local playground.

Besides the test itself, I had to extend `WithTestApp(uiMessage string)` with the message argument and bump the subinfo from `4.0.6` to `5.0.1`.

Tests are asserting local targets which are digged after `WaitForAppIsRunning` pass, otherwise digged localtargets will be `[]string{}`.

I also implemented InstanceStatus, which is a functionality that takes all available instance data, such as Dig, endpoint status, CoreDNS IP etc... The functionality is useful for tracing errors and possible assertions.

Signed-off-by: kuritka <kuritka@gmail.com>

terratest/examples/failover-playground.yaml

@@ -0,0 +1,17 @@
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: test-gslb
+spec:
+  ingress:
+    rules:
+      - host: playground-failover.cloud.example.com
+        http:
+          paths:
+          - backend:
+              serviceName: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records
+              servicePort: http
+            path: /
+  strategy:
+    type: failover
+    primaryGeoTag: "eu"

terratest/test/k8gb_failover_playground_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2021 The k8gb Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
+*/
+package test
+
+import (
+	"k8gbterratest/utils"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestFailoverPlayground is equal to k8gb failover test running on local playground.
+// see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover
+func TestFailoverPlayground(t *testing.T) {
+	t.Parallel()
+	const host = "playground-failover.cloud.example.com"
+	const gslbPath = "../examples/failover-playground.yaml"
+	const euGeoTag = "eu"
+	const usGeoTag = "us"
+
+	instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
+		WithGslb(gslbPath, host).
+		WithTestApp(euGeoTag).
+		Start()
+	require.NoError(t, err)
+	defer instanceEU.Kill()
+	instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
+		WithGslb(gslbPath, host).
+		WithTestApp(usGeoTag).
+		Start()
+	require.NoError(t, err)
+	defer instanceUS.Kill()
+
+	actAndAssert := func(test, geoTag string, localTargets []string) {
+		// waiting for DNS sync
+		err = instanceEU.WaitForExpected(localTargets)
+		require.NoError(t, err)
+		err = instanceUS.WaitForExpected(localTargets)
+		require.NoError(t, err)
+		// hit testApp from both clusters
+		httpResult := instanceEU.HitTestApp()
+		assert.Equal(t, geoTag, httpResult.Message)
+		httpResult = instanceUS.HitTestApp()
+		assert.Equal(t, geoTag, httpResult.Message)
+	}
+
+	t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) {
+		err = instanceEU.WaitForAppIsRunning()
+		require.NoError(t, err)
+		err = instanceUS.WaitForAppIsRunning()
+		require.NoError(t, err)
+	})
+
+	euLocalTargets := instanceEU.GetLocalTargets()
+	usLocalTargets := instanceUS.GetLocalTargets()
+
+	t.Run("stop podinfo on eu cluster", func(t *testing.T) {
+		instanceEU.StopTestApp()
+		actAndAssert(t.Name(), usGeoTag, usLocalTargets)
+	})
+
+	t.Run("start podinfo again on eu cluster", func(t *testing.T) {
+		instanceEU.StartTestApp()
+		actAndAssert(t.Name(), euGeoTag, euLocalTargets)
+	})
+}

terratest/test/k8gb_full_failover_test.go

@@ -29,58 +29,58 @@ func TestFullFailover(t *testing.T) {
 	const host = "terratest-failover.cloud.example.com"
 	const gslbPath = "../examples/failover.yaml"
 
-	instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
+	instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
 		WithGslb(gslbPath, host).
-		WithTestApp().
+		WithTestApp("eu").
 		Start()
 	require.NoError(t, err)
-	defer instance1.Kill()
-	instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
+	defer instanceEU.Kill()
+	instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
 		WithGslb(gslbPath, host).
-		WithTestApp().
+		WithTestApp("us").
 		Start()
 	require.NoError(t, err)
-	defer instance2.Kill()
-
-	instance1LocalTargets := instance1.GetLocalTargets()
-	instance2LocalTargets := instance2.GetLocalTargets()
+	defer instanceUS.Kill()
 
 	t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) {
-		err = instance1.WaitForExpected(instance1LocalTargets)
+		err = instanceEU.WaitForAppIsRunning()
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(instance1LocalTargets)
+		err = instanceUS.WaitForAppIsRunning()
 		require.NoError(t, err)
 	})
 
+	euLocalTargets := instanceEU.GetLocalTargets()
+	usLocalTargets := instanceUS.GetLocalTargets()
+
 	t.Run("kill podinfo on the second cluster", func(t *testing.T) {
-		instance2.StopTestApp()
-		err = instance2.WaitForExpected(instance1LocalTargets)
+		instanceUS.StopTestApp()
+		err = instanceUS.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
-		err = instance1.WaitForExpected(instance1LocalTargets)
+		err = instanceEU.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
 	})
 
 	t.Run("kill podinfo on the first cluster", func(t *testing.T) {
-		instance1.StopTestApp()
-		err = instance1.WaitForExpected([]string{})
+		instanceEU.StopTestApp()
+		err = instanceEU.WaitForExpected([]string{})
 		require.NoError(t, err)
-		err = instance2.WaitForExpected([]string{})
+		err = instanceUS.WaitForExpected([]string{})
 		require.NoError(t, err)
 	})
 
 	t.Run("start podinfo on the second cluster", func(t *testing.T) {
-		instance2.StartTestApp()
-		err = instance2.WaitForExpected(instance2LocalTargets)
+		instanceUS.StartTestApp()
+		err = instanceUS.WaitForExpected(usLocalTargets)
 		require.NoError(t, err)
-		err = instance1.WaitForExpected(instance2LocalTargets)
+		err = instanceEU.WaitForExpected(usLocalTargets)
 		require.NoError(t, err)
 	})
 
 	t.Run("start podinfo on the first cluster", func(t *testing.T) {
-		instance1.StartTestApp()
-		err = instance1.WaitForExpected(instance1LocalTargets)
+		instanceEU.StartTestApp()
+		err = instanceEU.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(instance1LocalTargets)
+		err = instanceUS.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
 	})
 }

terratest/test/k8gb_full_roundrobin_test.go

@@ -29,60 +29,60 @@ func TestFullRoundRobin(t *testing.T) {
 	const host = "roundrobin-test.cloud.example.com"
 	const gslbPath = "../examples/roundrobin2.yaml"
 
-	instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
+	instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
 		WithGslb(gslbPath, host).
-		WithTestApp().
+		WithTestApp("eu").
 		Start()
 	require.NoError(t, err)
-	defer instance1.Kill()
-	instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
+	defer instanceEU.Kill()
+	instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
 		WithGslb(gslbPath, host).
-		WithTestApp().
+		WithTestApp("us").
 		Start()
 	require.NoError(t, err)
-	defer instance2.Kill()
-
-	instance1LocalTargets := instance1.GetLocalTargets()
-	instance2LocalTargets := instance2.GetLocalTargets()
-	expectedIPs := append(instance1LocalTargets, instance2LocalTargets...)
+	defer instanceUS.Kill()
 
 	t.Run("round-robin on two concurrent clusters with podinfo running", func(t *testing.T) {
-		err = instance1.WaitForExpected(expectedIPs)
+		err = instanceEU.WaitForAppIsRunning()
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(expectedIPs)
+		err = instanceUS.WaitForAppIsRunning()
 		require.NoError(t, err)
 	})
 
+	euLocalTargets := instanceEU.GetLocalTargets()
+	usLocalTargets := instanceUS.GetLocalTargets()
+	expectedIPs := append(euLocalTargets, usLocalTargets...)
+
 	t.Run("kill podinfo on the second cluster", func(t *testing.T) {
-		instance2.StopTestApp()
-		err = instance1.WaitForExpected(instance1LocalTargets)
+		instanceUS.StopTestApp()
+		err = instanceEU.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(instance1LocalTargets)
+		err = instanceUS.WaitForExpected(euLocalTargets)
 		require.NoError(t, err)
 	})
 
 	t.Run("kill podinfo on the first cluster", func(t *testing.T) {
-		instance1.StopTestApp()
-		err = instance2.WaitForExpected([]string{})
+		instanceEU.StopTestApp()
+		err = instanceUS.WaitForExpected([]string{})
 		require.NoError(t, err)
-		err = instance1.WaitForExpected([]string{})
+		err = instanceEU.WaitForExpected([]string{})
 		require.NoError(t, err)
 	})
 
 	t.Run("start podinfo on the second cluster", func(t *testing.T) {
-		instance2.StartTestApp()
-		err = instance1.WaitForExpected(instance2LocalTargets)
+		instanceUS.StartTestApp()
+		err = instanceEU.WaitForExpected(usLocalTargets)
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(instance2LocalTargets)
+		err = instanceUS.WaitForExpected(usLocalTargets)
 		require.NoError(t, err)
 	})
 
 	t.Run("start podinfo on the first cluster", func(t *testing.T) {
 		// start app in the both clusters
-		instance1.StartTestApp()
-		err = instance1.WaitForExpected(expectedIPs)
+		instanceEU.StartTestApp()
+		err = instanceEU.WaitForExpected(expectedIPs)
 		require.NoError(t, err)
-		err = instance2.WaitForExpected(expectedIPs)
+		err = instanceUS.WaitForExpected(expectedIPs)
 		require.NoError(t, err)
 	})
 }