diff --git a/terratest/examples/failover-playground.yaml b/terratest/examples/failover-playground.yaml new file mode 100644 index 0000000000..39ed64bb62 --- /dev/null +++ b/terratest/examples/failover-playground.yaml @@ -0,0 +1,17 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + ingress: + rules: + - host: playground-failover.cloud.example.com + http: + paths: + - backend: + serviceName: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + servicePort: http + path: / + strategy: + type: failover + primaryGeoTag: "eu" diff --git a/terratest/test/k8gb_failover_playground_test.go b/terratest/test/k8gb_failover_playground_test.go new file mode 100644 index 0000000000..193a68035d --- /dev/null +++ b/terratest/test/k8gb_failover_playground_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ +package test + +import ( + "k8gbterratest/utils" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFailoverPlayground is equal to k8gb failover test running on local playground. +// see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover +func TestFailoverPlayground(t *testing.T) { + t.Parallel() + const host = "playground-failover.cloud.example.com" + const gslbPath = "../examples/failover-playground.yaml" + const euGeoTag = "eu" + const usGeoTag = "us" + + instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + WithGslb(gslbPath, host). + WithTestApp(euGeoTag). + Start() + require.NoError(t, err) + defer instanceEU.Kill() + instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + WithGslb(gslbPath, host). + WithTestApp(usGeoTag). + Start() + require.NoError(t, err) + defer instanceUS.Kill() + + actAndAssert := func(test, geoTag string, localTargets []string) { + // waiting for DNS sync + err = instanceEU.WaitForExpected(localTargets) + require.NoError(t, err) + err = instanceUS.WaitForExpected(localTargets) + require.NoError(t, err) + // hit testApp from both clusters + httpResult := instanceEU.HitTestApp() + assert.Equal(t, geoTag, httpResult.Message) + httpResult = instanceUS.HitTestApp() + assert.Equal(t, geoTag, httpResult.Message) + } + + t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) { + err = instanceEU.WaitForAppIsRunning() + require.NoError(t, err) + err = instanceUS.WaitForAppIsRunning() + require.NoError(t, err) + }) + + euLocalTargets := instanceEU.GetLocalTargets() + usLocalTargets := instanceUS.GetLocalTargets() + + t.Run("stop podinfo on eu cluster", func(t *testing.T) { + instanceEU.StopTestApp() + actAndAssert(t.Name(), usGeoTag, usLocalTargets) + }) + + t.Run("start podinfo again on eu cluster", func(t *testing.T) { + instanceEU.StartTestApp() + actAndAssert(t.Name(), euGeoTag, euLocalTargets) + }) +} diff --git a/terratest/test/k8gb_full_failover_test.go b/terratest/test/k8gb_full_failover_test.go index dd887b3593..900c0d49fa 100644 --- a/terratest/test/k8gb_full_failover_test.go +++ b/terratest/test/k8gb_full_failover_test.go @@ -29,58 +29,58 @@ func TestFullFailover(t *testing.T) { const host = "terratest-failover.cloud.example.com" const gslbPath = "../examples/failover.yaml" - instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp(). + WithTestApp("eu"). Start() require.NoError(t, err) - defer instance1.Kill() - instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + defer instanceEU.Kill() + instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp(). + WithTestApp("us"). Start() require.NoError(t, err) - defer instance2.Kill() - - instance1LocalTargets := instance1.GetLocalTargets() - instance2LocalTargets := instance2.GetLocalTargets() + defer instanceUS.Kill() t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) { - err = instance1.WaitForExpected(instance1LocalTargets) + err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForAppIsRunning() require.NoError(t, err) }) + euLocalTargets := instanceEU.GetLocalTargets() + usLocalTargets := instanceUS.GetLocalTargets() + t.Run("kill podinfo on the second cluster", func(t *testing.T) { - instance2.StopTestApp() - err = instance2.WaitForExpected(instance1LocalTargets) + instanceUS.StopTestApp() + err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) - err = instance1.WaitForExpected(instance1LocalTargets) + err = instanceEU.WaitForExpected(euLocalTargets) require.NoError(t, err) }) t.Run("kill podinfo on the first cluster", func(t *testing.T) { - instance1.StopTestApp() - err = instance1.WaitForExpected([]string{}) + instanceEU.StopTestApp() + err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) - err = instance2.WaitForExpected([]string{}) + err = instanceUS.WaitForExpected([]string{}) require.NoError(t, err) }) t.Run("start podinfo on the second cluster", func(t *testing.T) { - instance2.StartTestApp() - err = instance2.WaitForExpected(instance2LocalTargets) + instanceUS.StartTestApp() + err = instanceUS.WaitForExpected(usLocalTargets) require.NoError(t, err) - err = instance1.WaitForExpected(instance2LocalTargets) + err = instanceEU.WaitForExpected(usLocalTargets) require.NoError(t, err) }) t.Run("start podinfo on the first cluster", func(t *testing.T) { - instance1.StartTestApp() - err = instance1.WaitForExpected(instance1LocalTargets) + instanceEU.StartTestApp() + err = instanceEU.WaitForExpected(euLocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) }) } diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index 3238b8c01c..06f7af5404 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -29,60 +29,60 @@ func TestFullRoundRobin(t *testing.T) { const host = "roundrobin-test.cloud.example.com" const gslbPath = "../examples/roundrobin2.yaml" - instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp(). + WithTestApp("eu"). Start() require.NoError(t, err) - defer instance1.Kill() - instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + defer instanceEU.Kill() + instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp(). + WithTestApp("us"). Start() require.NoError(t, err) - defer instance2.Kill() - - instance1LocalTargets := instance1.GetLocalTargets() - instance2LocalTargets := instance2.GetLocalTargets() - expectedIPs := append(instance1LocalTargets, instance2LocalTargets...) + defer instanceUS.Kill() t.Run("round-robin on two concurrent clusters with podinfo running", func(t *testing.T) { - err = instance1.WaitForExpected(expectedIPs) + err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) - err = instance2.WaitForExpected(expectedIPs) + err = instanceUS.WaitForAppIsRunning() require.NoError(t, err) }) + euLocalTargets := instanceEU.GetLocalTargets() + usLocalTargets := instanceUS.GetLocalTargets() + expectedIPs := append(euLocalTargets, usLocalTargets...) + t.Run("kill podinfo on the second cluster", func(t *testing.T) { - instance2.StopTestApp() - err = instance1.WaitForExpected(instance1LocalTargets) + instanceUS.StopTestApp() + err = instanceEU.WaitForExpected(euLocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) }) t.Run("kill podinfo on the first cluster", func(t *testing.T) { - instance1.StopTestApp() - err = instance2.WaitForExpected([]string{}) + instanceEU.StopTestApp() + err = instanceUS.WaitForExpected([]string{}) require.NoError(t, err) - err = instance1.WaitForExpected([]string{}) + err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) }) t.Run("start podinfo on the second cluster", func(t *testing.T) { - instance2.StartTestApp() - err = instance1.WaitForExpected(instance2LocalTargets) + instanceUS.StartTestApp() + err = instanceEU.WaitForExpected(usLocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance2LocalTargets) + err = instanceUS.WaitForExpected(usLocalTargets) require.NoError(t, err) }) t.Run("start podinfo on the first cluster", func(t *testing.T) { // start app in the both clusters - instance1.StartTestApp() - err = instance1.WaitForExpected(expectedIPs) + instanceEU.StartTestApp() + err = instanceEU.WaitForExpected(expectedIPs) require.NoError(t, err) - err = instance2.WaitForExpected(expectedIPs) + err = instanceUS.WaitForExpected(expectedIPs) require.NoError(t, err) }) } diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index 094800f89f..618e500017 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -18,6 +18,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic package utils import ( + "encoding/json" "fmt" "io/ioutil" "path/filepath" @@ -27,7 +28,7 @@ import ( "time" "github.com/AbsaOSS/gopkg/dns" - + gopkgstr "github.com/AbsaOSS/gopkg/strings" "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" @@ -41,6 +42,7 @@ import ( type Workflow struct { error error namespace string + cluster string k8sOptions *k8s.KubectlOptions t *testing.T settings struct { @@ -52,6 +54,7 @@ type Workflow struct { namespaceCreated bool testApp struct { name string + message string isRunning bool isInstalled bool } @@ -68,6 +71,33 @@ type Instance struct { w *Workflow } +type TestAppResult struct { + Message string `json:"message"` + Version string `json:"version"` + Color string `json:"color"` + Pod string `json:"hostname"` + Body string +} + +// InstanceStatus provides a simplified overview of the instance status +type InstanceStatus struct { + Annotation string `json:"annotation"` + AppMessage string `json:"app-msg"` + AppRunning bool `json:"podinfo-running"` + AppReplicas string `json:"podinfo-replicas"` + LocalTargets []string `json:"local-targets-ip"` + Ingresses []string `json:"ingress-ip"` + Dig []string `json:"dig-result"` + CoreDNS string `json:"coredns-ip"` + GslbHealthStatus string `json:"gslb-status"` + Cluster string `json:"cluster"` + Namespace string `json:"namespace"` + Endpoint0DNSName string `json:"ep0-dns-name"` + Endpoint0Targets string `json:"ep0-dns-targets"` + Endpoint1DNSName string `json:"ep1-dns-name"` + Endpoint1Targets string `json:"ep1-dns-targets"` +} + func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { var err error if cluster == "" { @@ -77,6 +107,7 @@ func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { err = fmt.Errorf("invalid port") } w := new(Workflow) + w.cluster = cluster w.namespace = fmt.Sprintf("k8gb-test-%s", strings.ToLower(random.UniqueId())) w.k8sOptions = k8s.NewKubectlOptions(cluster, "", w.namespace) w.t = t @@ -116,9 +147,10 @@ func (w *Workflow) WithGslb(path, host string) *Workflow { return w } -func (w *Workflow) WithTestApp() *Workflow { +func (w *Workflow) WithTestApp(uiMessage string) *Workflow { w.state.testApp.isInstalled = true w.state.testApp.name = "frontend-podinfo" + w.state.testApp.message = uiMessage return w } @@ -163,11 +195,12 @@ func (w *Workflow) Start() (*Instance, error) { shell.RunCommand(w.t, helmRepoUpdate) helmOptions := helm.Options{ KubectlOptions: w.k8sOptions, - Version: "4.0.6", + Version: "5.1.1", + SetValues: map[string]string{"ui.message": w.state.testApp.message}, } helm.Install(w.t, &helmOptions, "podinfo/podinfo", "frontend") testAppFilter := metav1.ListOptions{ - LabelSelector: "app=" + w.state.testApp.name, + LabelSelector: "app.kubernetes.io/name=" + w.state.testApp.name, } k8s.WaitUntilNumPodsCreated(w.t, w.k8sOptions, testAppFilter, 1, 60, 1*time.Second) var testAppPods []corev1.Pod @@ -208,6 +241,19 @@ func (i *Instance) Kill() { } } +// GetCoreDNSIP gets core DNS IP address +func (i *Instance) GetCoreDNSIP() string { + cmd := shell.Command{ + Command: "kubectl", + Args: []string{"--context", i.w.k8sOptions.ContextName, "-n", "k8gb", "get", "svc", "k8gb-coredns", "--no-headers", "-o", "custom-columns=IP:spec.clusterIPs[0]"}, + Env: i.w.k8sOptions.Env, + } + out, err := shell.RunCommandAndGetOutputE(i.w.t, cmd) + require.NoError(i.w.t, err) + require.NotEqual(i.w.t, "", out) + return out +} + func (i *Instance) GetIngressIPs() []string { var ingressIPs []string ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.settings.ingressName) @@ -250,11 +296,31 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) { // WaitForExpected waits until GSLB dig doesnt return list of expected IP's func (i *Instance) WaitForExpected(expectedIPs []string) (err error) { _, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs) + if err != nil { + fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s",expectedIPs)).String()) + } return } -func (i *Instance) String() string { - return fmt.Sprintf("Instance: %s", i.w.namespace) +// WaitForAppIsRunning waits until app has one pod running +func (i *Instance) WaitForAppIsRunning() (err error) { + f := func()([]string, error){ + r := i.GetStatus("").AppReplicas + return []string{r}, nil + } + _, err = DoWithRetryWaitingForValueE( + i.w.t, + "Wait for failover to happen and coredns to pickup new values...", + 100, + time.Second*1, + func() ([]string, error) { return f() }, + []string{"1"}) + return +} + +// String retrieves rough information about cluster +func (i *Instance) String() (out string) { + return fmt.Sprintf("Instance: %s:%s", i.w.cluster, i.w.namespace) } // Dig returns a list of IP addresses from CoreDNS that belong to the instance @@ -272,6 +338,76 @@ func (i *Instance) GetLocalTargets() []string { return dig } +// HitTestApp makes HTTP GET to TestApp when installed otherwise panics. +// If the function successfully hits the TestApp, it returns the TestAppResult. +func (i *Instance) HitTestApp() (result *TestAppResult) { + require.True(i.w.t, i.w.state.testApp.isInstalled) + var err error + result = new(TestAppResult) + coreDDNSIP := i.GetCoreDNSIP() + command := fmt.Sprintf("echo nameserver %s > /etc/resolv.conf && wget -qO - %s", coreDDNSIP, i.w.state.gslb.host) + result.Body, err = RunBusyBoxCommand(i.w.t, i.w.k8sOptions, command) + require.NoError(i.w.t, err, "busybox", command, result.Body) + // unwrap json from busybox messages + parsedJson := strings.Split(result.Body, "}")[0] + parsedJson = strings.Split(parsedJson, "{")[1] + + err = json.Unmarshal([]byte("{"+parsedJson+"}"), result) + require.NoError(i.w.t, err, "unmarshall json", result.Body) + return +} + +// GetStatus reads overall status about instance. Status can be used for assertion as well as printed to test output +func (i *Instance) GetStatus(name string) (s *InstanceStatus){ + const na = "n/a" + var err error + s = new(InstanceStatus) + s.Annotation = name + s.Cluster = i.w.cluster + s.Namespace = i.w.namespace + s.Dig = i.Dig() + s.LocalTargets = i.GetLocalTargets() + s.Ingresses = i.GetIngressIPs() + s.CoreDNS = i.GetCoreDNSIP() + s.AppMessage = i.w.state.testApp.message + s.AppRunning = i.w.state.testApp.isRunning + s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments","frontend-podinfo", + "-o","custom-columns=STATUS:.status.replicas", "--no-headers") + if err != nil { + s.AppReplicas = na + } + s.GslbHealthStatus, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "gslb", i.w.state.gslb.name, "-o", + "custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers") + if err != nil { + s.GslbHealthStatus = na + } + s.Endpoint0DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[0].dnsName", "--no-headers") + if err != nil { + s.Endpoint0DNSName = na + } + s.Endpoint0Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[0].targets", "--no-headers") + if err != nil { + s.Endpoint0Targets = na + } + s.Endpoint1DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[1].dnsName", "--no-headers") + if err != nil { + s.Endpoint1DNSName = na + } + s.Endpoint1Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[1].targets", "--no-headers") + if err != nil { + s.Endpoint1Targets = na + } + return +} + +func (s *InstanceStatus) String() string { + return gopkgstr.ToString(s) +} + func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []string) (output []string, err error) { return DoWithRetryWaitingForValueE( t, diff --git a/terratest/utils/utils.go b/terratest/utils/utils.go index 5b15140a04..6eac356168 100644 --- a/terratest/utils/utils.go +++ b/terratest/utils/utils.go @@ -171,7 +171,6 @@ func AssertGslbStatus(t *testing.T, options *k8s.KubectlOptions, gslbName, servi t.Helper() actualHealthStatus := func() ([]string, error) { - //-o custom-columns=SERVICESTATUS:.status.serviceHealth --no-headers k8gbServiceHealth, err := k8s.RunKubectlAndGetOutputE(t, options, "get", "gslb", gslbName, "-o", "custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers") if err != nil { @@ -249,3 +248,13 @@ func EqualStringSlices(a, b []string) bool { } return true } + +// RunBusyBoxCommand the command argument is executed inside the busybox pod. It can be for example an HTTP request etc. +func RunBusyBoxCommand(t *testing.T, options *k8s.KubectlOptions, command string) (out string, err error) { + cmd := shell.Command{ + Command: "kubectl", + Args: []string{"--context", options.ContextName, "-n", options.Namespace, "run", "-i", "--rm", "busybox", "--restart", "Never", "--image", "busybox", "--", "sh", "-c", command}, + Env: options.Env, + } + return shell.RunCommandAndGetOutputE(t, cmd) +}