From 7839c524e67a68af3c9ef7a9ea308eae6d373663 Mon Sep 17 00:00:00 2001 From: kuritka Date: Wed, 7 Jul 2021 10:14:48 +0200 Subject: [PATCH] GetStatus implemented --- ...st.go => k8gb_failover_playground_test.go} | 9 +- terratest/test/k8gb_full_failover_test.go | 44 ++++----- terratest/test/k8gb_full_roundrobin_test.go | 44 ++++----- terratest/utils/extensions.go | 96 ++++++++++++++++--- terratest/utils/utils.go | 1 - 5 files changed, 130 insertions(+), 64 deletions(-) rename terratest/test/{k8gb_failover_http_test.go => k8gb_failover_playground_test.go} (91%) diff --git a/terratest/test/k8gb_failover_http_test.go b/terratest/test/k8gb_failover_playground_test.go similarity index 91% rename from terratest/test/k8gb_failover_http_test.go rename to terratest/test/k8gb_failover_playground_test.go index 7061dab578..fc4f201773 100644 --- a/terratest/test/k8gb_failover_http_test.go +++ b/terratest/test/k8gb_failover_playground_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) @@ -51,7 +50,7 @@ func TestFailoverPlayground(t *testing.T) { euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - waitAndHitTestApp := func(geoTag string, localTargets []string) { + actAndAssert := func(test, geoTag string, localTargets []string) { // waiting for DNS sync err = instanceEU.WaitForExpected(localTargets) require.NoError(t, err) @@ -65,16 +64,16 @@ func TestFailoverPlayground(t *testing.T) { } t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) { - waitAndHitTestApp(euGeoTag, euLocalTargets) + actAndAssert(t.Name(), euGeoTag, euLocalTargets) }) t.Run("stop podinfo on eu cluster", func(t *testing.T) { instanceEU.StopTestApp() - waitAndHitTestApp(usGeoTag, usLocalTargets) + actAndAssert(t.Name(), usGeoTag, usLocalTargets) }) t.Run("start podinfo again on eu cluster", func(t *testing.T) { instanceEU.StartTestApp() - waitAndHitTestApp(euGeoTag, euLocalTargets) + actAndAssert(t.Name(), euGeoTag, euLocalTargets) }) } diff --git a/terratest/test/k8gb_full_failover_test.go b/terratest/test/k8gb_full_failover_test.go index 4893e2b36e..69bbfbbd9d 100644 --- a/terratest/test/k8gb_full_failover_test.go +++ b/terratest/test/k8gb_full_failover_test.go @@ -29,58 +29,58 @@ func TestFullFailover(t *testing.T) { const host = "terratest-failover.cloud.example.com" const gslbPath = "../examples/failover.yaml" - instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("us"). + WithTestApp("eu"). Start() require.NoError(t, err) - defer instance1.Kill() - instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + defer instanceEU.Kill() + instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("eu"). + WithTestApp("us"). Start() require.NoError(t, err) - defer instance2.Kill() + defer instanceUS.Kill() - instance1LocalTargets := instance1.GetLocalTargets() - instance2LocalTargets := instance2.GetLocalTargets() + instance1LocalTargets := instanceEU.GetLocalTargets() + instance2LocalTargets := instanceUS.GetLocalTargets() t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) { - err = instance1.WaitForExpected(instance1LocalTargets) + err = instanceEU.WaitForExpected(instance1LocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForExpected(instance1LocalTargets) require.NoError(t, err) }) t.Run("kill podinfo on the second cluster", func(t *testing.T) { - instance2.StopTestApp() - err = instance2.WaitForExpected(instance1LocalTargets) + instanceUS.StopTestApp() + err = instanceUS.WaitForExpected(instance1LocalTargets) require.NoError(t, err) - err = instance1.WaitForExpected(instance1LocalTargets) + err = instanceEU.WaitForExpected(instance1LocalTargets) require.NoError(t, err) }) t.Run("kill podinfo on the first cluster", func(t *testing.T) { - instance1.StopTestApp() - err = instance1.WaitForExpected([]string{}) + instanceEU.StopTestApp() + err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) - err = instance2.WaitForExpected([]string{}) + err = instanceUS.WaitForExpected([]string{}) require.NoError(t, err) }) t.Run("start podinfo on the second cluster", func(t *testing.T) { - instance2.StartTestApp() - err = instance2.WaitForExpected(instance2LocalTargets) + instanceUS.StartTestApp() + err = instanceUS.WaitForExpected(instance2LocalTargets) require.NoError(t, err) - err = instance1.WaitForExpected(instance2LocalTargets) + err = instanceEU.WaitForExpected(instance2LocalTargets) require.NoError(t, err) }) t.Run("start podinfo on the first cluster", func(t *testing.T) { - instance1.StartTestApp() - err = instance1.WaitForExpected(instance1LocalTargets) + instanceEU.StartTestApp() + err = instanceEU.WaitForExpected(instance1LocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForExpected(instance1LocalTargets) require.NoError(t, err) }) } diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index d365c56365..b6f330af58 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -29,60 +29,60 @@ func TestFullRoundRobin(t *testing.T) { const host = "roundrobin-test.cloud.example.com" const gslbPath = "../examples/roundrobin2.yaml" - instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("us"). + WithTestApp("eu"). Start() require.NoError(t, err) - defer instance1.Kill() - instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + defer instanceEU.Kill() + instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("eu"). + WithTestApp("us"). Start() require.NoError(t, err) - defer instance2.Kill() + defer instanceUS.Kill() - instance1LocalTargets := instance1.GetLocalTargets() - instance2LocalTargets := instance2.GetLocalTargets() + instance1LocalTargets := instanceEU.GetLocalTargets() + instance2LocalTargets := instanceUS.GetLocalTargets() expectedIPs := append(instance1LocalTargets, instance2LocalTargets...) t.Run("round-robin on two concurrent clusters with podinfo running", func(t *testing.T) { - err = instance1.WaitForExpected(expectedIPs) + err = instanceEU.WaitForExpected(expectedIPs) require.NoError(t, err) - err = instance2.WaitForExpected(expectedIPs) + err = instanceUS.WaitForExpected(expectedIPs) require.NoError(t, err) }) t.Run("kill podinfo on the second cluster", func(t *testing.T) { - instance2.StopTestApp() - err = instance1.WaitForExpected(instance1LocalTargets) + instanceUS.StopTestApp() + err = instanceEU.WaitForExpected(instance1LocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance1LocalTargets) + err = instanceUS.WaitForExpected(instance1LocalTargets) require.NoError(t, err) }) t.Run("kill podinfo on the first cluster", func(t *testing.T) { - instance1.StopTestApp() - err = instance2.WaitForExpected([]string{}) + instanceEU.StopTestApp() + err = instanceUS.WaitForExpected([]string{}) require.NoError(t, err) - err = instance1.WaitForExpected([]string{}) + err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) }) t.Run("start podinfo on the second cluster", func(t *testing.T) { - instance2.StartTestApp() - err = instance1.WaitForExpected(instance2LocalTargets) + instanceUS.StartTestApp() + err = instanceEU.WaitForExpected(instance2LocalTargets) require.NoError(t, err) - err = instance2.WaitForExpected(instance2LocalTargets) + err = instanceUS.WaitForExpected(instance2LocalTargets) require.NoError(t, err) }) t.Run("start podinfo on the first cluster", func(t *testing.T) { // start app in the both clusters - instance1.StartTestApp() - err = instance1.WaitForExpected(expectedIPs) + instanceEU.StartTestApp() + err = instanceEU.WaitForExpected(expectedIPs) require.NoError(t, err) - err = instance2.WaitForExpected(expectedIPs) + err = instanceUS.WaitForExpected(expectedIPs) require.NoError(t, err) }) } diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index 9c8574ef45..97c99b5cf3 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -28,7 +28,7 @@ import ( "time" "github.com/AbsaOSS/gopkg/dns" - + gopkgstr "github.com/AbsaOSS/gopkg/strings" "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" @@ -79,6 +79,25 @@ type TestAppResult struct { Body string } +// InstanceStatus provides a simplified overview of the instance status +type InstanceStatus struct { + Annotation string `json:"name"` + AppMessage string `json:"app-msg"` + AppRunning bool `json:"app-runnig"` + AppReplicas string `json:"app-pod-runnig"` + LocalTargets []string `json:"local-targets-ip"` + Ingresses []string `json:"ingress-ip"` + Dig []string `json:"dig-result"` + CoreDNS string `json:"coredns-ip"` + GslbHealthStatus string `json:"gslb-status"` + Cluster string `json:"cluster"` + Namespace string `json:"namespace"` + Endpoint0DNSName string `json:"ep0-dns-name"` + Endpoint0Targets string `json:"ep0-dns-targets"` + Endpoint1DNSName string `json:"ep1-dns-name"` + Endpoint1Targets string `json:"ep1-dns-targets"` +} + func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { var err error if cluster == "" { @@ -88,6 +107,7 @@ func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { err = fmt.Errorf("invalid port") } w := new(Workflow) + w.cluster = cluster w.namespace = fmt.Sprintf("k8gb-test-%s", strings.ToLower(random.UniqueId())) w.k8sOptions = k8s.NewKubectlOptions(cluster, "", w.namespace) w.t = t @@ -276,21 +296,15 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) { // WaitForExpected waits until GSLB dig doesnt return list of expected IP's func (i *Instance) WaitForExpected(expectedIPs []string) (err error) { _, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs) + if err != nil { + fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s",expectedIPs)).String()) + } return } +// String retrieves rough information about cluster func (i *Instance) String() (out string) { - testAppMsg := "-" - out = fmt.Sprintf("------ %s/%s ------", i.w.cluster, i.w.namespace) - out += fmt.Sprintf("\n local targets IP's: %s", i.GetLocalTargets()) - out += fmt.Sprintf(" ingress IP's: %s", i.GetIngressIPs()) - out += fmt.Sprintf(" core DNS IP: %s", i.GetCoreDNSIP()) - if i.w.state.testApp.isRunning { - testAppMsg = i.w.state.testApp.message - } - out += fmt.Sprintf(" test app message: %s", testAppMsg) - out += fmt.Sprintf("-----------------") - return + return fmt.Sprintf("Instance: %s:%s", i.w.cluster, i.w.namespace) } // Dig returns a list of IP addresses from CoreDNS that belong to the instance @@ -318,12 +332,66 @@ func (i *Instance) HitTestApp() (result *TestAppResult) { command := fmt.Sprintf("echo nameserver %s > /etc/resolv.conf && wget -qO - %s", coreDDNSIP, i.w.state.gslb.host) result.Body, err = RunBusyBoxCommand(i.w.t, i.w.k8sOptions, command) require.NoError(i.w.t, err, "busybox", command, result.Body) - parsedJson := strings.Split(result.Body, "}")[0] + "}" - err = json.Unmarshal([]byte(parsedJson), result) + // unwrap json from busybox messages + parsedJson := strings.Split(result.Body, "}")[0] + parsedJson = strings.Split(parsedJson, "{")[1] + + err = json.Unmarshal([]byte("{"+parsedJson+"}"), result) require.NoError(i.w.t, err, "unmarshall json", result.Body) return } +// GetStatus reads overall status about instance. Status can be used for assertion as well as printed to test output +func (i *Instance) GetStatus(name string) (s *InstanceStatus){ + const na = "n/a" + var err error + s = new(InstanceStatus) + s.Annotation = name + s.Cluster = i.w.cluster + s.Namespace = i.w.namespace + s.Dig = i.Dig() + s.LocalTargets = i.GetLocalTargets() + s.Ingresses = i.GetIngressIPs() + s.CoreDNS = i.GetCoreDNSIP() + s.AppMessage = i.w.state.testApp.message + s.AppRunning = i.w.state.testApp.isRunning + s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments","frontend-podinfo", + "-o","custom-columns=STATUS:.status.replicas", "--no-headers") + if err != nil { + s.AppReplicas = na + } + s.GslbHealthStatus, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "gslb", i.w.state.gslb.name, "-o", + "custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers") + if err != nil { + s.GslbHealthStatus = na + } + s.Endpoint0DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[0].dnsName", "--no-headers") + if err != nil { + s.Endpoint0DNSName = na + } + s.Endpoint0Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[0].targets", "--no-headers") + if err != nil { + s.Endpoint0Targets = na + } + s.Endpoint1DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[1].dnsName", "--no-headers") + if err != nil { + s.Endpoint1DNSName = na + } + s.Endpoint1Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + "custom-columns=SERVICESTATUS:.spec.endpoints[1].targets", "--no-headers") + if err != nil { + s.Endpoint1Targets = na + } + return +} + +func (s *InstanceStatus) String() string { + return gopkgstr.ToString(s) +} + func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []string) (output []string, err error) { return DoWithRetryWaitingForValueE( t, diff --git a/terratest/utils/utils.go b/terratest/utils/utils.go index 863961dd66..78c2d016d5 100644 --- a/terratest/utils/utils.go +++ b/terratest/utils/utils.go @@ -171,7 +171,6 @@ func AssertGslbStatus(t *testing.T, options *k8s.KubectlOptions, gslbName, servi t.Helper() actualHealthStatus := func() ([]string, error) { - //-o custom-columns=SERVICESTATUS:.status.serviceHealth --no-headers k8gbServiceHealth, err := k8s.RunKubectlAndGetOutputE(t, options, "get", "gslb", gslbName, "-o", "custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers") if err != nil {