Skip to content

Commit

Permalink
GetStatus implemented
Browse files Browse the repository at this point in the history
  • Loading branch information
@kuritka
kuritka committed Jul 7, 2021
1 parent 305e4bf commit 7839c52
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 64 deletions.
9 changes: 4 additions & 5 deletions terratest/test/k8gb_failover_http_test.go → ...est/test/k8gb_failover_playground_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -51,7 +50,7 @@ func TestFailoverPlayground(t *testing.T) {
euLocalTargets := instanceEU.GetLocalTargets()
usLocalTargets := instanceUS.GetLocalTargets()

waitAndHitTestApp := func(geoTag string, localTargets []string) {
actAndAssert := func(test, geoTag string, localTargets []string) {
// waiting for DNS sync
err = instanceEU.WaitForExpected(localTargets)
require.NoError(t, err)
Expand All @@ -65,16 +64,16 @@ func TestFailoverPlayground(t *testing.T) {
}

t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) {
waitAndHitTestApp(euGeoTag, euLocalTargets)
actAndAssert(t.Name(), euGeoTag, euLocalTargets)
})

t.Run("stop podinfo on eu cluster", func(t *testing.T) {
instanceEU.StopTestApp()
waitAndHitTestApp(usGeoTag, usLocalTargets)
actAndAssert(t.Name(), usGeoTag, usLocalTargets)
})

t.Run("start podinfo again on eu cluster", func(t *testing.T) {
instanceEU.StartTestApp()
waitAndHitTestApp(euGeoTag, euLocalTargets)
actAndAssert(t.Name(), euGeoTag, euLocalTargets)
})
}
44 changes: 22 additions & 22 deletions terratest/test/k8gb_full_failover_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,58 +29,58 @@ func TestFullFailover(t *testing.T) {
const host = "terratest-failover.cloud.example.com"
const gslbPath = "../examples/failover.yaml"

instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
WithGslb(gslbPath, host).
WithTestApp("us").
WithTestApp("eu").
Start()
require.NoError(t, err)
defer instance1.Kill()
instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
defer instanceEU.Kill()
instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
WithGslb(gslbPath, host).
WithTestApp("eu").
WithTestApp("us").
Start()
require.NoError(t, err)
defer instance2.Kill()
defer instanceUS.Kill()

instance1LocalTargets := instance1.GetLocalTargets()
instance2LocalTargets := instance2.GetLocalTargets()
instance1LocalTargets := instanceEU.GetLocalTargets()
instance2LocalTargets := instanceUS.GetLocalTargets()

t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) {
err = instance1.WaitForExpected(instance1LocalTargets)
err = instanceEU.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
err = instance2.WaitForExpected(instance1LocalTargets)
err = instanceUS.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
})

t.Run("kill podinfo on the second cluster", func(t *testing.T) {
instance2.StopTestApp()
err = instance2.WaitForExpected(instance1LocalTargets)
instanceUS.StopTestApp()
err = instanceUS.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
err = instance1.WaitForExpected(instance1LocalTargets)
err = instanceEU.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
})

t.Run("kill podinfo on the first cluster", func(t *testing.T) {
instance1.StopTestApp()
err = instance1.WaitForExpected([]string{})
instanceEU.StopTestApp()
err = instanceEU.WaitForExpected([]string{})
require.NoError(t, err)
err = instance2.WaitForExpected([]string{})
err = instanceUS.WaitForExpected([]string{})
require.NoError(t, err)
})

t.Run("start podinfo on the second cluster", func(t *testing.T) {
instance2.StartTestApp()
err = instance2.WaitForExpected(instance2LocalTargets)
instanceUS.StartTestApp()
err = instanceUS.WaitForExpected(instance2LocalTargets)
require.NoError(t, err)
err = instance1.WaitForExpected(instance2LocalTargets)
err = instanceEU.WaitForExpected(instance2LocalTargets)
require.NoError(t, err)
})

t.Run("start podinfo on the first cluster", func(t *testing.T) {
instance1.StartTestApp()
err = instance1.WaitForExpected(instance1LocalTargets)
instanceEU.StartTestApp()
err = instanceEU.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
err = instance2.WaitForExpected(instance1LocalTargets)
err = instanceUS.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
})
}
44 changes: 22 additions & 22 deletions terratest/test/k8gb_full_roundrobin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,60 +29,60 @@ func TestFullRoundRobin(t *testing.T) {
const host = "roundrobin-test.cloud.example.com"
const gslbPath = "../examples/roundrobin2.yaml"

instance1, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
WithGslb(gslbPath, host).
WithTestApp("us").
WithTestApp("eu").
Start()
require.NoError(t, err)
defer instance1.Kill()
instance2, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
defer instanceEU.Kill()
instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
WithGslb(gslbPath, host).
WithTestApp("eu").
WithTestApp("us").
Start()
require.NoError(t, err)
defer instance2.Kill()
defer instanceUS.Kill()

instance1LocalTargets := instance1.GetLocalTargets()
instance2LocalTargets := instance2.GetLocalTargets()
instance1LocalTargets := instanceEU.GetLocalTargets()
instance2LocalTargets := instanceUS.GetLocalTargets()
expectedIPs := append(instance1LocalTargets, instance2LocalTargets...)

t.Run("round-robin on two concurrent clusters with podinfo running", func(t *testing.T) {
err = instance1.WaitForExpected(expectedIPs)
err = instanceEU.WaitForExpected(expectedIPs)
require.NoError(t, err)
err = instance2.WaitForExpected(expectedIPs)
err = instanceUS.WaitForExpected(expectedIPs)
require.NoError(t, err)
})

t.Run("kill podinfo on the second cluster", func(t *testing.T) {
instance2.StopTestApp()
err = instance1.WaitForExpected(instance1LocalTargets)
instanceUS.StopTestApp()
err = instanceEU.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
err = instance2.WaitForExpected(instance1LocalTargets)
err = instanceUS.WaitForExpected(instance1LocalTargets)
require.NoError(t, err)
})

t.Run("kill podinfo on the first cluster", func(t *testing.T) {
instance1.StopTestApp()
err = instance2.WaitForExpected([]string{})
instanceEU.StopTestApp()
err = instanceUS.WaitForExpected([]string{})
require.NoError(t, err)
err = instance1.WaitForExpected([]string{})
err = instanceEU.WaitForExpected([]string{})
require.NoError(t, err)
})

t.Run("start podinfo on the second cluster", func(t *testing.T) {
instance2.StartTestApp()
err = instance1.WaitForExpected(instance2LocalTargets)
instanceUS.StartTestApp()
err = instanceEU.WaitForExpected(instance2LocalTargets)
require.NoError(t, err)
err = instance2.WaitForExpected(instance2LocalTargets)
err = instanceUS.WaitForExpected(instance2LocalTargets)
require.NoError(t, err)
})

t.Run("start podinfo on the first cluster", func(t *testing.T) {
// start app in the both clusters
instance1.StartTestApp()
err = instance1.WaitForExpected(expectedIPs)
instanceEU.StartTestApp()
err = instanceEU.WaitForExpected(expectedIPs)
require.NoError(t, err)
err = instance2.WaitForExpected(expectedIPs)
err = instanceUS.WaitForExpected(expectedIPs)
require.NoError(t, err)
})
}
96 changes: 82 additions & 14 deletions terratest/utils/extensions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"time"

"github.com/AbsaOSS/gopkg/dns"

gopkgstr "github.com/AbsaOSS/gopkg/strings"
"github.com/gruntwork-io/terratest/modules/helm"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/gruntwork-io/terratest/modules/random"
Expand Down Expand Up @@ -79,6 +79,25 @@ type TestAppResult struct {
Body string
}

// InstanceStatus provides a simplified overview of the instance status
type InstanceStatus struct {
Annotation string `json:"name"`
AppMessage string `json:"app-msg"`
AppRunning bool `json:"app-runnig"`
AppReplicas string `json:"app-pod-runnig"`
LocalTargets []string `json:"local-targets-ip"`
Ingresses []string `json:"ingress-ip"`
Dig []string `json:"dig-result"`
CoreDNS string `json:"coredns-ip"`
GslbHealthStatus string `json:"gslb-status"`
Cluster string `json:"cluster"`
Namespace string `json:"namespace"`
Endpoint0DNSName string `json:"ep0-dns-name"`
Endpoint0Targets string `json:"ep0-dns-targets"`
Endpoint1DNSName string `json:"ep1-dns-name"`
Endpoint1Targets string `json:"ep1-dns-targets"`
}

func NewWorkflow(t *testing.T, cluster string, port int) *Workflow {
var err error
if cluster == "" {
Expand All @@ -88,6 +107,7 @@ func NewWorkflow(t *testing.T, cluster string, port int) *Workflow {
err = fmt.Errorf("invalid port")
}
w := new(Workflow)
w.cluster = cluster
w.namespace = fmt.Sprintf("k8gb-test-%s", strings.ToLower(random.UniqueId()))
w.k8sOptions = k8s.NewKubectlOptions(cluster, "", w.namespace)
w.t = t
Expand Down Expand Up @@ -276,21 +296,15 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) {
// WaitForExpected waits until GSLB dig doesnt return list of expected IP's
func (i *Instance) WaitForExpected(expectedIPs []string) (err error) {
_, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
if err != nil {
fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s",expectedIPs)).String())
}
return
}

// String retrieves rough information about cluster
func (i *Instance) String() (out string) {
testAppMsg := "-"
out = fmt.Sprintf("------ %s/%s ------", i.w.cluster, i.w.namespace)
out += fmt.Sprintf("\n local targets IP's: %s", i.GetLocalTargets())
out += fmt.Sprintf(" ingress IP's: %s", i.GetIngressIPs())
out += fmt.Sprintf(" core DNS IP: %s", i.GetCoreDNSIP())
if i.w.state.testApp.isRunning {
testAppMsg = i.w.state.testApp.message
}
out += fmt.Sprintf(" test app message: %s", testAppMsg)
out += fmt.Sprintf("-----------------")
return
return fmt.Sprintf("Instance: %s:%s", i.w.cluster, i.w.namespace)
}

// Dig returns a list of IP addresses from CoreDNS that belong to the instance
Expand Down Expand Up @@ -318,12 +332,66 @@ func (i *Instance) HitTestApp() (result *TestAppResult) {
command := fmt.Sprintf("echo nameserver %s > /etc/resolv.conf && wget -qO - %s", coreDDNSIP, i.w.state.gslb.host)
result.Body, err = RunBusyBoxCommand(i.w.t, i.w.k8sOptions, command)
require.NoError(i.w.t, err, "busybox", command, result.Body)
parsedJson := strings.Split(result.Body, "}")[0] + "}"
err = json.Unmarshal([]byte(parsedJson), result)
// unwrap json from busybox messages
parsedJson := strings.Split(result.Body, "}")[0]
parsedJson = strings.Split(parsedJson, "{")[1]

err = json.Unmarshal([]byte("{"+parsedJson+"}"), result)
require.NoError(i.w.t, err, "unmarshall json", result.Body)
return
}

// GetStatus reads overall status about instance. Status can be used for assertion as well as printed to test output
func (i *Instance) GetStatus(name string) (s *InstanceStatus){
const na = "n/a"
var err error
s = new(InstanceStatus)
s.Annotation = name
s.Cluster = i.w.cluster
s.Namespace = i.w.namespace
s.Dig = i.Dig()
s.LocalTargets = i.GetLocalTargets()
s.Ingresses = i.GetIngressIPs()
s.CoreDNS = i.GetCoreDNSIP()
s.AppMessage = i.w.state.testApp.message
s.AppRunning = i.w.state.testApp.isRunning
s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments","frontend-podinfo",
"-o","custom-columns=STATUS:.status.replicas", "--no-headers")
if err != nil {
s.AppReplicas = na
}
s.GslbHealthStatus, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "gslb", i.w.state.gslb.name, "-o",
"custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers")
if err != nil {
s.GslbHealthStatus = na
}
s.Endpoint0DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[0].dnsName", "--no-headers")
if err != nil {
s.Endpoint0DNSName = na
}
s.Endpoint0Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[0].targets", "--no-headers")
if err != nil {
s.Endpoint0Targets = na
}
s.Endpoint1DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[1].dnsName", "--no-headers")
if err != nil {
s.Endpoint1DNSName = na
}
s.Endpoint1Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[1].targets", "--no-headers")
if err != nil {
s.Endpoint1Targets = na
}
return
}

func (s *InstanceStatus) String() string {
return gopkgstr.ToString(s)
}

func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []string) (output []string, err error) {
return DoWithRetryWaitingForValueE(
t,
Expand Down
1 change: 0 additions & 1 deletion terratest/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ func AssertGslbStatus(t *testing.T, options *k8s.KubectlOptions, gslbName, servi
t.Helper()

actualHealthStatus := func() ([]string, error) {
//-o custom-columns=SERVICESTATUS:.status.serviceHealth --no-headers
k8gbServiceHealth, err := k8s.RunKubectlAndGetOutputE(t, options, "get", "gslb", gslbName, "-o",
"custom-columns=SERVICESTATUS:.status.serviceHealth", "--no-headers")
if err != nil {
Expand Down

0 comments on commit 7839c52

Please sign in to comment.