From 2841bc9a79d1398d3989bb3a516403dbcb1b2f0e Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 10 Oct 2018 08:52:17 -0700 Subject: [PATCH] E2E: actually fail when no InternalIP, ssh master tweaks, delete retries (#3982) --- test/e2e/kubernetes/deployment/deployment.go | 39 +++-- test/e2e/kubernetes/job/job.go | 6 +- test/e2e/kubernetes/kubernetes_test.go | 157 ++++++------------ .../persistentvolumeclaims.go | 21 ++- test/e2e/kubernetes/pod/pod.go | 23 ++- test/e2e/kubernetes/service/service.go | 21 ++- 6 files changed, 122 insertions(+), 145 deletions(-) diff --git a/test/e2e/kubernetes/deployment/deployment.go b/test/e2e/kubernetes/deployment/deployment.go index 3b9cb0431f..55c4448798 100644 --- a/test/e2e/kubernetes/deployment/deployment.go +++ b/test/e2e/kubernetes/deployment/deployment.go @@ -134,23 +134,36 @@ func Get(name, namespace string) (*Deployment, error) { } // Delete will delete a deployment in a given namespace -func (d *Deployment) Delete() error { - cmd := exec.Command("kubectl", "delete", "deploy", "-n", d.Metadata.Namespace, d.Metadata.Name) - out, err := util.RunAndLogCommand(cmd) - if err != nil { - log.Printf("Error while trying to delete deployment %s in namespace %s:%s\n", d.Metadata.Namespace, d.Metadata.Name, string(out)) - return err +func (d *Deployment) Delete(retries int) error { + var kubectlOutput []byte + var kubectlError error + for i := 0; i < retries; i++ { + cmd := exec.Command("kubectl", "delete", "deploy", "-n", d.Metadata.Namespace, d.Metadata.Name) + kubectlOutput, kubectlError = util.RunAndLogCommand(cmd) + if kubectlError != nil { + log.Printf("Error while trying to delete deployment %s in namespace %s:%s\n", d.Metadata.Namespace, d.Metadata.Name, string(kubectlOutput)) + continue + } + break + } + + if kubectlError != nil { + return kubectlError } - // Delete any associated HPAs + if d.Metadata.HasHPA { - cmd := exec.Command("kubectl", "delete", "hpa", "-n", d.Metadata.Namespace, d.Metadata.Name) - out, err := util.RunAndLogCommand(cmd) - if err != nil { - log.Printf("Deployment %s has associated HPA but unable to delete in namespace %s:%s\n", d.Metadata.Namespace, d.Metadata.Name, string(out)) - return err + for i := 0; i < retries; i++ { + cmd := exec.Command("kubectl", "delete", "hpa", "-n", d.Metadata.Namespace, d.Metadata.Name) + kubectlOutput, kubectlError = util.RunAndLogCommand(cmd) + if kubectlError != nil { + log.Printf("Deployment %s has associated HPA but unable to delete in namespace %s:%s\n", d.Metadata.Namespace, d.Metadata.Name, string(kubectlOutput)) + continue + } + break } } - return nil + + return kubectlError } // Expose will create a load balancer and expose the deployment on a given port diff --git a/test/e2e/kubernetes/job/job.go b/test/e2e/kubernetes/job/job.go index 0f177c0870..fcb4d4b05f 100644 --- a/test/e2e/kubernetes/job/job.go +++ b/test/e2e/kubernetes/job/job.go @@ -175,9 +175,5 @@ func (j *Job) Delete(retries int) error { break } - if kubectlError != nil { - return kubectlError - } - - return nil + return kubectlError } diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index e9bb46c2e0..20f4898f7f 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -30,14 +30,16 @@ import ( ) const ( - WorkloadDir = "workloads" - PolicyDir = "workloads/policies" - SSHPort = "50001" + WorkloadDir = "workloads" + PolicyDir = "workloads/policies" + deleteResourceRetries = 10 ) var ( - cfg config.Config - eng engine.Engine + cfg config.Config + eng engine.Engine + masterSSHPort string + masterSSHPrivateKeyFilepath string ) var _ = BeforeSuite(func() { @@ -59,6 +61,15 @@ var _ = BeforeSuite(func() { ClusterDefinition: csInput, ExpandedDefinition: csGenerated, } + masterNodes, err := node.GetByPrefix("k8s-master") + Expect(err).NotTo(HaveOccurred()) + masterName := masterNodes[0].Metadata.Name + if strings.Contains(masterName, "vmss") { + masterSSHPort = "50001" + } else { + masterSSHPort = "22" + } + masterSSHPrivateKeyFilepath = cfg.GetSSHKeyPath() }) var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() { @@ -67,18 +78,10 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu kubeConfig, err := GetConfig() Expect(err).NotTo(HaveOccurred()) master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName()) - sshKeyPath := cfg.GetSSHKeyPath() - masterNodes, err := node.GetByPrefix("k8s-master") - Expect(err).NotTo(HaveOccurred()) - masterName := masterNodes[0].Metadata.Name - lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r") + lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r") var cmd *exec.Cmd - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd) util.PrintCommand(cmd) out, err := cmd.CombinedOutput() log.Printf("%s\n", out) @@ -87,11 +90,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } kernelVerCmd := fmt.Sprintf("cat /proc/version") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -143,19 +142,10 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu kubeConfig, err := GetConfig() Expect(err).NotTo(HaveOccurred()) master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName()) - sshKeyPath := cfg.GetSSHKeyPath() - - masterNodes, err := node.GetByPrefix("k8s-master") - Expect(err).NotTo(HaveOccurred()) - masterName := masterNodes[0].Metadata.Name ifconfigCmd := fmt.Sprintf("ifconfig -a -v") var cmd *exec.Cmd - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd) util.PrintCommand(cmd) out, err := cmd.CombinedOutput() log.Printf("%s\n", out) @@ -164,11 +154,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } resolvCmd := fmt.Sprintf("cat /etc/resolv.conf") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -178,11 +164,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring that we have a valid connection to our resolver") digCmd := fmt.Sprintf("dig +short +search +answer `hostname`") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -196,11 +178,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring that we get a DNS lookup answer response for each node hostname") digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name) - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -212,22 +190,14 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring that we get a DNS lookup answer response for external names") digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() if err != nil { log.Printf("Error while querying DNS: %s\n", out) } digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -237,11 +207,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring that we get a DNS lookup answer response for external names using external resolver") digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -249,11 +215,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu log.Printf("Error while querying DNS: %s\n", err) } digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'") - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd) util.PrintCommand(cmd) out, err = cmd.CombinedOutput() log.Printf("%s\n", out) @@ -265,7 +227,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default") Expect(err).NotTo(HaveOccurred()) ready, err := j.WaitOnReady(5*time.Second, cfg.Timeout) - delErr := j.Delete(10) + delErr := j.Delete(deleteResourceRetries) if delErr != nil { fmt.Printf("could not delete job %s\n", j.Metadata.Name) fmt.Println(delErr) @@ -372,10 +334,6 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(err).NotTo(HaveOccurred()) master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName()) - sshKeyPath := cfg.GetSSHKeyPath() - masterNodes, err := node.GetByPrefix("k8s-master") - Expect(err).NotTo(HaveOccurred()) - masterName := masterNodes[0].Metadata.Name if dashboardPort == 80 { By("Ensuring that we can connect via HTTP to the dashboard on any one node") } else { @@ -390,15 +348,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu if address == nil { log.Printf("One of our nodes does not have an InternalIP value!: %s\n", node.Metadata.Name) } - Expect(address).NotTo(Equal(nil)) + Expect(address).NotTo(BeNil()) dashboardURL := fmt.Sprintf("http://%s:%v", address.Address, port) curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL) var cmd *exec.Cmd - if strings.Contains(masterName, "vmss") { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD) - } else { - cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD) - } + cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD) util.PrintCommand(cmd) out, err := cmd.CombinedOutput() if err == nil { @@ -687,11 +641,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } } By("Cleaning up after ourselves") - err = curlDeploy.Delete() + err = curlDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = deploy.Delete() + err = deploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = s.Delete() + err = s.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("No linux agent was provisioned for this Cluster Definition") @@ -754,11 +708,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(err).NotTo(HaveOccurred()) By("Cleaning up after ourselves") - err = loadTestDeploy.Delete() + err = loadTestDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = phpApacheDeploy.Delete() + err = phpApacheDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = s.Delete() + err = s.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("This flavor/version of Kubernetes doesn't support hpa autoscale") @@ -801,9 +755,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } By("Cleaning up after ourselves") - err = nginxDeploy.Delete() + err = nginxDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = s.Delete() + err = s.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("No linux agent was provisioned for this Cluster Definition") @@ -824,7 +778,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "cuda-vector-add.yaml"), "cuda-vector-add", "default") Expect(err).NotTo(HaveOccurred()) ready, err := j.WaitOnReady(30*time.Second, cfg.Timeout) - delErr := j.Delete(10) + delErr := j.Delete(deleteResourceRetries) if delErr != nil { fmt.Printf("could not delete job %s\n", j.Metadata.Name) fmt.Println(delErr) @@ -835,7 +789,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "nvidia-smi.yaml"), "nvidia-smi", "default") Expect(err).NotTo(HaveOccurred()) ready, err := j.WaitOnReady(30*time.Second, cfg.Timeout) - delErr := j.Delete(10) + delErr := j.Delete(deleteResourceRetries) if delErr != nil { fmt.Printf("could not delete job %s\n", j.Metadata.Name) fmt.Println(delErr) @@ -941,9 +895,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(nodeZone == pvZone).To(Equal(true)) By("Cleaning up after ourselves") - err = testPod.Delete() + err = testPod.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = pvc.Delete() + err = pvc.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("Availability zones was not configured for this Cluster Definition") @@ -961,7 +915,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(running).To(Equal(true)) restarts := pod.Status.ContainerStatuses[0].RestartCount if cfg.SoakClusterName == "" { - err = pod.Delete() + err = pod.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) Expect(restarts).To(Equal(0)) } else { @@ -1076,11 +1030,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Cleaning up after ourselves") networkpolicy.DeleteNetworkPolicy(networkPolicyName, namespace) - err = clientOneDeploy.Delete() + err = clientOneDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = clientTwoDeploy.Delete() + err = clientTwoDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = serverDeploy.Delete() + err = serverDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("Calico or Azure network policy was not provisioned for this Cluster Definition") @@ -1125,9 +1079,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } By("Verifying pods & services can be deleted") - err = iisDeploy.Delete() + err = iisDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = s.Delete() + err = s.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("No windows agent was provisioned for this Cluster Definition") @@ -1194,13 +1148,13 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(successes).To(Equal(cfg.StabilityIterations)) By("Cleaning up after ourselves") - err = windowsIISDeployment.Delete() + err = windowsIISDeployment.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = linuxNginxDeploy.Delete() + err = linuxNginxDeploy.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = windowsService.Delete() + err = windowsService.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) - err = linuxService.Delete() + err = linuxService.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("No windows agent was provisioned for this Cluster Definition") @@ -1242,14 +1196,13 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu kubeConfig, err := GetConfig() Expect(err).NotTo(HaveOccurred()) master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName()) - sshKeyPath := cfg.GetSSHKeyPath() for _, iisPod := range iisPods { - valid := iisPod.ValidateHostPort("(IIS Windows Server)", 10, 10*time.Second, master, sshKeyPath) + valid := iisPod.ValidateHostPort("(IIS Windows Server)", 10, 10*time.Second, master, masterSSHPrivateKeyFilepath) Expect(valid).To(BeTrue()) } - err = iisDeploy.Delete() + err = iisDeploy.Delete(kubectlOutput) Expect(err).NotTo(HaveOccurred()) } else { Skip("No windows agent was provisioned for this Cluster Definition") @@ -1291,7 +1244,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(valid).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) - err = iisPod.Delete() + err = iisPod.Delete(deleteResourceRetries) Expect(err).NotTo(HaveOccurred()) } else { Skip("Kubernetes version needs to be 1.8 and up for Azure File test") diff --git a/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go b/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go index a0d42149e9..df02ed7e06 100644 --- a/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go +++ b/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go @@ -85,15 +85,20 @@ func Describe(pvcName, namespace string) error { } // Delete will delete a PersistentVolumeClaims in a given namespace -func (pvc *PersistentVolumeClaims) Delete() error { - cmd := exec.Command("kubectl", "delete", "pvc", "-n", pvc.Metadata.NameSpace, pvc.Metadata.Name) - util.PrintCommand(cmd) - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("Error while trying to delete PVC %s in namespace %s:%s\n", pvc.Metadata.Name, pvc.Metadata.NameSpace, string(out)) - return err +func (pvc *PersistentVolumeClaims) Delete(retries int) error { + var kubectlOutput []byte + var kubectlError error + for i := 0; i < retries; i++ { + cmd := exec.Command("kubectl", "delete", "pvc", "-n", pvc.Metadata.NameSpace, pvc.Metadata.Name) + kubectlOutput, kubectlError = util.RunAndLogCommand(cmd) + if kubectlError != nil { + log.Printf("Error while trying to delete PVC %s in namespace %s:%s\n", pvc.Metadata.Name, pvc.Metadata.NameSpace, string(kubectlOutput)) + continue + } + break } - return nil + + return kubectlError } // WaitOnReady will block until PersistentVolumeClaims is available diff --git a/test/e2e/kubernetes/pod/pod.go b/test/e2e/kubernetes/pod/pod.go index b5a0c6968f..bf84dbbe22 100644 --- a/test/e2e/kubernetes/pod/pod.go +++ b/test/e2e/kubernetes/pod/pod.go @@ -219,7 +219,7 @@ func RunCommandMultipleTimes(podRunnerCmd podRunnerCmd, image, name, command str log.Printf("%s\n", string(out[:])) } - err = p.Delete() + err = p.Delete(3) if err != nil { return successfulAttempts, err } @@ -489,15 +489,20 @@ func (p *Pod) Exec(c ...string) ([]byte, error) { } // Delete will delete a Pod in a given namespace -func (p *Pod) Delete() error { - cmd := exec.Command("kubectl", "delete", "po", "-n", p.Metadata.Namespace, p.Metadata.Name) - util.PrintCommand(cmd) - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("Error while trying to delete Pod %s in namespace %s:%s\n", p.Metadata.Namespace, p.Metadata.Name, string(out)) - return err +func (p *Pod) Delete(retries int) error { + var kubectlOutput []byte + var kubectlError error + for i := 0; i < retries; i++ { + cmd := exec.Command("kubectl", "delete", "po", "-n", p.Metadata.Namespace, p.Metadata.Name) + kubectlOutput, kubectlError = util.RunAndLogCommand(cmd) + if kubectlError != nil { + log.Printf("Error while trying to delete Pod %s in namespace %s:%s\n", p.Metadata.Namespace, p.Metadata.Name, string(kubectlOutput)) + continue + } + break } - return nil + + return kubectlError } // CheckLinuxOutboundConnection will keep retrying the check if an error is received until the timeout occurs or it passes. This helps us when DNS may not be available for some time after a pod starts. diff --git a/test/e2e/kubernetes/service/service.go b/test/e2e/kubernetes/service/service.go index 88c8a105fd..2996c6436a 100644 --- a/test/e2e/kubernetes/service/service.go +++ b/test/e2e/kubernetes/service/service.go @@ -74,15 +74,20 @@ func Get(name, namespace string) (*Service, error) { } // Delete will delete a service in a given namespace -func (s *Service) Delete() error { - cmd := exec.Command("kubectl", "delete", "svc", "-n", s.Metadata.Namespace, s.Metadata.Name) - util.PrintCommand(cmd) - out, err := cmd.CombinedOutput() - if err != nil { - log.Printf("Error while trying to delete service %s in namespace %s:%s\n", s.Metadata.Namespace, s.Metadata.Name, string(out)) - return err +func (s *Service) Delete(retries int) error { + var kubectlOutput []byte + var kubectlError error + for i := 0; i < retries; i++ { + cmd := exec.Command("kubectl", "delete", "svc", "-n", s.Metadata.Namespace, s.Metadata.Name) + kubectlOutput, kubectlError = util.RunAndLogCommand(cmd) + if kubectlError != nil { + log.Printf("Error while trying to delete service %s in namespace %s:%s\n", s.Metadata.Namespace, s.Metadata.Name, string(kubectlOutput)) + continue + } + break } - return nil + + return kubectlError } // GetNodePort will return the node port for a given pod