Fix DestroyWorkers not retrying operations

kubermatic · Aug 28, 2019 · 01eb04c · 01eb04c
1 parent 295e263
commit 01eb04c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 11 deletions.
diff --git a/pkg/installer/installation/reset.go b/pkg/installer/installation/reset.go
@@ -55,24 +55,49 @@ func Reset(s *state.State) error {
 }
 
 func destroyWorkers(s *state.State) error {
+	var lastErr error
 	s.Logger.Infoln("Destroying worker nodes…")
 
-	waitErr := wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
-		err := kubeconfig.BuildKubernetesClientset(s)
-		return err == nil, errors.Wrap(err, "unable to build kubernetes clientset")
+	_ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
+		lastErr = kubeconfig.BuildKubernetesClientset(s)
+		if lastErr != nil {
+			s.Logger.Warn("Unable to connect to the control plane API. Retrying…")
+			return false, nil
+		}
+		return true, nil
+
 	})
-	if waitErr != nil {
+	if lastErr != nil {
 		s.Logger.Warn("Unable to connect to the control plane API and destroy worker nodes")
 		s.Logger.Warn("You can skip destroying worker nodes and destroy them manually using `--destroy-workers=false`")
-		return waitErr
+		return errors.Wrap(lastErr, "unable to build kubernetes clientset")
 	}
 
-	waitErr = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
-		err := machinecontroller.DestroyWorkers(s)
-		return err == nil, errors.Wrap(err, "unable to delete all worker nodes")
+	_ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
+		lastErr = machinecontroller.DestroyWorkers(s)
+		if lastErr != nil {
+			s.Logger.Warn("Unable to destroy worker nodes. Retrying…")
+			return false, nil
+		}
+		return true, nil
 	})
+	if lastErr != nil {
+		return errors.Wrap(lastErr, "unable to delete all worker nodes")
+	}
 
-	return waitErr
+	_ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
+		lastErr = machinecontroller.WaitDestroy(s)
+		if lastErr != nil {
+			s.Logger.Warn("Waiting for all machines to be deleted…")
+			return false, nil
+		}
+		return true, nil
+	})
+	if lastErr != nil {
+		return errors.Wrap(lastErr, "error waiting for machines to be deleted")
+	}
+
+	return nil
 }
 
 func resetNode(s *state.State, _ *kubeoneapi.HostConfig, conn ssh.Connection) error {

diff --git a/pkg/templates/machinecontroller/helper.go b/pkg/templates/machinecontroller/helper.go
@@ -160,8 +160,14 @@ func DestroyWorkers(s *state.State) error {
 		}
 	}
 
-	// Wait for all Machines to be deleted
+	return nil
+}
+
+// WaitDestroy waits for all Machines to be deleted
+func WaitDestroy(s *state.State) error {
 	s.Logger.Info("Waiting for all machines to get deleted…")
+
+	bgCtx := context.Background()
 	return wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) {
 		list := &clusterv1alpha1.MachineList{}
 		if err := s.DynamicClient.List(bgCtx, dynclient.InNamespace(MachineControllerNamespace), list); err != nil {
@@ -172,4 +178,4 @@ func DestroyWorkers(s *state.State) error {
 		}
 		return true, nil
 	})
-}
+}