diff --git a/pkg/installer/installation/reset.go b/pkg/installer/installation/reset.go index e9bd60bf5..6965f521a 100644 --- a/pkg/installer/installation/reset.go +++ b/pkg/installer/installation/reset.go @@ -55,24 +55,49 @@ func Reset(s *state.State) error { } func destroyWorkers(s *state.State) error { + var lastErr error s.Logger.Infoln("Destroying worker nodes…") - waitErr := wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { - err := kubeconfig.BuildKubernetesClientset(s) - return err == nil, errors.Wrap(err, "unable to build kubernetes clientset") + _ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { + lastErr = kubeconfig.BuildKubernetesClientset(s) + if lastErr != nil { + s.Logger.Warn("Unable to connect to the control plane API. Retrying…") + return false, nil + } + return true, nil + }) - if waitErr != nil { + if lastErr != nil { s.Logger.Warn("Unable to connect to the control plane API and destroy worker nodes") s.Logger.Warn("You can skip destroying worker nodes and destroy them manually using `--destroy-workers=false`") - return waitErr + return errors.Wrap(lastErr, "unable to build kubernetes clientset") } - waitErr = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { - err := machinecontroller.DestroyWorkers(s) - return err == nil, errors.Wrap(err, "unable to delete all worker nodes") + _ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { + lastErr = machinecontroller.DestroyWorkers(s) + if lastErr != nil { + s.Logger.Warn("Unable to destroy worker nodes. Retrying…") + return false, nil + } + return true, nil }) + if lastErr != nil { + return errors.Wrap(lastErr, "unable to delete all worker nodes") + } - return waitErr + _ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) { + lastErr = machinecontroller.WaitDestroy(s) + if lastErr != nil { + s.Logger.Warn("Waiting for all machines to be deleted…") + return false, nil + } + return true, nil + }) + if lastErr != nil { + return errors.Wrap(lastErr, "error waiting for machines to be deleted") + } + + return nil } func resetNode(s *state.State, _ *kubeoneapi.HostConfig, conn ssh.Connection) error { diff --git a/pkg/templates/machinecontroller/helper.go b/pkg/templates/machinecontroller/helper.go index f7de7b1be..8a6ce09e8 100644 --- a/pkg/templates/machinecontroller/helper.go +++ b/pkg/templates/machinecontroller/helper.go @@ -160,8 +160,14 @@ func DestroyWorkers(s *state.State) error { } } - // Wait for all Machines to be deleted + return nil +} + +// WaitDestroy waits for all Machines to be deleted +func WaitDestroy(s *state.State) error { s.Logger.Info("Waiting for all machines to get deleted…") + + bgCtx := context.Background() return wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) { list := &clusterv1alpha1.MachineList{} if err := s.DynamicClient.List(bgCtx, dynclient.InNamespace(MachineControllerNamespace), list); err != nil {