Don't remove controlnode from cluster too early (#887)

Signed-off-by: Alexey Makhov <amakhov@mirantis.com>

internal/controller/controlplane/helper.go

@@ -344,6 +344,44 @@ func (c *K0sController) markChildControlNodeToLeave(ctx context.Context, name st
 	return nil
 }
 
+func (c *K0sController) deleteOldControlNodes(ctx context.Context, cluster *clusterv1.Cluster) error {
+	kubeClient, err := c.getKubeClient(ctx, cluster)
+	if err != nil {
+		return fmt.Errorf("error getting kube client: %w", err)
+	}
+	machines, err := collections.GetFilteredMachinesForCluster(ctx, c, cluster, collections.ControlPlaneMachines(cluster.Name))
+	if err != nil {
+		return fmt.Errorf("error getting all machines: %w", err)
+	}
+
+	var controlNodeList unstructured.UnstructuredList
+	err = kubeClient.RESTClient().
+		Get().
+		AbsPath("/apis/autopilot.k0sproject.io/v1beta2/controlnodes").
+		Do(ctx).
+		Into(&controlNodeList)
+
+	if err != nil && !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	existingMachineNames := make(map[string]struct{})
+	for _, n := range machines.Names() {
+		existingMachineNames[n] = struct{}{}
+	}
+
+	for _, controlNode := range controlNodeList.Items {
+		if _, ok := existingMachineNames[controlNode.GetName()]; !ok {
+			err := c.deleteControlNode(ctx, controlNode.GetName(), kubeClient)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (c *K0sController) deleteControlNode(ctx context.Context, name string, clientset *kubernetes.Clientset) error {
 	if clientset == nil {
 		return nil

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -373,6 +373,13 @@ func (c *K0sController) reconcileMachines(ctx context.Context, cluster *clusterv
 	}
 	log.Log.Info("Collected machines", "count", machines.Len(), "desired", kcp.Spec.Replicas, "updating", clusterIsUpdating, "deleting", len(machineNamesToDelete), "desiredMachines", desiredMachineNames)
 
+	go func() {
+		err = c.deleteOldControlNodes(ctx, cluster)
+		if err != nil {
+			logger.Error(err, "Error deleting old control nodes")
+		}
+	}()
+
 	if clusterIsUpdating {
 		log.Log.Info("Cluster is updating", "currentVersion", currentVersion, "newVersion", kcp.Spec.Version, "strategy", kcp.Spec.UpdateStrategy)
 		if kcp.Spec.UpdateStrategy == cpv1beta1.UpdateRecreate {
@@ -499,10 +506,6 @@ func (c *K0sController) runMachineDeletionSequence(ctx context.Context, logger l
 		return fmt.Errorf("error checking machine left: %w", err)
 	}
 
-	if err := c.deleteControlNode(ctx, machine.Name, kubeClient); err != nil {
-		return fmt.Errorf("error deleting controlnode: %w", err)
-	}
-
 	if err := c.deleteBootstrapConfig(ctx, machine.Name, kcp); err != nil {
 		return fmt.Errorf("error deleting machine from template: %w", err)
 	}