Generate and approve CSRs for conntrol-plane and static workers nodes

Signed-off-by: Artiom Diomin <kron82@gmail.com>

.prow.yaml

@@ -125,7 +125,7 @@ presubmits:
       preset-goproxy: "true"
     spec:
       containers:
-        - image: golangci/golangci-lint:v1.43.0
+        - image: golangci/golangci-lint:v1.44.0
           command:
             - make
           args:

pkg/tasks/certs.go

@@ -19,6 +19,7 @@ package tasks
 import (
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"io/fs"
 	"time"
 
@@ -33,6 +34,27 @@ import (
 	"k8c.io/kubeone/pkg/ssh"
 	"k8c.io/kubeone/pkg/ssh/sshiofs"
 	"k8c.io/kubeone/pkg/state"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1"
+)
+
+const (
+	nodeUser = "system:node"
+
+	groupNodes         = "system:nodes"
+	groupAuthenticated = "system:authenticated"
+)
+
+var (
+	allowedUsages = []certificatesv1.KeyUsage{
+		certificatesv1.UsageDigitalSignature,
+		certificatesv1.UsageKeyEncipherment,
+		certificatesv1.UsageServerAuth,
+	}
 )
 
 func renewControlPlaneCerts(s *state.State) error {
@@ -158,3 +180,108 @@ func saveCABundleOnControlPlane(s *state.State, _ *kubeoneapi.HostConfig, conn s
 	_, _, err = s.Runner.RunRaw(cmd)
 	return err
 }
+
+func approvePendingCSR(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error {
+	s.Logger.Infof("Looking for CSRs for %q to approve...", node.Hostname)
+
+	// Need to wait for the second CSR to appear
+	time.Sleep(20 * time.Second)
+
+	csrList := certificatesv1.CertificateSigningRequestList{}
+	if err := s.DynamicClient.List(s.Context, &csrList); err != nil {
+		return err
+	}
+
+	certv1Client, err := certificatesv1client.NewForConfig(s.RESTConfig)
+	if err != nil {
+		return err
+	}
+	certClient := certv1Client.CertificateSigningRequests()
+
+	for _, csr := range csrList.Items {
+		if csr.Spec.SignerName != certificatesv1.KubeletServingSignerName {
+			continue
+		}
+
+		var approved bool
+		for _, cond := range csr.Status.Conditions {
+			if cond.Type == certificatesv1.CertificateApproved && cond.Status == corev1.ConditionTrue {
+				approved = true
+			}
+		}
+		if approved {
+			continue
+		}
+
+		if err := validateCSR(csr.Spec, node); err != nil {
+			return fmt.Errorf("failed to validate CSR: %w", err)
+		}
+
+		csr := csr.DeepCopy()
+		csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
+			Type:   certificatesv1.CertificateApproved,
+			Reason: "kubeone approved node serving cert",
+			Status: corev1.ConditionTrue,
+		})
+
+		s.Logger.Infof("Approve pending CSR %q for username %q", csr.Name, csr.Spec.Username)
+		if _, err := certClient.UpdateApproval(s.Context, csr.Name, csr, metav1.UpdateOptions{}); err != nil {
+			return fmt.Errorf("failed to approve CSR %q: %w", csr.Name, err)
+		}
+	}
+
+	return nil
+}
+
+func validateCSR(spec certificatesv1.CertificateSigningRequestSpec, node *kubeoneapi.HostConfig) error {
+	if fmt.Sprintf("%s:%s", nodeUser, node.Hostname) != spec.Username {
+		return errors.New("")
+	}
+
+	if !sets.NewString(spec.Groups...).HasAll(groupNodes, groupAuthenticated) {
+		return errors.New("")
+	}
+
+	for _, usage := range spec.Usages {
+		if !isUsageInUsageList(usage, allowedUsages) {
+			return errors.New("")
+		}
+	}
+
+	csrBlock, rest := pem.Decode(spec.Request)
+	if csrBlock == nil {
+		return fmt.Errorf("no certificate request found for the given CSR")
+	}
+
+	if len(rest) != 0 {
+		return fmt.Errorf("found more than one PEM encoded block in the result")
+	}
+
+	certReq, err := x509.ParseCertificateRequest(csrBlock.Bytes)
+	if err != nil {
+		return err
+	}
+
+	if certReq.Subject.CommonName != spec.Username {
+		return fmt.Errorf("commonName %q is different then CSR username %q", certReq.Subject.CommonName, spec.Username)
+	}
+
+	if len(certReq.Subject.Organization) != 1 {
+		return fmt.Errorf("expected only one organization but got %d instead", len(certReq.Subject.Organization))
+	}
+
+	if certReq.Subject.Organization[0] != groupNodes {
+		return fmt.Errorf("organization %q doesn't match node group %q", certReq.Subject.Organization[0], groupNodes)
+	}
+
+	return nil
+}
+
+func isUsageInUsageList(usage certificatesv1.KeyUsage, usageList []certificatesv1.KeyUsage) bool {
+	for _, usageListItem := range usageList {
+		if usage == usageListItem {
+			return true
+		}
+	}
+	return false
+}

pkg/tasks/controlplane.go

@@ -46,7 +46,11 @@ func joinControlPlaneNodeInternal(s *state.State, node *kubeoneapi.HostConfig, c
 	}
 
 	_, _, err = s.Runner.RunRaw(cmd)
-	return err
+	if err != nil {
+		return err
+	}
+
+	return approvePendingCSR(s, node, conn)
 }
 
 func kubeadmCertsExecutor(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error {
@@ -77,7 +81,6 @@ func initKubernetesLeader(s *state.State) error {
 		}
 
 		_, _, err = s.Runner.RunRaw(cmd)
-
 		return err
 	})
 }

pkg/tasks/probes.go

@@ -95,7 +95,7 @@ func safeguard(s *state.State) error {
 			}
 
 			return errors.Errorf(
-				"Container runtime on node %q is %q, but %q is configured. %s.",
+				"container runtime on node %q is %q, but %q is configured. %s",
 				node.Name,
 				nodesContainerRuntime,
 				configuredClusterContainerRuntime,

pkg/tasks/static_wokers.go

@@ -37,5 +37,9 @@ func joinStaticWorkerInternal(s *state.State, node *kubeoneapi.HostConfig, conn
 	}
 
 	_, _, err = s.Runner.RunRaw(cmd)
-	return err
+	if err != nil {
+		return err
+	}
+
+	return approvePendingCSR(s, node, conn)
 }

pkg/tasks/tasks.go

@@ -153,6 +153,12 @@ func WithFullInstall(t Tasks) Tasks {
 			},
 			{Fn: initKubernetesLeader, ErrMsg: "failed to init kubernetes on leader"},
 			{Fn: kubeconfig.BuildKubernetesClientset, ErrMsg: "failed to build kubernetes clientset"},
+			{
+				Fn: func(s *state.State) error {
+					return s.RunTaskOnLeader(approvePendingCSR)
+				},
+				ErrMsg: "failed to approve leader's kubelet CSR",
+			},
 			{Fn: repairClusterIfNeeded, ErrMsg: "failed to repair cluster"},
 			{Fn: joinControlplaneNode, ErrMsg: "failed to join other masters a cluster"},
 			{Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"},

pkg/templates/kubeadm/v1beta2/kubeadm.go

@@ -169,6 +169,7 @@ func NewConfig(s *state.State, host kubeoneapi.HostConfig) ([]runtime.Object, er
 		CgroupDriver:       "systemd",
 		ReadOnlyPort:       0,
 		RotateCertificates: true,
+		ServerTLSBootstrap: true,
 		ClusterDNS:         []string{resources.NodeLocalDNSVirtualIP},
 		Authentication: kubeletconfigv1beta1.KubeletAuthentication{
 			Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{
@@ -360,6 +361,7 @@ func NewConfigWorker(s *state.State, host kubeoneapi.HostConfig) ([]runtime.Obje
 		CgroupDriver:       "systemd",
 		ReadOnlyPort:       0,
 		RotateCertificates: true,
+		ServerTLSBootstrap: true,
 		ClusterDNS:         []string{resources.NodeLocalDNSVirtualIP},
 		Authentication: kubeletconfigv1beta1.KubeletAuthentication{
 			Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{

pkg/templates/kubeadm/v1beta3/kubeadm.go

@@ -169,6 +169,7 @@ func NewConfig(s *state.State, host kubeoneapi.HostConfig) ([]runtime.Object, er
 		CgroupDriver:       "systemd",
 		ReadOnlyPort:       0,
 		RotateCertificates: true,
+		ServerTLSBootstrap: true,
 		ClusterDNS:         []string{resources.NodeLocalDNSVirtualIP},
 		Authentication: kubeletconfigv1beta1.KubeletAuthentication{
 			Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{
@@ -360,6 +361,7 @@ func NewConfigWorker(s *state.State, host kubeoneapi.HostConfig) ([]runtime.Obje
 		CgroupDriver:       "systemd",
 		ReadOnlyPort:       0,
 		RotateCertificates: true,
+		ServerTLSBootstrap: true,
 		ClusterDNS:         []string{resources.NodeLocalDNSVirtualIP},
 		Authentication: kubeletconfigv1beta1.KubeletAuthentication{
 			Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{