WIP: implement automatic certificate rotation while updating cluster

This commit is a WIP version of automated certificate rotation
implementation, which right now occurs on every cluster update, but as
we discussed, this functionality needs to be moved into separate
subcommand.

Some highlights:
- Added config checksum to kube-apiserver and kube-controller-manager
  to trigger automatic recreation of pods when secrets are updated.

- Added ca-syncer init container to kubelet which ensures that
  bootstrap kubeconfig and one generated by TLS bootstrapping used by
  kubelet and client-ca certificate are always up to date, as kubelet
  seems to be not capable of updating them themselve.

- Temporarily force kube-controller-manager to sign certificates with
  only 45 minutes duration to make sure certificate rotation works as
  expected and all used certificates are rotated.

- When any of certificates used by etcd is changed, trigger
  copy-controller-secrets null_resource to copy etcd certificates on
  all controller nodes and then restart etcd to ensure it picks up
  new certificates, as it can only automatically pick up new client
  certificates and private key, but not CA certificate.

- Temporarily switch certificate validity hours on AWS to 1 hour to
  enforce running 'lokoctl cluster apply' every hour to keep cluster
  alive to make sure certificate rotation works as expected.

- 'lokoctl cluster apply' now taints all certificate resources on
  cluster update and after cluster is updated, it waits for all service
  account token secrets to be updated with the new CA certificate, then
  triggers a restart on all system deployments to make sure they pick up
  new CA certificate. This is core of certificate rotation logic.

TODO:
- Split into multiple commits.
- Add support for platforms other than AWS.
- Add support for single controller node clusters.
- Move rotation code to separate subcommand.
- Refactor code to avoid large functions.
- Refactor logic for waiting for Deployment and DaemonSet to converge,
  so it can also be re-used in testing framework.

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

assets/charts/control-plane/kube-apiserver/templates/kube-apiserver-ds.yaml

@@ -24,6 +24,8 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-apiserver-secret.yaml") . | sha256sum }}
     spec:
 {{- template "containers" . }}
 {{- end }}

assets/charts/control-plane/kube-apiserver/templates/kube-apiserver.yaml

@@ -25,6 +25,8 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-apiserver-secret.yaml") . | sha256sum }}
     spec:
 {{- template "containers" . }}
 {{- end }}

assets/charts/control-plane/kubelet/templates/kubelet-ds.yaml

@@ -17,7 +17,23 @@ spec:
         tier: node
         k8s-app: kubelet
     spec:
-      automountServiceAccountToken: false
+      initContainers:
+      - name: ca-syncer
+        image: {{ .Values.image }}
+        command:
+        - bash
+        - -c
+        - |
+          sed -i "s/^    certificate-authority-data:.*$/    certificate-authority-data: {{ .Values.kubernetesCACert }}/g" /var/lib/kubelet/kubeconfig /etc/kubernetes/kubeconfig
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: var-lib-kubelet
+          mountPath: /var/lib/kubelet
+          readOnly: false
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+          readOnly: false
       containers:
       - name: kubelet
         image: {{ .Values.image }}

assets/charts/control-plane/kubelet/values.yaml

@@ -2,3 +2,4 @@ image: quay.io/poseidon/kubelet:v1.19.4
 clusterDNS: 10.0.0.10
 clusterDomain: cluster.local
 enableTLSBootstrap: true
+kubernetesCACert: ""

assets/charts/control-plane/kubernetes/templates/_helpers.tpl

@@ -55,6 +55,7 @@
         - --pod-eviction-timeout=1m
         - --root-ca-file=/etc/kubernetes/secrets/ca.crt
         - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
+        - --cluster-signing-duration=45m
         livenessProbe:
           httpGet:
             scheme: HTTPS

assets/charts/control-plane/kubernetes/templates/kube-controller-manager-ds.yaml

@@ -23,6 +23,8 @@ spec:
         k8s-app: kube-controller-manager
       annotations:
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-controller-manager-secret.yaml") . | sha256sum }}
     spec:
 {{- template "controller-manager-containers" . }}
 {{- end }}

assets/charts/control-plane/kubernetes/templates/kube-controller-manager.yaml

@@ -24,6 +24,8 @@ spec:
         k8s-app: kube-controller-manager
       annotations:
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
+        # Automatically rolls update when secret changes.
+        checksum/secret: {{ include (print $.Template.BasePath "/kube-apiserver-secret.yaml") . | sha256sum }}
     spec:
 {{- template "controller-manager-containers" . }}
 {{- end }}

assets/terraform-modules/aws/flatcar-linux/kubernetes/ssh.tf

@@ -70,11 +70,15 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo chown -R etcd:etcd /etc/ssl/etcd",
       "sudo chmod -R 500 /etc/ssl/etcd",
+      "sudo systemctl restart etcd",
     ]
   }
 
   triggers = {
-    controller_id = aws_instance.controllers[count.index].id
+    controller_id    = aws_instance.controllers[count.index].id
+    etcd_ca_cert     = module.bootkube.etcd_ca_cert
+    etcd_server_cert = module.bootkube.etcd_server_cert
+    etcd_peer_cert   = module.bootkube.etcd_peer_cert
   }
 }
 

assets/terraform-modules/bootkube/assets.tf

@@ -108,6 +108,7 @@ locals {
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
     enable_tls_bootstrap   = var.enable_tls_bootstrap
+    kubernetes_ca_cert     = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
   })
 
   kubeconfig_kubelet_content = templatefile("${path.module}/resources/kubeconfig-kubelet", {

assets/terraform-modules/bootkube/resources/charts/kubelet.yaml

@@ -2,3 +2,4 @@ image: ${kubelet_image}
 clusterDNS: ${cluster_dns_service_ip}
 clusterDomain: ${cluster_domain_suffix}
 enableTLSBootstrap: ${enable_tls_bootstrap}
+kubernetesCACert: ${kubernetes_ca_cert}