[Breaking Change] TLS Bootstrapping

assets/charts/control-plane/bootstrap-secrets/Chart.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+name: bootstrap-secrets
+description: A Helm chart for installing bootstrap secrets
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0

assets/charts/control-plane/bootstrap-secrets/templates/bootstrap-secret.yaml

@@ -0,0 +1,15 @@
+{{- range $token := .Values.bootstrapTokens }}
+---
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>".
+  name: bootstrap-token-{{ $token.tokenID }}
+  namespace: kube-system
+stringData:
+  description: "Lokomotive generated bootstrap token"
+  token-id: {{ $token.tokenID }}
+  token-secret: {{ $token.tokenSecret }}
+  usage-bootstrap-authentication: "true"
+{{- end }}

assets/charts/control-plane/bootstrap-secrets/values.yaml

@@ -0,0 +1,4 @@
+bootstrapTokens: []
+# bootstrapTokens:
+# - tokenID: ""
+#   tokenSecret: ""

assets/charts/control-plane/kube-apiserver/templates/kube-apiserver.yaml

@@ -83,11 +83,20 @@ spec:
           --advertise-address=$(POD_IP) \
           --allow-privileged=true \
           --anonymous-auth=false \
+          {{- if .Values.apiserver.enableTLSBootstrap }}
+          --authorization-mode=Node,RBAC \
+          {{- else }}
           --authorization-mode=RBAC \
+          {{- end }}
           --bind-address=$(cat /run/kube-apiserver/address) \
           --client-ca-file=/etc/kubernetes/secrets/ca.crt \
           --cloud-provider={{ .Values.apiserver.cloudProvider }} \
+          {{- if .Values.apiserver.enableTLSBootstrap }}
+          --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy,NodeRestriction \
+          --enable-bootstrap-token-auth=true \
+          {{- else }}
           --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy \
+          {{- end }}
           --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt \
           --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt \
           --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key \
@@ -119,11 +128,20 @@ spec:
         - --advertise-address=$(POD_IP)
         - --allow-privileged=true
         - --anonymous-auth=false
+        {{- if .Values.apiserver.enableTLSBootstrap }}
+        - --authorization-mode=Node,RBAC
+        {{- else }}
         - --authorization-mode=RBAC
+        {{- end }}
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider={{ .Values.apiserver.cloudProvider }}
+        {{- if .Values.apiserver.enableTLSBootstrap }}
+        - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy,NodeRestriction
+        - --enable-bootstrap-token-auth=true
+        {{- else }}
         - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority,PodSecurityPolicy
+        {{- end }}
         - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
         - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
         - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key

assets/charts/control-plane/kube-apiserver/values.yaml

@@ -21,3 +21,4 @@ apiserver:
   # If false, it will be exposed only on HostIP on port 6443.
   exposeOnAllInterfaces: false
   extraFlags: []
+  enableTLSBootstrap:

assets/charts/control-plane/kubelet/templates/kubelet-ds.yaml

@@ -39,7 +39,13 @@ spec:
           --cluster_domain={{ .Values.clusterDomain }} \
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
+          {{- if .Values.enableTLSBootstrap }}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          {{- else }}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          {{- end }}
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --pod-manifest-path=/etc/kubernetes/manifests \

assets/charts/control-plane/kubelet/values.yaml

@@ -1,3 +1,4 @@
 image: quay.io/poseidon/kubelet:v1.18.8
 clusterDNS: 10.0.0.10
 clusterDomain: cluster.local
+enableTLSBootstrap: true

assets/charts/control-plane/kubernetes/templates/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,41 @@
+# Enable bootstrapping nodes to create CSR.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: create-csrs-for-bootstrapping
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:node-bootstrapper
+  apiGroup: rbac.authorization.k8s.io
+---
+# Approve all CSRs for the group "system:bootstrappers".
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-csrs-for-group
+subjects:
+- kind: Group
+  name: system:bootstrappers
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+  apiGroup: rbac.authorization.k8s.io
+---
+# Approve renewal CSRs for the group "system:nodes".
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auto-approve-renewals-for-nodes
+subjects:
+- kind: Group
+  name: system:nodes
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+  apiGroup: rbac.authorization.k8s.io

assets/charts/control-plane/kubernetes/templates/kubelet-nodes-cluster-role-binding.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.kubelet.enableTLSBootstrap }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -10,3 +11,4 @@ subjects:
 - kind: Group
   name: system:nodes
   apiGroup: rbac.authorization.k8s.io
+{{- end }}

assets/charts/control-plane/kubernetes/values.yaml

@@ -24,3 +24,5 @@ coredns:
   clusterIP: 10.0.0.10
 etcd:
   endpoints: []
+kubelet:
+  enableTLSBootstrap: true

assets/terraform-modules/aws/flatcar-linux/kubernetes/bootkube.tf

@@ -1,9 +1,13 @@
+locals {
+  api_server = format("%s.%s", var.cluster_name, var.dns_zone)
+}
+
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
   source = "../../../bootkube"
 
   cluster_name                = var.cluster_name
-  api_servers                 = [format("%s.%s", var.cluster_name, var.dns_zone)]
+  api_servers                 = [local.api_server]
   etcd_servers                = aws_route53_record.etcds.*.fqdn
   etcd_endpoints              = aws_instance.controllers.*.private_ip
   asset_dir                   = var.asset_dir
@@ -23,4 +27,7 @@ module "bootkube" {
   #
   # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
   blocked_metadata_cidrs = ["169.254.169.254/32"]
+
+  bootstrap_tokens     = var.enable_tls_bootstrap ? concat([local.controller_bootstrap_token], var.worker_bootstrap_tokens) : []
+  enable_tls_bootstrap = var.enable_tls_bootstrap
 }

assets/terraform-modules/aws/flatcar-linux/kubernetes/bootstrap-configs.tf

@@ -0,0 +1,24 @@
+locals {
+  controller_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -90,7 +90,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \

assets/terraform-modules/aws/flatcar-linux/kubernetes/controllers.tf

@@ -133,6 +133,7 @@ data "template_file" "controller-configs" {
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
+    enable_tls_bootstrap   = var.enable_tls_bootstrap
   }
 }
 

assets/terraform-modules/aws/flatcar-linux/kubernetes/outputs.tf

@@ -35,6 +35,14 @@ output "kubeconfig" {
   value = module.bootkube.kubeconfig-kubelet
 }
 
+output "ca_cert" {
+  value = module.bootkube.ca_cert
+}
+
+output "apiserver" {
+  value = local.api_server
+}
+
 # Outputs for custom load balancing
 
 output "nlb_arn" {
@@ -66,3 +74,7 @@ output "calico_values" {
 output "lokomotive_values" {
   value = module.bootkube.lokomotive_values
 }
+
+output "bootstrap-secrets_values" {
+  value = module.bootkube.bootstrap-secrets_values
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/ssh.tf

@@ -45,7 +45,13 @@ resource "null_resource" "copy-controller-secrets" {
   }
 
   provisioner "file" {
-    content     = module.bootkube.kubeconfig-kubelet
+    content = var.enable_tls_bootstrap ? templatefile("${path.module}/workers/cl/bootstrap-kubeconfig.yaml.tmpl", {
+      token_id     = random_string.bootstrap_token_id[0].result
+      token_secret = random_string.bootstrap_token_secret[0].result
+      ca_cert      = module.bootkube.ca_cert
+      server       = "https://${local.api_server}:6443"
+    }) : module.bootkube.kubeconfig-kubelet
+
     destination = "$HOME/kubeconfig"
   }
 

assets/terraform-modules/aws/flatcar-linux/kubernetes/variables.tf

@@ -162,6 +162,16 @@ variable "disable_self_hosted_kubelet" {
   type        = bool
 }
 
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}
+
+variable "worker_bootstrap_tokens" {
+  description = "List of token-id and token-secret of each node."
+  type        = list(any)
+}
+
 # Certificates
 
 variable "certs_validity_period_hours" {

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/bootstrap-configs.tf

@@ -0,0 +1,24 @@
+locals {
+  worker_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/bootstrap-kubeconfig.yaml.tmpl

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -64,7 +64,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \
@@ -141,7 +147,11 @@ storage:
             --net=host \
             --dns=host \
             -- \
+            %{~ if enable_tls_bootstrap ~}
+            kubectl --kubeconfig=/var/lib/kubelet/kubeconfig delete node $(hostname)
+            %{~ else ~}
             kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            %{ endif }
 passwd:
   users:
     - name: core

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/outputs.tf

@@ -7,3 +7,7 @@ output "target_group_https" {
   description = "ARN of a target group of workers for HTTPS traffic"
   value       = aws_lb_target_group.workers_https.arn
 }
+
+output "worker_bootstrap_token" {
+  value = local.worker_bootstrap_token
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf

@@ -121,6 +121,16 @@ variable "kubeconfig" {
   description = "Must be set to `kubeconfig` output by cluster"
 }
 
+variable "ca_cert" {
+  description = "Kubernetes CA certificate needed in the kubeconfig file."
+  type        = string
+}
+
+variable "apiserver" {
+  description = "Apiserver private endpoint needed in the kubeconfig file."
+  type        = string
+}
+
 variable "ssh_keys" {
   type        = list(string)
   description = "SSH public keys for user 'core'"
@@ -158,3 +168,8 @@ variable "lb_https_port" {
   type        = number
   default     = 443
 }
+
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}