Merge pull request #1426 from kinvolk/imran/reverting-baremetal-changes

Reverting Baremetal changes to avoid breaking Baremetal changes
kinvolk · Mar 11, 2021 · 106ea42 · 106ea42
2 parents ac85cbb + 9065238
commit 106ea42
Show file tree

Hide file tree

Showing 34 changed files with 829 additions and 608 deletions.
diff --git a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootkube.tf b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootkube.tf
@@ -2,10 +2,9 @@
 module "bootkube" {
   source = "../../../bootkube"
 
-  cluster_name = var.cluster_name
-  api_servers  = [format("%s.%s", var.cluster_name, var.k8s_domain_name)]
-  # Each instance of controller module generates the same set of etcd_servers.
-  etcd_servers                    = module.controller[0].etcd_servers
+  cluster_name                    = var.cluster_name
+  api_servers                     = [var.k8s_domain_name]
+  etcd_servers                    = var.controller_domains
   etcd_endpoints                  = []
   asset_dir                       = var.asset_dir
   network_mtu                     = var.network_mtu
@@ -22,7 +21,7 @@ module "bootkube" {
   # Disable the self hosted kubelet.
   disable_self_hosted_kubelet = var.disable_self_hosted_kubelet
 
-  bootstrap_tokens     = concat(module.controller.*.bootstrap_token, module.worker.*.bootstrap_token)
+  bootstrap_tokens     = [local.controller_bootstrap_token, local.worker_bootstrap_token]
   enable_tls_bootstrap = true
   encrypt_pod_traffic  = var.encrypt_pod_traffic
 

diff --git a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootstrap-configs.tf b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootstrap-configs.tf
@@ -0,0 +1,39 @@
+locals {
+  controller_bootstrap_token = {
+    token_id     = random_string.bootstrap_token_id_controller.result
+    token_secret = random_string.bootstrap_token_secret_controller.result
+  }
+
+  worker_bootstrap_token = {
+    token_id     = random_string.bootstrap_token_id_worker.result
+    token_secret = random_string.bootstrap_token_secret_worker.result
+  }
+}
+
+# Generate a cryptographically random token id (public).
+resource "random_string" "bootstrap_token_id_controller" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource "random_string" "bootstrap_token_secret_controller" {
+  length  = 16
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token id (public).
+resource "random_string" "bootstrap_token_id_worker" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource "random_string" "bootstrap_token_secret_worker" {
+  length  = 16
+  upper   = false
+  special = false
+}
diff --git a/...s/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/bootstrap-kubeconfig.yaml.tmpl b/...s/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/bootstrap-kubeconfig.yaml.tmpl
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet
diff --git a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
@@ -0,0 +1,239 @@
+---
+systemd:
+  units:
+    - name: etcd.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=etcd (System Application Container)
+        Documentation=https://github.com/etcd-io/etcd
+        Wants=docker.service
+        After=docker.service
+        [Service]
+        Type=simple
+        Restart=always
+        RestartSec=5s
+        TimeoutStartSec=0
+        LimitNOFILE=40000
+        EnvironmentFile=/etc/kubernetes/etcd.env
+        ExecStartPre=-docker rm -f etcd
+        ExecStartPre=sh -c "docker run -d \
+          --name=etcd \
+          --restart=unless-stopped \
+          --log-driver=journald \
+          --network=host \
+          -u $(id -u \"$${USER}\"):$(id -u \"$${USER}\") \
+          -v $${ETCD_DATA_DIR}:$${ETCD_DATA_DIR}:rw \
+          -v $${SSL_DIR}:$${SSL_DIR}:ro \
+          --env-file /etc/kubernetes/etcd.env \
+          $${IMAGE_URL}:$${IMAGE_TAG}"
+        ExecStart=docker logs -f etcd
+        ExecStop=docker stop etcd
+        ExecStopPost=docker rm etcd
+        [Install]
+        WantedBy=multi-user.target
+    - name: docker.service
+      enable: true
+    - name: locksmithd.service
+      mask: true
+    - name: kubelet.path
+      enable: true
+      contents: |
+        [Unit]
+        Description=Watch for kubeconfig
+        [Path]
+        PathExists=/etc/kubernetes/kubeconfig
+        [Install]
+        WantedBy=multi-user.target
+    - name: wait-for-dns.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Restart=on-failure
+        RestartSec=5s
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+        RequiredBy=etcd-member.service
+    - name: kubelet.service
+      contents: |
+        [Unit]
+        Description=Kubelet
+        Wants=rpc-statd.service
+        [Service]
+        EnvironmentFile=/etc/kubernetes/kubelet.env
+        ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
+        ExecStartPre=-docker rm -f kubelet
+        ExecStartPre=docker run -d \
+          --name=kubelet \
+          --restart=unless-stopped \
+          --log-driver=journald \
+          --network=host \
+          --pid=host \
+          --privileged \
+          -v /dev:/dev:rw \
+          -v /etc/cni/net.d:/etc/cni/net.d:ro \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run:rw \
+          -v /sys:/sys:rw \
+          -v /opt/cni/bin:/opt/cni/bin:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/cni:/var/lib/cni:rw \
+          -v /var/lib/docker:/var/lib/docker:rw \
+          -v /var/log/pods:/var/log/pods:rw \
+          --mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
+          --mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
+          $${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
+          --anonymous-auth=false \
+          --authentication-token-webhook \
+          --authorization-mode=Webhook \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --cluster_dns=${cluster_dns_service_ip} \
+          --cluster_domain=${cluster_domain_suffix} \
+          --cni-conf-dir=/etc/cni/net.d \
+          --config=/etc/kubernetes/kubelet.config \
+          --exit-on-lock-contention \
+          --hostname-override=${domain_name} \
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          --lock-file=/var/run/lock/kubelet.lock \
+          --network-plugin=cni \
+          --node-labels=$${NODE_LABELS} \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --read-only-port=0 \
+          --register-with-taints=$${NODE_TAINTS} \
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+    - name: bootkube.service
+      contents: |
+        [Unit]
+        Description=Bootstrap a Kubernetes control plane with a temp api-server
+        ConditionPathExists=!/opt/bootkube/init_bootkube.done
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        WorkingDirectory=/opt/bootkube
+        ExecStart=/opt/bootkube/bootkube-start
+        ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
+storage:
+  files:
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=quay.io/kinvolk/kubelet
+          KUBELET_IMAGE_TAG=v1.19.4
+          NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
+          NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+    - path: /etc/kubernetes/etcd.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          IMAGE_TAG=v3.4.14
+          IMAGE_URL=quay.io/coreos/etcd
+          SSL_DIR=/etc/ssl/etcd
+          USER=etcd
+          ETCD_DATA_DIR=/var/lib/etcd
+          ETCD_NAME=${etcd_name}
+          ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379
+          ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380
+          ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
+          ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
+          ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
+          ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
+          ETCD_STRICT_RECONFIG_CHECK=true
+          ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/server-ca.crt
+          ETCD_CERT_FILE=/etc/ssl/etcd/etcd/server.crt
+          ETCD_KEY_FILE=/etc/ssl/etcd/etcd/server.key
+          ETCD_CLIENT_CERT_AUTH=true
+          ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/peer-ca.crt
+          ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd/peer.crt
+          ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd/peer.key
+          ETCD_PEER_CLIENT_CERT_AUTH=true
+    - path: /etc/hostname
+      filesystem: root
+      mode: 0644
+      contents:
+        inline:
+          ${domain_name}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+    - path: /opt/bootkube/bootkube-start
+      filesystem: root
+      mode: 0544
+      user:
+        id: 500
+      group:
+        id: 500
+      contents:
+        inline: |
+          #!/bin/bash
+          # Wrapper for bootkube start
+          set -e
+          # Move experimental manifests
+          [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
+          exec docker run \
+            -v /opt/bootkube/assets:/assets:ro \
+            -v /etc/kubernetes:/etc/kubernetes:rw \
+            --network=host \
+            quay.io/kinvolk/bootkube:v0.14.0-helm4 \
+            /bootkube start --asset-dir=/assets
+    - path: /etc/kubernetes/configure-kubelet-cgroup-driver
+      filesystem: root
+      mode: 0744
+      contents:
+        inline: |
+          #!/bin/bash
+          set -e
+          readonly docker_cgroup_driver="$(docker info -f '{{.CgroupDriver}}')"
+          cat <<EOF >/etc/kubernetes/kubelet.config
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          cgroupDriver: "$${docker_cgroup_driver}"
+          EOF
+    - path: /etc/tmpfiles.d/etcd-wrapper.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          d    /var/lib/etcd 0700 etcd etcd - -
+    - path: /etc/docker/daemon.json
+      filesystem: root
+      mode: 0500
+      contents:
+        inline: |
+          {
+            "live-restore": true,
+            "log-opts": {
+              "max-size": "100m",
+              "max-file": "3"
+            }
+          }
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys: ${ssh_keys}
diff --git a/...chbox-flatcar/templates/install.yaml.tmpl → ...car-linux/kubernetes/cl/install.yaml.tmpl b/...chbox-flatcar/templates/install.yaml.tmpl → ...car-linux/kubernetes/cl/install.yaml.tmpl
@@ -12,9 +12,9 @@ systemd:
         ExecStart=/opt/installer
         [Install]
         WantedBy=multi-user.target
-    # Avoid using the standard SSH port so Terraform apply cannot SSH until
+    # Avoid using the standard SSH port so terraform apply cannot SSH until
     # post-install. But admins may SSH to debug disk install problems.
-    # After install, sshd will use port 22 and users/Terraform can connect.
+    # After install, sshd will use port 22 and users/terraform can connect.
     - name: sshd.socket
       dropins:
         - name: 10-sshd-port.conf