From 379160727efec126704011587b522ba54b09115f Mon Sep 17 00:00:00 2001
From: Suraj Deshmukh <suraj@kinvolk.io>
Date: Fri, 11 Sep 2020 13:33:19 +0530
Subject: [PATCH] baremetal: Remove rkt

This commit removes rkt and uses docker to start services.

etcd(for controllers only):

  - Move the env var to a separate env var file.
  - Add etcd service file.
  - Change the name of service from etcd-member to etcd.

Bootkube(for controllers only):

  Use `docker run` instead of rkt.

Kubelet(for controllers and workers):

  Remove some of the folder creation in `ExecStartPre`, these are
  automatically created by docker, when mounted using `-v` flag.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>
---
 .../kubernetes/cl/controller.yaml.tmpl        | 162 ++++++++++--------
 .../kubernetes/cl/worker.yaml.tmpl            |  58 ++++---
 2 files changed, 123 insertions(+), 97 deletions(-)

diff --git a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
index e56a68bb1..c9948e03d 100644
--- a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
+++ b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
@@ -1,32 +1,37 @@
 ---
 systemd:
   units:
-    - name: etcd-member.service
+    - name: etcd.service
       enable: true
-      dropins:
-        - name: 40-etcd-cluster.conf
-          contents: |
-            [Service]
-            Environment="ETCD_IMAGE_TAG=v3.4.13"
-            Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
-            Environment="RKT_RUN_ARGS=--insecure-options=image"
-            Environment="ETCD_NAME=${etcd_name}"
-            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379"
-            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380"
-            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
-            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
-            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
-            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
-            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
-            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
-            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
-            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
-            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
-            Environment="ETCD_CLIENT_CERT_AUTH=true"
-            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
-            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
-            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
-            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
+      contents: |
+        [Unit]
+        Description=etcd (System Application Container)
+        Documentation=https://github.com/etcd-io/etcd
+        Wants=docker.service
+        After=docker.service
+        [Service]
+        Type=simple
+        Restart=always
+        RestartSec=5s
+        TimeoutStartSec=0
+        LimitNOFILE=40000
+        EnvironmentFile=/etc/kubernetes/etcd.env
+        ExecStartPre=-docker rm -f etcd
+        ExecStartPre=sh -c "docker run -d \
+          --name=etcd \
+          --restart=unless-stopped \
+          --log-driver=journald \
+          --network=host \
+          -u $(id -u \"$${ETCD_USER}\"):$(id -u \"$${ETCD_USER}\") \
+          -v $${ETCD_DATA_DIR}:$${ETCD_DATA_DIR}:rw \
+          -v $${ETCD_SSL_DIR}:$${ETCD_SSL_DIR}:ro \
+          --env-file /etc/kubernetes/etcd.env \
+          $${ETCD_IMAGE_URL}:$${ETCD_IMAGE_TAG}"
+        ExecStart=docker logs -f etcd
+        ExecStop=docker stop etcd
+        ExecStopPost=docker rm etcd
+        [Install]
+        WantedBy=multi-user.target
     - name: docker.service
       enable: true
     - name: locksmithd.service
@@ -61,36 +66,35 @@ systemd:
         Wants=rpc-statd.service
         [Service]
         EnvironmentFile=/etc/kubernetes/kubelet.env
-        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --volume=resolv,kind=host,source=/etc/resolv.conf \
-          --mount volume=resolv,target=/etc/resolv.conf \
-          --volume var-lib-cni,kind=host,source=/var/lib/cni \
-          --mount volume=var-lib-cni,target=/var/lib/cni \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
-          --mount volume=etc-cni-netd,target=/etc/cni/net.d \
-          --volume iscsiconf,kind=host,source=/etc/iscsi/ \
-          --mount volume=iscsiconf,target=/etc/iscsi/ \
-          --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
-          --mount volume=iscsiadm,target=/sbin/iscsiadm \
-          --insecure-options=image"
-        ExecStartPre=/bin/mkdir -p /opt/cni/bin
-        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
-        ExecStartPre=/bin/mkdir -p /etc/cni/net.d
-        ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
-        ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
-        ExecStartPre=/bin/mkdir -p /var/lib/cni
-        ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+        ExecStartPre=-docker rm -f kubelet
+        ExecStartPre=docker run -d \
+          --name=kubelet \
+          --restart=unless-stopped \
+          --log-driver=journald \
+          --network=host \
+          --pid=host \
+          --privileged \
+          -v /dev:/dev:rw \
+          -v /etc/cni/net.d:/etc/cni/net.d:ro \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run:rw \
+          -v /sys:/sys:rw \
+          -v /opt/cni/bin:/opt/cni/bin:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/cni:/var/lib/cni:rw \
+          -v /var/lib/docker:/var/lib/docker:rw \
+          -v /var/log/pods:/var/log/pods:rw \
+          --mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
+          --mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
+          $${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -115,9 +119,11 @@ systemd:
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
-        RestartSec=10
+        RestartSec=5
         [Install]
         WantedBy=multi-user.target
     - name: bootkube.service
@@ -138,11 +144,36 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
+          KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
           KUBELET_IMAGE_TAG=v1.18.8
-          KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+    - path: /etc/kubernetes/etcd.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          ETCD_IMAGE_TAG=v3.4.13
+          ETCD_IMAGE_URL=quay.io/coreos/etcd
+          ETCD_SSL_DIR=/etc/ssl/etcd
+          ETCD_DATA_DIR=/var/lib/etcd
+          ETCD_USER=etcd
+          ETCD_NAME=${etcd_name}
+          ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379
+          ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380
+          ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
+          ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
+          ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
+          ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
+          ETCD_STRICT_RECONFIG_CHECK=true
+          ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/server-ca.crt
+          ETCD_CERT_FILE=/etc/ssl/etcd/etcd/server.crt
+          ETCD_KEY_FILE=/etc/ssl/etcd/etcd/server.key
+          ETCD_CLIENT_CERT_AUTH=true
+          ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/peer-ca.crt
+          ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd/peer.crt
+          ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd/peer.key
+          ETCD_PEER_CLIENT_CERT_AUTH=true
     - path: /etc/hostname
       filesystem: root
       mode: 0644
@@ -170,19 +201,12 @@ storage:
           docker pull quay.io/poseidon/kubelet:v1.18.8
           # Move experimental manifests
           [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
-          exec /usr/bin/rkt run \
-            --trust-keys-from-https \
-            --volume assets,kind=host,source=/opt/bootkube/assets \
-            --mount volume=assets,target=/assets \
-            --volume bootstrap,kind=host,source=/etc/kubernetes \
-            --mount volume=bootstrap,target=/etc/kubernetes \
-            $${RKT_OPTS} \
-            docker://quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
-            --insecure-options=image \
-            --net=host \
-            --dns=host \
-            --hosts-entry=host \
-            --exec=/bootkube -- start --asset-dir=/assets "$@"
+          exec docker run \
+            -v /opt/bootkube/assets:/assets:ro \
+            -v /etc/kubernetes:/etc/kubernetes:rw \
+            --network=host \
+            quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
+            /bootkube start --asset-dir=/assets
     - path: /etc/kubernetes/configure-kubelet-cgroup-driver
       filesystem: root
       mode: 0744
diff --git a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl
index dbafbdaa1..f52ec3910 100644
--- a/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl
+++ b/assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl
@@ -34,34 +34,35 @@ systemd:
         Wants=rpc-statd.service
         [Service]
         EnvironmentFile=/etc/kubernetes/kubelet.env
-        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --volume=resolv,kind=host,source=/etc/resolv.conf \
-          --mount volume=resolv,target=/etc/resolv.conf \
-          --volume var-lib-cni,kind=host,source=/var/lib/cni \
-          --mount volume=var-lib-cni,target=/var/lib/cni \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
-          --mount volume=etc-cni-netd,target=/etc/cni/net.d \
-          --volume iscsiconf,kind=host,source=/etc/iscsi/ \
-          --mount volume=iscsiconf,target=/etc/iscsi/ \
-          --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
-          --mount volume=iscsiadm,target=/sbin/iscsiadm \
-          --insecure-options=image"
-        ExecStartPre=/bin/mkdir -p /opt/cni/bin
-        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
-        ExecStartPre=/bin/mkdir -p /etc/cni/net.d
-        ExecStartPre=/bin/mkdir -p /var/lib/cni
-        ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+        ExecStartPre=-docker rm -f kubelet
+        ExecStartPre=docker run -d \
+          --name=kubelet \
+          --restart=unless-stopped \
+          --log-driver=journald \
+          --network=host \
+          --pid=host \
+          --privileged \
+          -v /dev:/dev:rw \
+          -v /etc/cni/net.d:/etc/cni/net.d:ro \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run:rw \
+          -v /sys:/sys:rw \
+          -v /opt/cni/bin:/opt/cni/bin:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/cni:/var/lib/cni:rw \
+          -v /var/lib/docker:/var/lib/docker:rw \
+          -v /var/log/pods:/var/log/pods:rw \
+          --mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
+          --mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
+          $${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -85,7 +86,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]
@@ -97,9 +100,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
+          KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
           KUBELET_IMAGE_TAG=v1.18.8
-          KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="${join(",", [for k, v in kubelet_labels : "${k}=${v}"])}"
     - path: /etc/hostname
       filesystem: root