Skip to content
This repository has been archived by the owner on Jun 29, 2022. It is now read-only.

Commit

Permalink
baremetal: Remove rkt
Browse files Browse the repository at this point in the history 
This commit removes rkt and uses docker to start services.

etcd(for controllers only):

  - Move the env var to a separate env var file.
  - Add etcd service file.
  - Change the name of service from etcd-member to etcd.

Bootkube(for controllers only):

  Use `docker run` instead of rkt.

Kubelet(for controllers and workers):

  Remove some of the folder creation in `ExecStartPre`, these are
  automatically created by docker, when mounted using `-v` flag.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>
  • Loading branch information
@surajssd
surajssd committed Sep 11, 2020
1 parent 727ad3c commit 2ff781e
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 96 deletions.
144 changes: 76 additions & 68 deletions assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,29 @@
---
systemd:
units:
- name: etcd-member.service
- name: etcd.service
enable: true
dropins:
- name: 40-etcd-cluster.conf
contents: |
[Service]
Environment="ETCD_IMAGE_TAG=v3.4.13"
Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
Environment="RKT_RUN_ARGS=--insecure-options=image"
Environment="ETCD_NAME=${etcd_name}"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
contents: |
[Unit]
Description=etcd (System Application Container)
Documentation=https://github.com/etcd-io/etcd
Wants=docker.service
After=docker.service
[Service]
Type=simple
Restart=always
RestartSec=10s
TimeoutStartSec=0
LimitNOFILE=40000
EnvironmentFile=/etc/kubernetes/etcd.env
ExecStart=sh -c "docker run --network=host \
-u $(id -u \"$${ETCD_USER}\") \
-v $${ETCD_DATA_DIR}:$${ETCD_DATA_DIR}:rw \
-v $${ETCD_SSL_DIR}:$${ETCD_SSL_DIR}:ro \
--env-file /etc/kubernetes/etcd.env \
$${ETCD_IMAGE_URL}:$${ETCD_IMAGE_TAG}"
[Install]
WantedBy=multi-user.target
- name: docker.service
enable: true
- name: locksmithd.service
Expand Down Expand Up @@ -61,36 +58,30 @@ systemd:
Wants=rpc-statd.service
[Service]
EnvironmentFile=/etc/kubernetes/kubelet.env
Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
--volume=resolv,kind=host,source=/etc/resolv.conf \
--mount volume=resolv,target=/etc/resolv.conf \
--volume var-lib-cni,kind=host,source=/var/lib/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
--volume var-lib-calico,kind=host,source=/var/lib/calico \
--mount volume=var-lib-calico,target=/var/lib/calico \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
--volume var-log,kind=host,source=/var/log \
--mount volume=var-log,target=/var/log \
--volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
--mount volume=etc-cni-netd,target=/etc/cni/net.d \
--volume iscsiconf,kind=host,source=/etc/iscsi/ \
--mount volume=iscsiconf,target=/etc/iscsi/ \
--volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
--mount volume=iscsiadm,target=/sbin/iscsiadm \
--insecure-options=image"
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStartPre=/bin/mkdir -p /var/lib/calico
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
ExecStart=/usr/lib/coreos/kubelet-wrapper \
ExecStart=docker run \
--network=host \
--pid=host \
--privileged \
-v /dev:/dev:rw \
-v /etc/cni/net.d:/etc/cni/net.d:rw \
-v /etc/kubernetes:/etc/kubernetes:ro \
-v /etc/machine-id:/etc/machine-id \
-v /lib/modules:/lib/modules \
-v /run:/run:rw \
-v /sys:/sys:rw \
-v /opt/cni/bin:/opt/cni/bin:ro \
-v /usr/lib/os-release:/etc/os-release:ro \
-v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
-v /var/lib/calico:/var/lib/calico:ro \
-v /var/lib/cni:/var/lib/cni:rw \
-v /var/lib/docker:/var/lib/docker:rw \
-v /var/log/pods:/var/log/pods:rw \
--mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
--mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
$${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
Expand All @@ -115,7 +106,6 @@ systemd:
--read-only-port=0 \
--register-with-taints=$${NODE_TAINTS} \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
Restart=always
RestartSec=10
[Install]
Expand All @@ -138,11 +128,36 @@ storage:
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
KUBELET_IMAGE_TAG=v1.18.8
KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
- path: /etc/kubernetes/etcd.env
filesystem: root
mode: 0644
contents:
inline: |
ETCD_IMAGE_TAG=v3.4.13
ETCD_IMAGE_URL=quay.io/coreos/etcd
ETCD_SSL_DIR=/etc/ssl/etcd
ETCD_DATA_DIR=/var/lib/etcd
ETCD_USER=etcd
ETCD_NAME=${etcd_name}
ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380
ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
ETCD_STRICT_RECONFIG_CHECK=true
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/server-ca.crt
ETCD_CERT_FILE=/etc/ssl/etcd/etcd/server.crt
ETCD_KEY_FILE=/etc/ssl/etcd/etcd/server.key
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/peer-ca.crt
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd/peer.crt
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd/peer.key
ETCD_PEER_CLIENT_CERT_AUTH=true
- path: /etc/hostname
filesystem: root
mode: 0644
Expand Down Expand Up @@ -170,19 +185,12 @@ storage:
docker pull quay.io/poseidon/kubelet:v1.18.8
# Move experimental manifests
[ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
exec /usr/bin/rkt run \
--trust-keys-from-https \
--volume assets,kind=host,source=/opt/bootkube/assets \
--mount volume=assets,target=/assets \
--volume bootstrap,kind=host,source=/etc/kubernetes \
--mount volume=bootstrap,target=/etc/kubernetes \
$${RKT_OPTS} \
docker://quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
--insecure-options=image \
--net=host \
--dns=host \
--hosts-entry=host \
--exec=/bootkube -- start --asset-dir=/assets "$@"
exec docker run \
-v /opt/bootkube/assets:/assets \
-v /etc/kubernetes:/etc/kubernetes \
--network=host \
quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
/bootkube start --asset-dir=/assets
- path: /etc/kubernetes/configure-kubelet-cgroup-driver
filesystem: root
mode: 0744
Expand Down
50 changes: 22 additions & 28 deletions assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,34 +34,30 @@ systemd:
Wants=rpc-statd.service
[Service]
EnvironmentFile=/etc/kubernetes/kubelet.env
Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
--volume=resolv,kind=host,source=/etc/resolv.conf \
--mount volume=resolv,target=/etc/resolv.conf \
--volume var-lib-cni,kind=host,source=/var/lib/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
--volume var-lib-calico,kind=host,source=/var/lib/calico \
--mount volume=var-lib-calico,target=/var/lib/calico \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
--volume var-log,kind=host,source=/var/log \
--mount volume=var-log,target=/var/log \
--volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
--mount volume=etc-cni-netd,target=/etc/cni/net.d \
--volume iscsiconf,kind=host,source=/etc/iscsi/ \
--mount volume=iscsiconf,target=/etc/iscsi/ \
--volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
--mount volume=iscsiadm,target=/sbin/iscsiadm \
--insecure-options=image"
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStartPre=/bin/mkdir -p /var/lib/calico
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
ExecStart=/usr/lib/coreos/kubelet-wrapper \
ExecStart=docker run \
--network=host \
--pid=host \
--privileged \
-v /dev:/dev:rw \
-v /etc/cni/net.d:/etc/cni/net.d:rw \
-v /etc/kubernetes:/etc/kubernetes:ro \
-v /etc/machine-id:/etc/machine-id \
-v /lib/modules:/lib/modules \
-v /run:/run:rw \
-v /sys:/sys:rw \
-v /opt/cni/bin:/opt/cni/bin:ro \
-v /usr/lib/os-release:/etc/os-release:ro \
-v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
-v /var/lib/calico:/var/lib/calico:ro \
-v /var/lib/cni:/var/lib/cni:rw \
-v /var/lib/docker:/var/lib/docker:rw \
-v /var/log/pods:/var/log/pods:rw \
--mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
--mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
$${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
Expand All @@ -85,7 +81,6 @@ systemd:
--pod-manifest-path=/etc/kubernetes/manifests \
--read-only-port=0 \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
Restart=always
RestartSec=5
[Install]
Expand All @@ -97,9 +92,8 @@ storage:
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
KUBELET_IMAGE_TAG=v1.18.8
KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
NODE_LABELS="${join(",", [for k, v in kubelet_labels : "${k}=${v}"])}"
- path: /etc/hostname
filesystem: root
Expand Down

0 comments on commit 2ff781e

Please sign in to comment.