Skip to content
This repository has been archived by the owner on Jun 29, 2022. It is now read-only.

aws,packet,baremetal: Remove rkt #946

Merged
merged 6 commits into from
Oct 8, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions assets/charts/control-plane/kubelet/templates/kubelet-ds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,36 +65,52 @@ spec:
securityContext:
privileged: true
volumeMounts:
# This has two directories `networks` and `cache`.
# cache: This directory is used by CNI to store cache files.
# networks: Certain network providers use this directory.
- mountPath: /var/lib/cni
name: coreos-var-lib-cni
readOnly: false
# This is needed so that the Calico CNI plugin can read the file named `nodename` in the
# following path. This file is created by `calico-node` daemonset pod.
- mountPath: /var/lib/calico
name: coreos-var-lib-calico
readOnly: true
# This directory has CNI plugin binaries.
- mountPath: /opt/cni/bin
name: coreos-opt-cni-bin
readOnly: true
# TODO check if this is needed
- name: dev
mountPath: /dev
readOnly: false
# Here kubelet stores the lock file and unix sockets.
- name: run
mountPath: /run
readOnly: false
- name: sys
mountPath: /sys
readOnly: false
# This is mounted so that node local storage works fine.
- name: mnt
mountPath: /mnt
mountPropagation: Bidirectional
# This directory has certs that kubelet needs to authenticate.
- name: etc-kubernetes
mountPath: /etc/kubernetes
readOnly: true
# This is needed so that kubelet can access the logs (which are symlinked) of the
# containers. The CAdvisor (which is baked into kubelet) needs access to monitor the
# container and report via kubelet monitoring endpoint.
- name: var-lib-docker
mountPath: /var/lib/docker
readOnly: false
- name: var-lib-kubelet
mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
# Logs from the kubelet pods
# Logs from the pods.
- name: logs
mountPath: /var/log/pods
readOnly: false
# This is mounted from host to make sure that the kubelet showcases OS as Flatcar and not
# Debian from the kubelet image.
- name: os-release
Expand All @@ -106,12 +122,17 @@ spec:
- name: etc-resolv
mountPath: /etc/resolv.conf
readOnly: true
# This is mounted so that storage works fine.
- name: iscsiadm
mountPath: /usr/sbin/iscsiadm
readOnly: false
- name: modules
mountPath: /lib/modules
readOnly: true
# Kubelet looks for the CNI config files here.
- name: etc-cni-netd
mountPath: /etc/cni/net.d
readOnly: true
hostNetwork: true
hostPID: true
# Tolerate all the taints. This ensures that the pod runs on all the nodes.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,33 +1,38 @@
---
systemd:
units:
- name: etcd-member.service
- name: etcd.service
invidian marked this conversation as resolved.
Show resolved Hide resolved
enable: true
dropins:
- name: 40-etcd-cluster.conf
contents: |
[Service]
Environment="ETCD_IMAGE_TAG=v3.4.13"
Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
Environment="RKT_RUN_ARGS=--insecure-options=image"
Environment="ETCD_NAME=${etcd_name}"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
ExecStopPost=-/opt/etcd-rejoin
contents: |
[Unit]
Description=etcd (System Application Container)
Documentation=https://github.com/etcd-io/etcd
Wants=docker.service
After=docker.service
[Service]
Type=simple
Restart=always
RestartSec=5s
TimeoutStartSec=0
LimitNOFILE=40000
invidian marked this conversation as resolved.
Show resolved Hide resolved
EnvironmentFile=/etc/kubernetes/etcd.env
ExecStartPre=-docker rm -f etcd
ExecStartPre=sh -c "docker run -d \
--name=etcd \
--restart=unless-stopped \
--log-driver=journald \
--network=host \
-u $(id -u \"$${ETCD_USER}\"):$(id -u \"$${ETCD_USER}\") \
-v $${ETCD_DATA_DIR}:$${ETCD_DATA_DIR}:rw \
-v $${ETCD_SSL_DIR}:$${ETCD_SSL_DIR}:ro \
--env-file /etc/kubernetes/etcd.env \
$${ETCD_IMAGE_URL}:$${ETCD_IMAGE_TAG}"
ExecStart=docker logs -f etcd
ExecStop=docker stop etcd
ExecStopPost=docker rm etcd
ExecStopPost=-/opt/etcd-rejoin
[Install]
WantedBy=multi-user.target
- name: docker.service
enable: true
- name: locksmithd.service
Expand Down Expand Up @@ -55,32 +60,35 @@ systemd:
[Service]
ConditionPathExists=/etc/kubernetes/kubeconfig
EnvironmentFile=/etc/kubernetes/kubelet.env
Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
--volume=resolv,kind=host,source=/etc/resolv.conf \
--mount volume=resolv,target=/etc/resolv.conf \
--volume var-lib-cni,kind=host,source=/var/lib/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
--volume var-lib-calico,kind=host,source=/var/lib/calico \
--mount volume=var-lib-calico,target=/var/lib/calico \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
--volume var-log,kind=host,source=/var/log \
--mount volume=var-log,target=/var/log \
--volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
--mount volume=etc-cni-netd,target=/etc/cni/net.d \
--insecure-options=image"
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStartPre=/bin/mkdir -p /var/lib/calico
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
ExecStart=/usr/lib/coreos/kubelet-wrapper \
ExecStartPre=-docker rm -f kubelet
surajssd marked this conversation as resolved.
Show resolved Hide resolved
ExecStartPre=docker run -d \
--name=kubelet \
--restart=unless-stopped \
--log-driver=journald \
--network=host \
--pid=host \
--privileged \
-v /dev:/dev:rw \
-v /etc/cni/net.d:/etc/cni/net.d:ro \
-v /etc/kubernetes:/etc/kubernetes:ro \
-v /etc/machine-id:/etc/machine-id:ro \
-v /lib/modules:/lib/modules:ro \
-v /run:/run:rw \
Copy link
Member

@rata rata Sep 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not rw just to a specific path only for the kubelet files?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It uses unspecified files from here and a lock file:

# lsof | grep '/run' | grep kubelet                                                                                                           
kubelet   10931                   root    5uW     REG               0,21         0      30352 /run/lock/kubelet.lock        
kubelet   10931                   root   10u     unix 0xffff9fd3b65b6400       0t0     311644 /var/run/661904757 type=STREAM
kubelet   10931                   root   14u     unix 0xffff9fd3b65b4c00       0t0     311194 /var/run/661904757 type=STREAM
kubelet   10931                   root   20u     unix 0xffff9fd3b65b5400       0t0     311195 /var/run/661904757 type=STREAM
kubelet   10931                   root   31u     unix 0xffff9fd2b4d93800       0t0     311748 /var/run/661904757 type=STREAM
kubelet   10931 10948             root    5uW     REG               0,21         0      30352 /run/lock/kubelet.lock        
kubelet   10931 10948             root   10u     unix 0xffff9fd3b65b6400       0t0     311644 /var/run/661904757 type=STREAM
kubelet   10931 10948             root   14u     unix 0xffff9fd3b65b4c00       0t0     311194 /var/run/661904757 type=STREAM
kubelet   10931 10948             root   20u     unix 0xffff9fd3b65b5400       0t0     311195 /var/run/661904757 type=STREAM
kubelet   10931 10948             root   31u     unix 0xffff9fd2b4d93800       0t0     311748 /var/run/661904757 type=STREAM
...

Copy link
Member

@rata rata Sep 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, and the path can't be configured to use another one? If we can lock it down more, IMHO seems better. But we were running like this before with rkt, so no biggie :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, /run/lock must be shared via host path between host kubelet and self-hosted one. But /var/run is separate and only in the container, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Between /var and /var/run, one is symlink of other

-v /sys:/sys:rw \
-v /opt/cni/bin:/opt/cni/bin:ro \
-v /usr/lib/os-release:/etc/os-release:ro \
-v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
-v /var/lib/calico:/var/lib/calico:ro \
invidian marked this conversation as resolved.
Show resolved Hide resolved
-v /var/lib/cni:/var/lib/cni:rw \
-v /var/lib/docker:/var/lib/docker:rw \
surajssd marked this conversation as resolved.
Show resolved Hide resolved
-v /var/log/pods:/var/log/pods:rw \
--mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
--mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
rata marked this conversation as resolved.
Show resolved Hide resolved
$${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
Expand All @@ -104,9 +112,11 @@ systemd:
--read-only-port=0 \
--register-with-taints=$${NODE_TAINTS} \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
rata marked this conversation as resolved.
Show resolved Hide resolved
ExecStart=docker logs -f kubelet
ExecStop=docker stop kubelet
ExecStopPost=docker rm kubelet
Restart=always
RestartSec=10
RestartSec=5
[Install]
WantedBy=multi-user.target
- name: bootkube.service
Expand All @@ -129,11 +139,36 @@ storage:
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
KUBELET_IMAGE_TAG=v1.18.8
KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
- path: /etc/kubernetes/etcd.env
filesystem: root
mode: 0644
contents:
inline: |
ETCD_IMAGE_TAG=v3.4.13
ETCD_IMAGE_URL=quay.io/coreos/etcd
ETCD_SSL_DIR=/etc/ssl/etcd
ETCD_DATA_DIR=/var/lib/etcd
ETCD_USER=etcd
ETCD_NAME=${etcd_name}
ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380
ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
ETCD_STRICT_RECONFIG_CHECK=true
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/server-ca.crt
ETCD_CERT_FILE=/etc/ssl/etcd/etcd/server.crt
ETCD_KEY_FILE=/etc/ssl/etcd/etcd/server.key
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/etcd/peer-ca.crt
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd/peer.crt
invidian marked this conversation as resolved.
Show resolved Hide resolved
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd/peer.key
ETCD_PEER_CLIENT_CERT_AUTH=true
- path: /etc/sysctl.d/max-user-watches.conf
filesystem: root
contents:
Expand All @@ -155,19 +190,12 @@ storage:
docker pull quay.io/poseidon/kubelet:v1.18.8
# Move experimental manifests
[ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
exec /usr/bin/rkt run \
--trust-keys-from-https \
--volume assets,kind=host,source=/opt/bootkube/assets \
--mount volume=assets,target=/assets \
--volume bootstrap,kind=host,source=/etc/kubernetes \
--mount volume=bootstrap,target=/etc/kubernetes \
$${RKT_OPTS} \
docker://quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
--insecure-options=image \
--net=host \
--dns=host \
--hosts-entry=host \
--exec=/bootkube -- start --asset-dir=/assets "$@"
exec docker run \
-v /opt/bootkube/assets:/assets:ro \
-v /etc/kubernetes:/etc/kubernetes:rw \
--network=host \
quay.io/kinvolk/bootkube:v0.14.0-helm-ec64535-amd64 \
/bootkube start --asset-dir=/assets
- path: /etc/tmpfiles.d/etcd-wrapper.conf
filesystem: root
mode: 0644
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,35 @@ systemd:
Wants=rpc-statd.service
[Service]
EnvironmentFile=/etc/kubernetes/kubelet.env
Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
--volume=resolv,kind=host,source=/etc/resolv.conf \
--mount volume=resolv,target=/etc/resolv.conf \
--volume var-lib-cni,kind=host,source=/var/lib/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
--volume var-lib-calico,kind=host,source=/var/lib/calico \
--mount volume=var-lib-calico,target=/var/lib/calico \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
--volume var-log,kind=host,source=/var/log \
--mount volume=var-log,target=/var/log \
--volume etc-cni-netd,kind=host,source=/etc/cni/net.d \
--mount volume=etc-cni-netd,target=/etc/cni/net.d \
--volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
--mount volume=iscsiadm,target=/usr/sbin/iscsiadm \
--insecure-options=image"
ExecStartPre=/bin/mkdir -p /opt/cni/bin
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
ExecStartPre=/bin/mkdir -p /var/lib/cni
ExecStartPre=/bin/mkdir -p /var/lib/calico
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
ExecStartPre=/etc/kubernetes/configure-kubelet-cgroup-driver
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
ExecStart=/usr/lib/coreos/kubelet-wrapper \
ExecStartPre=-docker rm -f kubelet
ExecStartPre=docker run -d \
--name=kubelet \
--restart=unless-stopped \
--log-driver=journald \
--network=host \
--pid=host \
--privileged \
-v /dev:/dev:rw \
-v /etc/cni/net.d:/etc/cni/net.d:ro \
-v /etc/kubernetes:/etc/kubernetes:ro \
-v /etc/machine-id:/etc/machine-id:ro \
-v /lib/modules:/lib/modules:ro \
-v /run:/run:rw \
-v /sys:/sys:rw \
-v /opt/cni/bin:/opt/cni/bin:ro \
-v /usr/lib/os-release:/etc/os-release:ro \
-v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm:rw \
-v /var/lib/calico:/var/lib/calico:ro \
-v /var/lib/cni:/var/lib/cni:rw \
-v /var/lib/docker:/var/lib/docker:rw \
-v /var/log/pods:/var/log/pods:rw \
--mount type=bind,source=/mnt,target=/mnt,bind-propagation=rshared \
--mount type=bind,source=/var/lib/kubelet,target=/var/lib/kubelet,bind-propagation=rshared \
$${KUBELET_IMAGE_URL}:$${KUBELET_IMAGE_TAG} \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
Expand All @@ -78,7 +81,9 @@ systemd:
--read-only-port=0 \
--register-with-taints=$${NODE_TAINTS} \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
ExecStart=docker logs -f kubelet
ExecStop=docker stop kubelet
ExecStopPost=docker rm kubelet
Restart=always
RestartSec=5
[Install]
Expand Down Expand Up @@ -108,9 +113,8 @@ storage:
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
KUBELET_IMAGE_URL=quay.io/poseidon/kubelet
KUBELET_IMAGE_TAG=v1.18.8
KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
NODE_LABELS="node.kubernetes.io/node,${node_labels}"
NODE_TAINTS="${taints}"
- path: /etc/sysctl.d/max-user-watches.conf
Expand Down Expand Up @@ -138,19 +142,16 @@ storage:
inline: |
#!/bin/bash
set -e
exec /usr/bin/rkt run \
--trust-keys-from-https \
--volume config,kind=host,source=/etc/kubernetes \
--mount volume=config,target=/etc/kubernetes \
--insecure-options=image \
docker://quay.io/poseidon/kubelet:v1.18.8 \
--net=host \
--dns=host \
-- \
exec docker run \
--network=host \
-v /etc/kubernetes:/etc/kubernetes:ro \
-v /var/lib/kubelet:/var/lib/kubelet:ro \
--entrypoint=/usr/local/bin/kubectl \
quay.io/poseidon/kubelet:v1.18.8 \
%{~ if enable_tls_bootstrap ~}
kubectl --kubeconfig=/var/lib/kubelet/kubeconfig delete node $(hostname)
--kubeconfig=/var/lib/kubelet/kubeconfig delete node $(hostname)
%{~ else ~}
kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
--kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
%{ endif }
passwd:
users:
Expand Down
Loading