Kubelet configuration Machine annotations (#1135)

* New v1.kubelet-config.machine-controller.kubermatic.io annotation To pass some kubeletConfiguraton custom values. Possible values are: * SystemReserved * KubeReserved * EvictionHard If those found, serialized map will be used to popualate corresponding kubeletConfiguraton field. Signed-off-by: Artiom Diomin <kron82@gmail.com> * Update fixtures Signed-off-by: Artiom Diomin <kron82@gmail.com>
kubermatic · Dec 19, 2021 · 953f239 · 953f239
1 parent 791cfbc
commit 953f239
Show file tree

Hide file tree

Showing 73 changed files with 371 additions and 20 deletions.
diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go
@@ -134,10 +134,17 @@ const (
 	ExternalCloudProviderKubeletFlag KubeletFlags = "ExternalCloudProvider"
 )
 
+const (
+	SystemReservedKubeletConfig = "SystemReserved"
+	KubeReservedKubeletConfig   = "KubeReserved"
+	EvictionHardKubeletConfig   = "EvictionHard"
+)
+
 const (
 	// Annotation prefixes, used on Machine objects to indicate the parameters that been used to create those Machines
 	KubeletFeatureGatesAnnotationPrefixV1 = "v1.kubelet-featuregates.machine-controller.kubermatic.io"
 	KubeletFlagsGroupAnnotationPrefixV1   = "v1.kubelet-flags.machine-controller.kubermatic.io"
+	KubeletConfigAnnotationPrefixV1       = "v1.kubelet-config.machine-controller.kubermatic.io"
 )
 
 // SetKubeletFeatureGates marshal and save featureGates into metaobject annotations with
@@ -165,11 +172,25 @@ func SetKubeletFlags(metaobj metav1.Object, flags map[KubeletFlags]string) {
 	metaobj.SetAnnotations(annts)
 }
 
-func GetKubeletFeatureGates(metaobj metav1.Object) map[string]bool {
+func GetKubeletConfigs(annotations map[string]string) map[string]string {
+	configs := map[string]string{}
+	for name, value := range annotations {
+		if strings.HasPrefix(name, KubeletConfigAnnotationPrefixV1) {
+			nameConfigValue := strings.SplitN(name, "/", 2)
+			if len(nameConfigValue) != 2 {
+				continue
+			}
+			configs[nameConfigValue[1]] = value
+		}
+	}
+	return configs
+}
+
+func GetKubeletFeatureGates(annotations map[string]string) map[string]bool {
 	result := map[string]bool{}
-	for name, value := range metaobj.GetAnnotations() {
+	for name, value := range annotations {
 		if strings.HasPrefix(name, KubeletFeatureGatesAnnotationPrefixV1) {
-			nameGateValue := strings.Split(name, "/")
+			nameGateValue := strings.SplitN(name, "/", 2)
 			if len(nameGateValue) != 2 {
 				continue
 			}
@@ -180,11 +201,11 @@ func GetKubeletFeatureGates(metaobj metav1.Object) map[string]bool {
 	return result
 }
 
-func GetKubeletFlags(metaobj metav1.Object) map[KubeletFlags]string {
+func GetKubeletFlags(annotations map[string]string) map[KubeletFlags]string {
 	result := map[KubeletFlags]string{}
-	for name, value := range metaobj.GetAnnotations() {
+	for name, value := range annotations {
 		if strings.HasPrefix(name, KubeletFlagsGroupAnnotationPrefixV1) {
-			nameFlagValue := strings.Split(name, "/")
+			nameFlagValue := strings.SplitN(name, "/", 2)
 			if len(nameFlagValue) != 2 {
 				continue
 			}

diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go
@@ -51,6 +51,7 @@ type UserDataRequest struct {
 	NoProxy               string
 	PauseImage            string
 	KubeletFeatureGates   map[string]bool
+	KubeletConfigs        map[string]string
 	ContainerRuntime      containerruntime.Config
 	PodCIDR               string
 	NodePortRange         string

diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go
@@ -705,14 +705,15 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 			}
 
 			// grab kubelet featureGates from the annotations
-			kubeletFeatureGates := common.GetKubeletFeatureGates(machine)
+			kubeletFeatureGates := common.GetKubeletFeatureGates(machine.GetAnnotations())
 			if len(kubeletFeatureGates) == 0 {
 				// fallback to command-line input
 				kubeletFeatureGates = r.nodeSettings.KubeletFeatureGates
 			}
 
 			// grab kubelet general options from the annotations
-			kubeletFlags := common.GetKubeletFlags(machine)
+			kubeletFlags := common.GetKubeletFlags(machine.GetAnnotations())
+			KubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations())
 
 			// look up for ExternalCloudProvider feature, with fallback to command-line input
 			externalCloudProvider := r.nodeSettings.ExternalCloudProvider
@@ -729,6 +730,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 				DNSIPs:                r.nodeSettings.ClusterDNSIPs,
 				PauseImage:            r.nodeSettings.PauseImage,
 				KubeletFeatureGates:   kubeletFeatureGates,
+				KubeletConfigs:        KubeletConfigs,
 				NoProxy:               r.nodeSettings.NoProxy,
 				HTTPProxy:             r.nodeSettings.HTTPProxy,
 				ContainerRuntime:      r.nodeSettings.ContainerRuntime,

diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go
@@ -265,7 +265,7 @@ write_files:
 
 - path: "/etc/kubernetes/kubelet.conf"
   content: |
-{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }}
+{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }}
 
 - path: "/etc/kubernetes/pki/ca.crt"
   content: |

diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml
@@ -300,6 +300,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml
@@ -297,6 +297,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml
@@ -297,6 +297,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml
@@ -297,6 +297,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml
@@ -297,6 +297,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml
@@ -314,6 +314,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml
@@ -314,6 +314,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml
@@ -305,6 +305,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml
@@ -300,6 +300,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go
@@ -265,7 +265,7 @@ write_files:
 
 - path: "/etc/kubernetes/kubelet.conf"
   content: |
-{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }}
+{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }}
 
 - path: "/etc/kubernetes/pki/ca.crt"
   content: |

diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml
@@ -301,6 +301,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml
@@ -302,6 +302,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml
@@ -302,6 +302,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml
@@ -302,6 +302,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml
@@ -302,6 +302,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml
@@ -319,6 +319,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml
@@ -319,6 +319,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml
@@ -310,6 +310,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml
@@ -301,6 +301,11 @@ write_files:
     cgroupDriver: systemd
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go
@@ -277,7 +277,7 @@ storage:
       mode: 0644
       contents:
         inline: |
-{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 10 }}
+{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 10 }}
 
     - path: /opt/load-kernel-modules.sh
       filesystem: root
@@ -551,7 +551,7 @@ write_files:
 - path: "/etc/kubernetes/kubelet.conf"
   permissions: "0644"
   content: |
-{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }}
+{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }}
 
 - path: /opt/load-kernel-modules.sh
   permissions: "0755"

diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml
@@ -175,6 +175,11 @@ write_files:
     - 10.10.10.10
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml
@@ -175,6 +175,11 @@ write_files:
     - 10.10.10.10
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml
@@ -175,6 +175,11 @@ write_files:
     - 10.10.10.10
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true

diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml
@@ -175,6 +175,11 @@ write_files:
     - 10.10.10.10
     clusterDomain: cluster.local
     cpuManagerReconcilePeriod: 0s
+    evictionHard:
+      imagefs.available: 15%
+      memory.available: 100Mi
+      nodefs.available: 10%
+      nodefs.inodesFree: 5%
     evictionPressureTransitionPeriod: 0s
     featureGates:
       RotateKubeletServerCertificate: true