aws: Add cpu_manager_policy to workers

This allows a user to choose the cpu manager policy on a worker pool.
Possible values are: `none` and `static`.

To make this work, kubelet also needs a static allocation of system
reserved and kubernetes reserved CPUs to be defined. So this commit also
adds the default of 300m cores for kube-reserved-cpu and 1500m cores for
system-reserved-cpu when `cpu_manager_policy` is set.

Signed-off-by: Suraj Deshmukh <suraj@kinvolk.io>

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -140,6 +140,13 @@ storage:
           apiVersion: kubelet.config.k8s.io/v1beta1
           kind: KubeletConfiguration
           cgroupDriver: "$${docker_cgroup_driver}"
+          %{~ if cpu_manager_policy == "static" ~}
+          cpuManagerPolicy: ${cpu_manager_policy}
+          systemReserved:
+            cpu: ${system_reserved_cpu}
+          kubeReserved:
+            cpu: ${kube_reserved_cpu}
+          %{~ endif ~}
           EOF
     - path: /etc/kubernetes/delete-node
       filesystem: root

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf

@@ -173,3 +173,22 @@ variable "enable_csi" {
   type        = bool
   default     = false
 }
+
+
+variable "cpu_manager_policy" {
+  description = "CPU Manager policy to use for the worker pool. Possible values: `none`, `static`."
+  default     = "none"
+  type        = string
+}
+
+variable "kube_reserved_cpu" {
+  description = "CPU cores reserved for the Worker Kubernetes components like kubelet, etc."
+  default     = "300m"
+  type        = string
+}
+
+variable "system_reserved_cpu" {
+  description = "CPU cores reserved for the host services like Docker, sshd, kernel, etc."
+  default     = "1500m"
+  type        = string
+}

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/workers.tf

@@ -160,6 +160,10 @@ data "ct_config" "worker-ignition" {
     node_labels            = merge({ "node.kubernetes.io/node" = "" }, var.labels)
     taints                 = var.taints
     enable_tls_bootstrap   = var.enable_tls_bootstrap
+    cpu_manager_policy     = var.cpu_manager_policy
+    system_reserved_cpu    = var.system_reserved_cpu
+    kube_reserved_cpu      = var.kube_reserved_cpu
+
   })
   pretty_print = false
   snippets     = var.clc_snippets

pkg/platform/aws/aws.go

@@ -31,23 +31,24 @@ import (
 )
 
 type workerPool struct {
-	Name         string            `hcl:"pool_name,label"`
-	Count        int               `hcl:"count"`
-	SSHPubKeys   []string          `hcl:"ssh_pubkeys"`
-	InstanceType string            `hcl:"instance_type,optional"`
-	OSChannel    string            `hcl:"os_channel,optional"`
-	OSVersion    string            `hcl:"os_version,optional"`
-	Labels       map[string]string `hcl:"labels,optional"`
-	Taints       map[string]string `hcl:"taints,optional"`
-	DiskSize     int               `hcl:"disk_size,optional"`
-	DiskType     string            `hcl:"disk_type,optional"`
-	DiskIOPS     int               `hcl:"disk_iops,optional"`
-	SpotPrice    string            `hcl:"spot_price,optional"`
-	TargetGroups []string          `hcl:"target_groups,optional"`
-	CLCSnippets  []string          `hcl:"clc_snippets,optional"`
-	Tags         map[string]string `hcl:"tags,optional"`
-	LBHTTPPort   int               `hcl:"lb_http_port,optional"`
-	LBHTTPSPort  int               `hcl:"lb_https_port,optional"`
+	Name             string            `hcl:"pool_name,label"`
+	Count            int               `hcl:"count"`
+	CPUManagerPolicy string            `hcl:"cpu_manager_policy,optional"`
+	SSHPubKeys       []string          `hcl:"ssh_pubkeys"`
+	InstanceType     string            `hcl:"instance_type,optional"`
+	OSChannel        string            `hcl:"os_channel,optional"`
+	OSVersion        string            `hcl:"os_version,optional"`
+	Labels           map[string]string `hcl:"labels,optional"`
+	Taints           map[string]string `hcl:"taints,optional"`
+	DiskSize         int               `hcl:"disk_size,optional"`
+	DiskType         string            `hcl:"disk_type,optional"`
+	DiskIOPS         int               `hcl:"disk_iops,optional"`
+	SpotPrice        string            `hcl:"spot_price,optional"`
+	TargetGroups     []string          `hcl:"target_groups,optional"`
+	CLCSnippets      []string          `hcl:"clc_snippets,optional"`
+	Tags             map[string]string `hcl:"tags,optional"`
+	LBHTTPPort       int               `hcl:"lb_http_port,optional"`
+	LBHTTPSPort      int               `hcl:"lb_https_port,optional"`
 }
 
 type config struct {
@@ -281,6 +282,7 @@ func (c *config) checkValidConfig() hcl.Diagnostics {
 	diagnostics = append(diagnostics, c.checkWorkerPoolNamesUnique()...)
 	diagnostics = append(diagnostics, c.checkNameSizes()...)
 	diagnostics = append(diagnostics, c.checkLBPortsUnique()...)
+	diagnostics = append(diagnostics, c.checkCPUManagerPolicy()...)
 
 	if c.ConntrackMaxPerCore < 0 {
 		diagnostics = append(diagnostics, &hcl.Diagnostic{
@@ -410,3 +412,12 @@ func (c *config) checkWorkerPoolNamesUnique() hcl.Diagnostics {
 
 	return diagnostics
 }
+
+func (c *config) checkCPUManagerPolicy() hcl.Diagnostics {
+	wp := make(map[string]string)
+	for _, w := range c.WorkerPools {
+		wp[w.Name] = w.CPUManagerPolicy
+	}
+
+	return platform.CheckCPUManagerPolicy(wp)
+}

pkg/platform/aws/template.go

@@ -198,6 +198,10 @@ module "worker-pool-{{ $index }}" {
   clc_snippets          = {{ (index $.WorkerpoolCfg $index "clc_snippets") }}
   {{- end }}
 
+  {{- if $pool.CPUManagerPolicy }}
+  cpu_manager_policy = "{{$pool.CPUManagerPolicy}}"
+  {{- end}}
+
   {{- if $pool.Tags }}
   tags                  = {{ index (index $.WorkerpoolCfg $index) "tags" }}
   {{- end }}