AWS: Add support for custom taints and labels

User can now use custom labels and taints as part of cluster creation. Signed-off-by: knrt10 <kautilya@kinvolk.io>
kinvolk · Aug 24, 2020 · ea85c7d · ea85c7d
1 parent a175b37
commit ea85c7d
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 7 deletions.
diff --git a/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl b/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl
@@ -70,6 +70,7 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
+          --register-with-taints=$${NODE_TAINTS} \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
@@ -104,7 +105,8 @@ storage:
           KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
           KUBELET_IMAGE_TAG=v1.18.6
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
-          NODE_LABELS="node.kubernetes.io/node"
+          NODE_LABELS="node.kubernetes.io/node,${node_labels}"
+          NODE_TAINTS="${taints}"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

diff --git a/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf b/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf
@@ -39,6 +39,18 @@ variable "instance_type" {
   description = "EC2 instance type"
 }
 
+variable "labels" {
+  type        = string
+  default     = ""
+  description = "Custom labels to assign to worker nodes. Provide comma separated key=value pairs as labels. e.g. 'foo=oof,bar=,baz=zab'"
+}
+
+variable "taints" {
+  type        = string
+  default     = ""
+  description = "Comma separated list of taints. eg. 'clusterType=staging:NoSchedule,nodeType=storage:NoSchedule'"
+}
+
 variable "os_name" {
   type        = string
   default     = "flatcar"

diff --git a/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/workers.tf b/assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/workers.tf
@@ -96,5 +96,7 @@ data "template_file" "worker-config" {
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
+    node_labels            = var.labels
+    taints                 = var.taints
   }
 }
diff --git a/pkg/assets/generated_assets.go b/pkg/assets/generated_assets.go
diff --git a/pkg/platform/aws/aws.go b/pkg/platform/aws/aws.go
@@ -39,6 +39,8 @@ type workerPool struct {
 	InstanceType string            `hcl:"instance_type,optional"`
 	OSChannel    string            `hcl:"os_channel,optional"`
 	OSVersion    string            `hcl:"os_version,optional"`
+	Labels       string            `hcl:"labels,optional"`
+	Taints       string            `hcl:"taints,optional"`
 	DiskSize     int               `hcl:"disk_size,optional"`
 	DiskType     string            `hcl:"disk_type,optional"`
 	DiskIOPS     int               `hcl:"disk_iops,optional"`

diff --git a/pkg/platform/aws/template.go b/pkg/platform/aws/template.go
@@ -139,6 +139,14 @@ module "worker-pool-{{ $index }}" {
   os_version            = "{{ $pool.OSVersion }}"
   {{- end }}
 
+  {{- if $pool.Labels }}
+  labels                = "{{ $pool.Labels }}"
+  {{- end}}
+
+  {{- if $pool.Taints }}
+  taints                = "{{ $pool.Taints }}"
+  {{- end}}
+
   {{- if $pool.DiskSize }}
   disk_size             = "{{ $pool.DiskSize }}"
   {{- end }}