AWS: Add support for custom taints and labels

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -70,6 +70,7 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
+          --register-with-taints=$${NODE_TAINTS} \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
@@ -104,7 +105,8 @@ storage:
           KUBELET_IMAGE_URL=docker://quay.io/poseidon/kubelet
           KUBELET_IMAGE_TAG=v1.18.6
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
-          NODE_LABELS="node.kubernetes.io/node"
+          NODE_LABELS="node.kubernetes.io/node,${node_labels}"
+          NODE_TAINTS="${taints}"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/variables.tf

@@ -39,6 +39,18 @@ variable "instance_type" {
   description = "EC2 instance type"
 }
 
+variable "labels" {
+  type        = string
+  default     = ""
+  description = "Custom labels to assign to worker nodes. Provide comma separated key=value pairs as labels. e.g. 'foo=oof,bar=,baz=zab'"
+}
+
+variable "taints" {
+  type        = string
+  default     = ""
+  description = "Comma separated list of taints. eg. 'clusterType=staging:NoSchedule,nodeType=storage:NoSchedule'"
+}
+
 variable "os_name" {
   type        = string
   default     = "flatcar"

assets/terraform-modules/aws/flatcar-linux/kubernetes/workers/workers.tf

@@ -96,5 +96,7 @@ data "template_file" "worker-config" {
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
+    node_labels            = var.labels
+    taints                 = var.taints
   }
 }

ci/aws/aws-cluster.lokocfg.envsubst

@@ -33,7 +33,7 @@ storage:
 EOF
   ]
 
-  worker_pool "$CLUSTER_ID-wp" {
+  worker_pool "$CLUSTER_ID-w1" {
     count         = 2
     ssh_pubkeys   = ["$PUB_KEY"]
     disk_size     = 30
@@ -59,6 +59,38 @@ EOF
     ]
   }
 
+  worker_pool "$CLUSTER_ID-w2" {
+    count         = 1
+    ssh_pubkeys   = ["$PUB_KEY"]
+    disk_size     = 30
+    instance_type = "t2.small"
+    spot_price    = "0.01"
+    labels        = "testing.io=yes,roleofnode=testing"
+    taints        = "nodeType=storage:NoSchedule"
+
+    # TODO: remove this once https://github.com/kinvolk/lokomotive/issues/839 is fixed.
+    lb_http_port  = 8080
+    lb_https_port = 8443
+    tags = {
+      "deployment" = "ci"
+    }
+    clc_snippets = [
+      <<EOF
+storage:
+  files:
+    - path: /opt/clc_snippet_hello
+      filesystem: root
+      contents:
+        inline: Hello, world!
+      mode: 0644
+      user:
+        id: 500
+      group:
+        id: 500
+EOF
+    ]
+  }
+
   # Adds oidc flags to API server with default values.
   # Acts as a smoke test to check if API server is functional after addition
   # of extra flags.

docs/configuration-reference/platforms/aws.md

@@ -141,6 +141,10 @@ cluster "aws" {
 
     os_version = "current"
 
+    labels = "foo=bar,baz=zab"
+
+    taints = "nodeType=storage:NoSchedule"
+
     disk_size = var.worker_disk_size
 
     disk_type = var.worker_disk_type
@@ -228,6 +232,8 @@ worker_pool "my-worker-pool" {
 | `worker_pool.ssh_pubkeys`     | List of SSH public keys for user `core`. Each element must be specified in a valid OpenSSH public key format, as defined in RFC 4253 Section 6.6, e.g. "ssh-rsa AAAAB3N...".               |        -        | list(string) |   true   |
 | `worker_pool.os_channel`      | Flatcar Container Linux channel to install from (stable, beta, alpha, edge).                                                                                                               |    "stable"     |    string    |  false   |
 | `worker_pool.os_version`      | Flatcar Container Linux version to install. Version such as "2303.3.1" or "current".                                                                                                       |    "current"    |    string    |  false   |
+| `worker_pool.labels`          | Custom labels to assign to worker nodes such as `foo=bar,baz=zab`.                                                                                                                         |        -        |    string    |  false   |
+| `worker_pool.taints`          | Taints to assign to worker nodes such as `nodeType=storage:NoSchedule`.                                                                                                                    |        -        |    string    |  false   |
 | `worker_pool.disk_size`       | Size of the EBS volume in GB.                                                                                                                                                              |       40        |    number    |  false   |
 | `worker_pool.disk_type`       | Type of the EBS volume (e.g. standard, gp2, io1).                                                                                                                                          |      "gp2"      |    string    |  false   |
 | `worker_pool.disk_iops`       | IOPS of the EBS volume (e.g 100).                                                                                                                                                          |        0        |    number    |  false   |

pkg/platform/aws/aws.go

@@ -39,6 +39,8 @@ type workerPool struct {
 	InstanceType string            `hcl:"instance_type,optional"`
 	OSChannel    string            `hcl:"os_channel,optional"`
 	OSVersion    string            `hcl:"os_version,optional"`
+	Labels       string            `hcl:"labels,optional"`
+	Taints       string            `hcl:"taints,optional"`
 	DiskSize     int               `hcl:"disk_size,optional"`
 	DiskType     string            `hcl:"disk_type,optional"`
 	DiskIOPS     int               `hcl:"disk_iops,optional"`

pkg/platform/aws/template.go

@@ -139,6 +139,14 @@ module "worker-pool-{{ $index }}" {
   os_version            = "{{ $pool.OSVersion }}"
   {{- end }}
 
+  {{- if $pool.Labels }}
+  labels = "{{ $pool.Labels }}"
+  {{- end}}
+
+  {{- if $pool.Taints }}
+  taints = "{{ $pool.Taints }}"
+  {{- end}}
+
   {{- if $pool.DiskSize }}
   disk_size             = "{{ $pool.DiskSize }}"
   {{- end }}

test/components/kubernetes/kubelet_disruptive_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build packet baremetal
+// +build packet baremetal aws
 // +build disruptivee2e
 
 package kubernetes

test/components/kubernetes/kubelet_labels_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build packet baremetal
+// +build packet baremetal aws
 // +build e2e
 
 package kubernetes //nolint:testpackage

test/components/kubernetes/kubelet_taint_test.go

@@ -0,0 +1,54 @@
+// Copyright 2020 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build aws
+// +build e2e
+
+package kubernetes_test
+
+import (
+	"context"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+func TestNodeHasTaints(t *testing.T) {
+	client := testutil.CreateKubeClient(t)
+
+	nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+		LabelSelector: "testing.io=yes,roleofnode=testing",
+	})
+	if err != nil {
+		t.Errorf("could not list nodes: %v", err)
+	}
+
+	if len(nodes.Items) == 0 {
+		t.Fatalf("no worker nodes found")
+	}
+
+	for _, items := range nodes.Items {
+		for _, taint := range items.Spec.Taints {
+			taintKey := taint.Key
+			taintValue := taint.Value
+			taintEffect := taint.Effect
+
+			if taintKey != "nodeType" || taintValue != "storage" || taintEffect != "NoSchedule" {
+				t.Fatalf("expected taint: %s, got: %s=%s:%s", "nodeType=storage:NoSchedule", taintKey, taintValue, taintEffect)
+			}
+		}
+	}
+}