Merge pull request #1405 from kinvolk/imran/baremetal-node-specific-l…

…abels

baremetal : new feature node_specific_labels for adding node specific labels.

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/controller.tf

@@ -9,6 +9,7 @@ module "controller" {
   ssh_keys               = var.ssh_keys
   apiserver              = format("%s.%s", var.cluster_name, var.k8s_domain_name)
   ca_cert                = module.bootkube.ca_cert
+  kubelet_labels         = lookup(var.node_specific_labels, var.controller_names[count.index], {})
   clc_snippets           = lookup(var.clc_snippets, var.controller_names[count.index], [])
   set_standard_hostname  = true
 }

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/variables.tf

@@ -203,3 +203,9 @@ variable "install_to_smallest_disk" {
   type        = bool
   default     = false
 }
+
+variable "node_specific_labels" {
+  type        = map(map(string))
+  description = "Map of node specific labels map."
+  default     = {}
+}

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/worker.tf

@@ -7,7 +7,7 @@ module "worker" {
   cluster_domain_suffix  = var.cluster_domain_suffix
   ca_cert                = module.bootkube.ca_cert
   apiserver              = format("%s.%s", var.cluster_name, var.k8s_domain_name)
-  kubelet_labels         = var.labels
+  kubelet_labels         = merge(lookup(var.node_specific_labels, var.worker_names[count.index], {}), var.labels)
   cluster_name           = var.cluster_name
   clc_snippets           = lookup(var.clc_snippets, var.worker_names[count.index], [])
   set_standard_hostname  = true

assets/terraform-modules/controller/main.tf

@@ -47,10 +47,10 @@ data "ct_config" "config" {
     kubelet_image_tag         = var.kubelet_image_tag
     kubelet_docker_extra_args = []
     hostname                  = var.set_standard_hostname == true ? "${var.cluster_name}-controller-${var.count_index}" : ""
-    kubelet_labels = {
+    kubelet_labels = merge(var.kubelet_labels, {
       "node.kubernetes.io/master"     = "",
       "node.kubernetes.io/controller" = "true",
-    }
+    })
     kubelet_taints = {
       "node-role.kubernetes.io/master" = ":NoSchedule"
     }

assets/terraform-modules/controller/variables.tf

@@ -83,3 +83,9 @@ variable "set_standard_hostname" {
   description = "Sets the hostname if true. Hostname is set as <cluster_name>-controller-<count_index>"
   default     = false
 }
+
+variable "kubelet_labels" {
+  type        = map(string)
+  description = "Node labels passed to kubelet --node-labels flag. E.g. { { \"node.kubernetes.io/node\" = \"\" }"
+  default     = {}
+}

ci/baremetal/baremetal-cluster.lokocfg.envsubst

@@ -41,6 +41,20 @@ cluster "bare-metal" {
     "roleofnode" = "testing",
   }
 
+  node_specific_labels = {
+    "node1" = {
+      "testkey": "testvalue"
+    }
+
+    "node2" = {
+      "ingressnode": "yes"
+    }
+
+    "node3" = {
+      "storagenode": "yes"
+    }
+  }
+
   conntrack_max_per_core = 65000
 
   install_to_smallest_disk = "true"

pkg/platform/baremetal/baremetal.go

@@ -30,6 +30,9 @@ import (
 	"github.com/kinvolk/lokomotive/pkg/terraform"
 )
 
+// Labels represent the map of key value string pairs added the kubelet.
+type Labels map[string]string
+
 type config struct {
 	AssetDir                     string              `hcl:"asset_dir"`
 	CachedInstall                string              `hcl:"cached_install,optional"`
@@ -51,7 +54,8 @@ type config struct {
 	WorkerNames                  []string            `hcl:"worker_names"`
 	WorkerMacs                   []string            `hcl:"worker_macs"`
 	WorkerDomains                []string            `hcl:"worker_domains"`
-	Labels                       map[string]string   `hcl:"labels,optional"`
+	Labels                       Labels              `hcl:"labels,optional"`
+	NodeSpecificLabels           map[string]Labels   `hcl:"node_specific_labels,optional"`
 	OIDC                         *oidc.Config        `hcl:"oidc,block"`
 	EncryptPodTraffic            bool                `hcl:"encrypt_pod_traffic,optional"`
 	IgnoreX509CNCheck            bool                `hcl:"ignore_x509_cn_check,optional"`
@@ -220,7 +224,8 @@ func createTerraformConfigFile(cfg *config, terraformPath string) error {
 		WorkerDomains                string
 		DisableSelfHostedKubelet     bool
 		KubeAPIServerExtraFlags      []string
-		Labels                       map[string]string
+		Labels                       Labels
+		NodeSpecificLabels           map[string]Labels
 		EncryptPodTraffic            bool
 		IgnoreX509CNCheck            bool
 		ConntrackMaxPerCore          int
@@ -252,6 +257,7 @@ func createTerraformConfigFile(cfg *config, terraformPath string) error {
 		DisableSelfHostedKubelet:     cfg.DisableSelfHostedKubelet,
 		KubeAPIServerExtraFlags:      cfg.KubeAPIServerExtraFlags,
 		Labels:                       cfg.Labels,
+		NodeSpecificLabels:           cfg.NodeSpecificLabels,
 		EncryptPodTraffic:            cfg.EncryptPodTraffic,
 		IgnoreX509CNCheck:            cfg.IgnoreX509CNCheck,
 		ConntrackMaxPerCore:          cfg.ConntrackMaxPerCore,

pkg/platform/baremetal/template.go

@@ -65,6 +65,20 @@ module "bare-metal-{{.ClusterName}}" {
   }
   {{- end}}
 
+  {{- if .NodeSpecificLabels}}
+  node_specific_labels = {
+    {{- range $nodeName, $mapOfLabels := .NodeSpecificLabels}}
+      {{- if $mapOfLabels }}
+        "{{$nodeName}}" = {
+          {{- range $key, $value := $mapOfLabels }}
+            "{{$key}}" = "{{$value}}",
+          {{- end }}
+        }
+      {{- end }}
+    {{- end }}
+  }
+  {{- end }}
+
   ignore_x509_cn_check   = {{.IgnoreX509CNCheck}}
   conntrack_max_per_core = {{.ConntrackMaxPerCore}}
 

test/components/kubernetes/baremetal_kubelet_labels_test.go

@@ -0,0 +1,82 @@
+// Copyright 2021 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build baremetal
+// +build e2e
+
+package kubernetes_test
+
+import (
+	"context"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkglabels "k8s.io/apimachinery/pkg/labels"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+func Test_Baremetal_NodeSpecificLabels(t *testing.T) {
+	// This map is copied from file ci/baremetal/baremetal-cluster.lokocfg.envsubst
+	// the field is `node_specific_labels`.
+	// The node name keys are constructed as follows: <cluster-name>-<controller/worker>-<index>.
+	// The labels include the labels common to every worker node derived from `labels` and
+	// labels specific to the node.
+	nodeNamesAndExpectedLabelsInCI := map[string]map[string]string{
+		"mercury-controller-0": {
+			"testkey":                       "testvalue",
+			"node.kubernetes.io/master":     "",
+			"node.kubernetes.io/controller": "true",
+		},
+		"mercury-worker-0": {
+			"ingressnode": "yes",
+			"testing.io":  "yes",
+			"roleofnode":  "testing",
+		},
+		"mercury-worker-1": {
+			"storagenode": "yes",
+			"testing.io":  "yes",
+			"roleofnode":  "testing",
+		},
+	}
+
+	client := testutil.CreateKubeClient(t)
+
+	nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatalf("could not list nodes: %v", err)
+	}
+
+	if len(nodes.Items) != len(nodeNamesAndExpectedLabelsInCI) {
+		t.Errorf("expected %d worker nodes, got: %d", len(nodeNamesAndExpectedLabelsInCI), len(nodes.Items))
+	}
+
+	for nodeName, labelsMap := range nodeNamesAndExpectedLabelsInCI {
+		labels := pkglabels.Set(labelsMap).String()
+
+		nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+			LabelSelector: labels,
+		})
+		if err != nil {
+			t.Errorf("expected node with name %q not found: %v", nodeName, err)
+		}
+
+		// Currently we assume that the labels added to the node in CI are unique to the worker node.
+		// This check confirms that the expected set of labels as per CI configuration were
+		// present on the correct node.
+		if len(nodes.Items) == 1 && nodes.Items[0].Name != nodeName {
+			t.Errorf("expected node %q to have the labels %s", nodes.Items[0].Name, labels)
+		}
+	}
+}