[Enhancement] Refactoring: normalize label flags (k3s node & runtime) (#598, @ejose19)

cmd/cluster/clusterCreate.go

@@ -263,8 +263,11 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`")
 	_ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port"))
 
-	cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
-	_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))
+	cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label"))
+
+	cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
 
 	/******************
 	 * "Normal" Flags *
@@ -464,10 +467,38 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 
 	log.Tracef("PortFilterMap: %+v", portFilterMap)
 
-	// --label
-	// labelFilterMap will add container label to applied node filters
-	labelFilterMap := make(map[string][]string, 1)
-	for _, labelFlag := range ppViper.GetStringSlice("cli.labels") {
+	// --k3s-node-label
+	// k3sNodeLabelFilterMap will add k3s node label to applied node filters
+	k3sNodeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") {
+
+		// split node filter from the specified label
+		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		// create new entry or append filter to existing entry
+		if _, exists := k3sNodeLabelFilterMap[label]; exists {
+			k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...)
+		} else {
+			k3sNodeLabelFilterMap[label] = nodeFilters
+		}
+	}
+
+	for label, nodeFilters := range k3sNodeLabelFilterMap {
+		cfg.K3sNodeLabels = append(cfg.K3sNodeLabels, conf.LabelWithNodeFilters{
+			Label:       label,
+			NodeFilters: nodeFilters,
+		})
+	}
+
+	log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
+
+	// --runtime-label
+	// runtimeLabelFilterMap will add container runtime label to applied node filters
+	runtimeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") {
 
 		// split node filter from the specified label
 		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
@@ -476,21 +507,21 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		}
 
 		// create new entry or append filter to existing entry
-		if _, exists := labelFilterMap[label]; exists {
-			labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...)
+		if _, exists := runtimeLabelFilterMap[label]; exists {
+			runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...)
 		} else {
-			labelFilterMap[label] = nodeFilters
+			runtimeLabelFilterMap[label] = nodeFilters
 		}
 	}
 
-	for label, nodeFilters := range labelFilterMap {
-		cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{
+	for label, nodeFilters := range runtimeLabelFilterMap {
+		cfg.RuntimeLabels = append(cfg.RuntimeLabels, conf.LabelWithNodeFilters{
 			Label:       label,
 			NodeFilters: nodeFilters,
 		})
 	}
 
-	log.Tracef("LabelFilterMap: %+v", labelFilterMap)
+	log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
 
 	// --env
 	// envFilterMap will add container env vars to applied node filters

cmd/node/nodeCreate.go

@@ -74,6 +74,7 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
 	cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
 
+	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
 	// done
@@ -127,17 +128,34 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 		log.Errorf("Provided memory limit value is invalid")
 	}
 
+	// --runtime-label
+	runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label")
+	if err != nil {
+		log.Errorln("No runtime-label specified")
+		log.Fatalln(err)
+	}
+
+	runtimeLabels := make(map[string]string, len(runtimeLabelsFlag))
+	for _, label := range runtimeLabelsFlag {
+		labelSplitted := strings.Split(label, "=")
+		if len(labelSplitted) != 2 {
+			log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
+		}
+		runtimeLabels[labelSplitted[0]] = labelSplitted[1]
+	}
+
+	// --k3s-node-label
 	k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
 	if err != nil {
-		log.Errorln("No node-label specified")
+		log.Errorln("No k3s-node-label specified")
 		log.Fatalln(err)
 	}
 
 	k3sNodeLabels := make(map[string]string, len(k3sNodeLabelsFlag))
 	for _, label := range k3sNodeLabelsFlag {
 		labelSplitted := strings.Split(label, "=")
 		if len(labelSplitted) != 2 {
-			log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label)
+			log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
 		}
 		k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
 	}
@@ -153,6 +171,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 				k3d.LabelRole: roleStr,
 			},
 			K3sNodeLabels: k3sNodeLabels,
+			RuntimeLabels: runtimeLabels,
 			Restart:       true,
 			Memory:        memory,
 		}

pkg/client/node.go

@@ -315,10 +315,17 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
 	for k, v := range node.Labels {
 		labels[k] = v
 	}
+	for k, v := range node.RuntimeLabels {
+		labels[k] = v
+	}
 	node.Labels = labels
 	// second most important: the node role label
 	node.Labels[k3d.LabelRole] = string(node.Role)
 
+	for k, v := range node.K3sNodeLabels {
+		node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
+	}
+
 	// ### Environment ###
 	node.Env = append(node.Env, k3d.DefaultNodeEnv...) // append default node env vars
 
@@ -418,10 +425,6 @@ func patchAgentSpec(node *k3d.Node) error {
 		node.Cmd = []string{"agent"}
 	}
 
-	for k, v := range node.K3sNodeLabels {
-		node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
-	}
-
 	return nil
 }
 

pkg/config/config_test.go

@@ -63,7 +63,13 @@ func TestReadSimpleConfig(t *testing.T) {
 				NodeFilters: []string{"loadbalancer"},
 			},
 		},
-		Labels: []conf.LabelWithNodeFilters{
+		K3sNodeLabels: []conf.LabelWithNodeFilters{
+			{
+				Label:       "foo=bar",
+				NodeFilters: []string{"server[0]", "loadbalancer"},
+			},
+		},
+		RuntimeLabels: []conf.LabelWithNodeFilters{
 			{
 				Label:       "foo=bar",
 				NodeFilters: []string{"server[0]", "loadbalancer"},

pkg/config/test_assets/config_test_simple.yaml

@@ -22,7 +22,12 @@ env:
   - envVar: bar=baz
     nodeFilters:
       - all
-labels:
+k3sNodeLabels:
+  - label: foo=bar
+    nodeFilters:
+      - server[0]
+      - loadbalancer
+runtimeLabels:
   - label: foo=bar
     nodeFilters:
       - server[0]
@@ -40,4 +45,4 @@ options:
     extraAgentArgs: []
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true

pkg/config/test_assets/config_test_simple_invalid_servers.yaml

@@ -22,7 +22,12 @@ env:
   - envVar: bar=baz
     nodeFilters:
       - all
-labels:
+k3sNodeLabels:
+  - label: foo=bar
+    nodeFilters:
+      - server[0]
+      - loadbalancer
+runtimeLabels:
   - label: foo=bar
     nodeFilters:
       - server[0]
@@ -40,4 +45,4 @@ options:
     extraAgentArgs: []
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true

pkg/config/transform.go

@@ -192,23 +192,44 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		}
 	}
 
-	// -> LABELS
-	for _, labelWithNodeFilters := range simpleConfig.Labels {
-		if len(labelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
-			return nil, fmt.Errorf("Labelmapping '%s' lacks a node filter, but there's more than one node", labelWithNodeFilters.Label)
+	// -> K3S NODE LABELS
+	for _, k3sNodeLabelWithNodeFilters := range simpleConfig.K3sNodeLabels {
+		if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
+			return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label)
 		}
 
-		nodes, err := util.FilterNodes(nodeList, labelWithNodeFilters.NodeFilters)
+		nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, node := range nodes {
-			if node.Labels == nil {
-				node.Labels = make(map[string]string) // ensure that the map is initialized
+			if node.K3sNodeLabels == nil {
+				node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized
 			}
-			k, v := util.SplitLabelKeyValue(labelWithNodeFilters.Label)
-			node.Labels[k] = v
+			k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label)
+			node.K3sNodeLabels[k] = v
+
+		}
+	}
+
+	// -> RUNTIME LABELS
+	for _, runtimeLabelWithNodeFilters := range simpleConfig.RuntimeLabels {
+		if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
+			return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label)
+		}
+
+		nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, node := range nodes {
+			if node.RuntimeLabels == nil {
+				node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized
+			}
+			k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label)
+			node.RuntimeLabels[k] = v
 		}
 	}
 

pkg/config/v1alpha2/schema.json

@@ -108,7 +108,22 @@
         "additionalProperties": false
       }
     },
-    "labels": {
+    "k3sNodeLabels": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "label": {
+            "type": "string"
+          },
+          "nodeFilters": {  
+            "$ref": "#/definitions/nodeFilters"
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "runtimeLabels": {
       "type": "array",
       "items": {
         "type": "object",

pkg/config/v1alpha2/types.go

@@ -119,21 +119,22 @@ type SimpleConfigOptionsK3s struct {
 
 // SimpleConfig describes the toplevel k3d configuration file.
 type SimpleConfig struct {
-	TypeMeta     `mapstructure:",squash" yaml:",inline"`
-	Name         string                  `mapstructure:"name" yaml:"name" json:"name,omitempty"`
-	Servers      int                     `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll    // default 1
-	Agents       int                     `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"`    //nolint:lll    // default 0
-	ExposeAPI    SimpleExposureOpts      `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
-	Image        string                  `mapstructure:"image" yaml:"image" json:"image,omitempty"`
-	Network      string                  `mapstructure:"network" yaml:"network" json:"network,omitempty"`
-	Subnet       string                  `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
-	ClusterToken string                  `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
-	Volumes      []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
-	Ports        []PortWithNodeFilters   `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
-	Labels       []LabelWithNodeFilters  `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"`
-	Options      SimpleConfigOptions     `mapstructure:"options" yaml:"options" json:"options,omitempty"`
-	Env          []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
-	Registries   struct {
+	TypeMeta      `mapstructure:",squash" yaml:",inline"`
+	Name          string                  `mapstructure:"name" yaml:"name" json:"name,omitempty"`
+	Servers       int                     `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll    // default 1
+	Agents        int                     `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"`    //nolint:lll    // default 0
+	ExposeAPI     SimpleExposureOpts      `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
+	Image         string                  `mapstructure:"image" yaml:"image" json:"image,omitempty"`
+	Network       string                  `mapstructure:"network" yaml:"network" json:"network,omitempty"`
+	Subnet        string                  `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
+	ClusterToken  string                  `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
+	Volumes       []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
+	Ports         []PortWithNodeFilters   `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
+	K3sNodeLabels []LabelWithNodeFilters  `mapstructure:"k3sNodeLabels" yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"`
+	RuntimeLabels []LabelWithNodeFilters  `mapstructure:"runtimeLabels" yaml:"runtimeLabels" json:"runtimeLabels,omitempty"`
+	Options       SimpleConfigOptions     `mapstructure:"options" yaml:"options" json:"options,omitempty"`
+	Env           []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
+	Registries    struct {
 		Use    []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
 		Create bool     `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
 		Config string   `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)

pkg/types/types.go

@@ -341,6 +341,7 @@ type Node struct {
 	Restart       bool              `yaml:"restart" json:"restart,omitempty"`
 	Created       string            `yaml:"created" json:"created,omitempty"`
 	Labels        map[string]string // filled automatically
+	RuntimeLabels map[string]string `yaml:"runtimeLabels" json:"runtimeLabels,omitempty"`
 	K3sNodeLabels map[string]string `yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"`
 	Networks      []string          // filled automatically
 	ExtraHosts    []string          // filled automatically

tests/assets/config_test_simple.yaml

@@ -22,7 +22,12 @@ env:
   - envVar: bar=baz,bob
     nodeFilters:
       - all
-labels:
+k3sNodeLabels:
+  - label: foo=bar
+    nodeFilters:
+      - server[0]
+      - loadbalancer
+runtimeLabels:
   - label: foo=bar
     nodeFilters:
       - server[0]
@@ -48,4 +53,4 @@ options:
     extraAgentArgs: []
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true