From e34257fdbd49fec1923f7b177cfa88c0be9ea9e0 Mon Sep 17 00:00:00 2001 From: ejose19 <8742215+ejose19@users.noreply.github.com> Date: Mon, 10 May 2021 14:04:34 -0300 Subject: [PATCH] refactor: normalize flags (k3s node & runtime) --- cmd/cluster/clusterCreate.go | 55 +++++++++++++++---- cmd/node/nodeCreate.go | 23 +++++++- pkg/client/node.go | 11 ++-- pkg/config/config_test.go | 8 ++- .../test_assets/config_test_simple.yaml | 9 ++- .../config_test_simple_invalid_servers.yaml | 9 ++- pkg/config/transform.go | 39 ++++++++++--- pkg/config/v1alpha2/schema.json | 17 +++++- pkg/config/v1alpha2/types.go | 31 ++++++----- pkg/types/types.go | 1 + tests/assets/config_test_simple.yaml | 9 ++- 11 files changed, 162 insertions(+), 50 deletions(-) diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index bb44f20807..3f0fb38dd3 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -263,8 +263,11 @@ func NewCmdClusterCreate() *cobra.Command { cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`") _ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port")) - cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`") - _ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label")) + cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`") + _ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label")) + + cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`") + _ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label")) /****************** * "Normal" Flags * @@ -464,10 +467,38 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) { log.Tracef("PortFilterMap: %+v", portFilterMap) - // --label - // labelFilterMap will add container label to applied node filters - labelFilterMap := make(map[string][]string, 1) - for _, labelFlag := range ppViper.GetStringSlice("cli.labels") { + // --k3s-node-label + // k3sNodeLabelFilterMap will add k3s node label to applied node filters + k3sNodeLabelFilterMap := make(map[string][]string, 1) + for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") { + + // split node filter from the specified label + label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag) + if err != nil { + log.Fatalln(err) + } + + // create new entry or append filter to existing entry + if _, exists := k3sNodeLabelFilterMap[label]; exists { + k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...) + } else { + k3sNodeLabelFilterMap[label] = nodeFilters + } + } + + for label, nodeFilters := range k3sNodeLabelFilterMap { + cfg.K3sNodeLabels = append(cfg.K3sNodeLabels, conf.LabelWithNodeFilters{ + Label: label, + NodeFilters: nodeFilters, + }) + } + + log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap) + + // --runtime-label + // runtimeLabelFilterMap will add container runtime label to applied node filters + runtimeLabelFilterMap := make(map[string][]string, 1) + for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") { // split node filter from the specified label label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag) @@ -476,21 +507,21 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) { } // create new entry or append filter to existing entry - if _, exists := labelFilterMap[label]; exists { - labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...) + if _, exists := runtimeLabelFilterMap[label]; exists { + runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...) } else { - labelFilterMap[label] = nodeFilters + runtimeLabelFilterMap[label] = nodeFilters } } - for label, nodeFilters := range labelFilterMap { - cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{ + for label, nodeFilters := range runtimeLabelFilterMap { + cfg.RuntimeLabels = append(cfg.RuntimeLabels, conf.LabelWithNodeFilters{ Label: label, NodeFilters: nodeFilters, }) } - log.Tracef("LabelFilterMap: %+v", labelFilterMap) + log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap) // --env // envFilterMap will add container env vars to applied node filters diff --git a/cmd/node/nodeCreate.go b/cmd/node/nodeCreate.go index e30aac7874..6b0d9dfb97 100644 --- a/cmd/node/nodeCreate.go +++ b/cmd/node/nodeCreate.go @@ -74,6 +74,7 @@ func NewCmdNodeCreate() *cobra.Command { cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.") cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.") + cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"") cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"") // done @@ -127,9 +128,26 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl log.Errorf("Provided memory limit value is invalid") } + // --runtime-label + runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label") + if err != nil { + log.Errorln("No runtime-label specified") + log.Fatalln(err) + } + + runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)) + for _, label := range runtimeLabelsFlag { + labelSplitted := strings.Split(label, "=") + if len(labelSplitted) != 2 { + log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label) + } + runtimeLabels[labelSplitted[0]] = labelSplitted[1] + } + + // --k3s-node-label k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label") if err != nil { - log.Errorln("No node-label specified") + log.Errorln("No k3s-node-label specified") log.Fatalln(err) } @@ -137,7 +155,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl for _, label := range k3sNodeLabelsFlag { labelSplitted := strings.Split(label, "=") if len(labelSplitted) != 2 { - log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label) + log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label) } k3sNodeLabels[labelSplitted[0]] = labelSplitted[1] } @@ -153,6 +171,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl k3d.LabelRole: roleStr, }, K3sNodeLabels: k3sNodeLabels, + RuntimeLabels: runtimeLabels, Restart: true, Memory: memory, } diff --git a/pkg/client/node.go b/pkg/client/node.go index 25062ea53f..2ff38e56d4 100644 --- a/pkg/client/node.go +++ b/pkg/client/node.go @@ -315,10 +315,17 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c for k, v := range node.Labels { labels[k] = v } + for k, v := range node.RuntimeLabels { + labels[k] = v + } node.Labels = labels // second most important: the node role label node.Labels[k3d.LabelRole] = string(node.Role) + for k, v := range node.K3sNodeLabels { + node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v)) + } + // ### Environment ### node.Env = append(node.Env, k3d.DefaultNodeEnv...) // append default node env vars @@ -418,10 +425,6 @@ func patchAgentSpec(node *k3d.Node) error { node.Cmd = []string{"agent"} } - for k, v := range node.K3sNodeLabels { - node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v)) - } - return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index e1d215c575..a45445c873 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -63,7 +63,13 @@ func TestReadSimpleConfig(t *testing.T) { NodeFilters: []string{"loadbalancer"}, }, }, - Labels: []conf.LabelWithNodeFilters{ + K3sNodeLabels: []conf.LabelWithNodeFilters{ + { + Label: "foo=bar", + NodeFilters: []string{"server[0]", "loadbalancer"}, + }, + }, + RuntimeLabels: []conf.LabelWithNodeFilters{ { Label: "foo=bar", NodeFilters: []string{"server[0]", "loadbalancer"}, diff --git a/pkg/config/test_assets/config_test_simple.yaml b/pkg/config/test_assets/config_test_simple.yaml index e264bd9ae1..65a77acc95 100644 --- a/pkg/config/test_assets/config_test_simple.yaml +++ b/pkg/config/test_assets/config_test_simple.yaml @@ -22,7 +22,12 @@ env: - envVar: bar=baz nodeFilters: - all -labels: +k3sNodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer +runtimeLabels: - label: foo=bar nodeFilters: - server[0] @@ -40,4 +45,4 @@ options: extraAgentArgs: [] kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true diff --git a/pkg/config/test_assets/config_test_simple_invalid_servers.yaml b/pkg/config/test_assets/config_test_simple_invalid_servers.yaml index 7f2442c343..bf3ab69b24 100644 --- a/pkg/config/test_assets/config_test_simple_invalid_servers.yaml +++ b/pkg/config/test_assets/config_test_simple_invalid_servers.yaml @@ -22,7 +22,12 @@ env: - envVar: bar=baz nodeFilters: - all -labels: +k3sNodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer +runtimeLabels: - label: foo=bar nodeFilters: - server[0] @@ -40,4 +45,4 @@ options: extraAgentArgs: [] kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 6b7373f8bc..fc502fdb30 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -192,23 +192,44 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim } } - // -> LABELS - for _, labelWithNodeFilters := range simpleConfig.Labels { - if len(labelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { - return nil, fmt.Errorf("Labelmapping '%s' lacks a node filter, but there's more than one node", labelWithNodeFilters.Label) + // -> K3S NODE LABELS + for _, k3sNodelabelWithNodeFilters := range simpleConfig.K3sNodeLabels { + if len(k3sNodelabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { + return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodelabelWithNodeFilters.Label) } - nodes, err := util.FilterNodes(nodeList, labelWithNodeFilters.NodeFilters) + nodes, err := util.FilterNodes(nodeList, k3sNodelabelWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { - if node.Labels == nil { - node.Labels = make(map[string]string) // ensure that the map is initialized + if node.K3sNodeLabels == nil { + node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized } - k, v := util.SplitLabelKeyValue(labelWithNodeFilters.Label) - node.Labels[k] = v + k, v := util.SplitLabelKeyValue(k3sNodelabelWithNodeFilters.Label) + node.K3sNodeLabels[k] = v + + } + } + + // -> RUNTIME LABELS + for _, runtimeLabelWithNodeFilters := range simpleConfig.RuntimeLabels { + if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { + return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label) + } + + nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters) + if err != nil { + return nil, err + } + + for _, node := range nodes { + if node.RuntimeLabels == nil { + node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized + } + k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label) + node.RuntimeLabels[k] = v } } diff --git a/pkg/config/v1alpha2/schema.json b/pkg/config/v1alpha2/schema.json index 5e9e26c46f..15415652a4 100644 --- a/pkg/config/v1alpha2/schema.json +++ b/pkg/config/v1alpha2/schema.json @@ -108,7 +108,22 @@ "additionalProperties": false } }, - "labels": { + "k3sNodeLabels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "nodeFilters": { + "$ref": "#/definitions/nodeFilters" + } + }, + "additionalProperties": false + } + }, + "runtimeLabels": { "type": "array", "items": { "type": "object", diff --git a/pkg/config/v1alpha2/types.go b/pkg/config/v1alpha2/types.go index 967ccda5af..b57f6b7218 100644 --- a/pkg/config/v1alpha2/types.go +++ b/pkg/config/v1alpha2/types.go @@ -119,21 +119,22 @@ type SimpleConfigOptionsK3s struct { // SimpleConfig describes the toplevel k3d configuration file. type SimpleConfig struct { - TypeMeta `mapstructure:",squash" yaml:",inline"` - Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"` - Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1 - Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0 - ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"` - Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"` - Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"` - Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"` - ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated - Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"` - Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"` - Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"` - Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"` - Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"` - Registries struct { + TypeMeta `mapstructure:",squash" yaml:",inline"` + Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"` + Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1 + Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0 + ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"` + Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"` + Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"` + Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"` + ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated + Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"` + Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"` + K3sNodeLabels []LabelWithNodeFilters `mapstructure:"k3sNodeLabels" yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"` + RuntimeLabels []LabelWithNodeFilters `mapstructure:"runtimeLabels" yaml:"runtimeLabels" json:"runtimeLabels,omitempty"` + Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"` + Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"` + Registries struct { Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"` Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"` Config string `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override) diff --git a/pkg/types/types.go b/pkg/types/types.go index e916a80465..b0297118fb 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -341,6 +341,7 @@ type Node struct { Restart bool `yaml:"restart" json:"restart,omitempty"` Created string `yaml:"created" json:"created,omitempty"` Labels map[string]string // filled automatically + RuntimeLabels map[string]string `yaml:"runtimeLabels" json:"runtimeLabels,omitempty"` K3sNodeLabels map[string]string `yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"` Networks []string // filled automatically ExtraHosts []string // filled automatically diff --git a/tests/assets/config_test_simple.yaml b/tests/assets/config_test_simple.yaml index 4d9f725560..4a0755e90b 100755 --- a/tests/assets/config_test_simple.yaml +++ b/tests/assets/config_test_simple.yaml @@ -22,7 +22,12 @@ env: - envVar: bar=baz,bob nodeFilters: - all -labels: +k3sNodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer +runtimeLabels: - label: foo=bar nodeFilters: - server[0] @@ -48,4 +53,4 @@ options: extraAgentArgs: [] kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true