From 334eae119ae6ac56a71e286bf65240dfc763fa61 Mon Sep 17 00:00:00 2001 From: Chris Kim <30601846+Oats87@users.noreply.github.com> Date: Thu, 11 Nov 2021 20:36:17 -0800 Subject: [PATCH] [release-1.21] Add etcd extra args support for K3s (#4471) * Export cli server flags and etcd restoration functions (#3527) * Export cli server flags and etfd restoration functions Signed-off-by: galal-hussein * export S3 Signed-off-by: galal-hussein Signed-off-by: Chris Kim * Add etcd extra args support for K3s Signed-off-by: Chris Kim * Remove integration test Signed-off-by: Chris Kim Co-authored-by: Hussein Galal --- pkg/cli/cmds/server.go | 103 ++++++++++++++++++------------- pkg/cli/server/server.go | 1 + pkg/cluster/bootstrap.go | 2 +- pkg/daemons/config/types.go | 1 + pkg/daemons/executor/etcd.go | 4 +- pkg/daemons/executor/executor.go | 55 +++++++++++++++-- pkg/etcd/etcd.go | 6 +- pkg/etcd/s3.go | 4 +- 8 files changed, 119 insertions(+), 57 deletions(-) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 1214923d5feb..5a5d39bee6df 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -38,6 +38,7 @@ type Server struct { TLSSan cli.StringSlice BindAddress string ExtraAPIArgs cli.StringSlice + ExtraEtcdArgs cli.StringSlice ExtraSchedulerArgs cli.StringSlice ExtraControllerArgs cli.StringSlice ExtraCloudControllerArgs cli.StringSlice @@ -83,7 +84,56 @@ type Server struct { EtcdS3Insecure bool } -var ServerConfig Server +var ( + ServerConfig Server + ClusterCIDR = cli.StringSliceFlag{ + Name: "cluster-cidr", + Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)", + Value: &ServerConfig.ClusterCIDR, + } + ServiceCIDR = cli.StringSliceFlag{ + Name: "service-cidr", + Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)", + Value: &ServerConfig.ServiceCIDR, + } + ServiceNodePortRange = cli.StringFlag{ + Name: "service-node-port-range", + Usage: "(networking) Port range to reserve for services with NodePort visibility", + Destination: &ServerConfig.ServiceNodePortRange, + Value: "30000-32767", + } + ClusterDNS = cli.StringSliceFlag{ + Name: "cluster-dns", + Usage: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", + Value: &ServerConfig.ClusterDNS, + } + ClusterDomain = cli.StringFlag{ + Name: "cluster-domain", + Usage: "(networking) Cluster Domain", + Destination: &ServerConfig.ClusterDomain, + Value: "cluster.local", + } + ExtraAPIArgs = cli.StringSliceFlag{ + Name: "kube-apiserver-arg", + Usage: "(flags) Customized flag for kube-apiserver process", + Value: &ServerConfig.ExtraAPIArgs, + } + ExtraEtcdArgs = cli.StringSliceFlag{ + Name: "etcd-arg", + Usage: "(flags) Customized flag for etcd process", + Value: &ServerConfig.ExtraEtcdArgs, + } + ExtraSchedulerArgs = cli.StringSliceFlag{ + Name: "kube-scheduler-arg", + Usage: "(flags) Customized flag for kube-scheduler process", + Value: &ServerConfig.ExtraSchedulerArgs, + } + ExtraControllerArgs = cli.StringSliceFlag{ + Name: "kube-controller-manager-arg", + Usage: "(flags) Customized flag for kube-controller-manager process", + Value: &ServerConfig.ExtraControllerArgs, + } +) func NewServerCommand(action func(*cli.Context) error) cli.Command { return cli.Command{ @@ -130,33 +180,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", Destination: &ServerConfig.DataDir, }, - cli.StringSliceFlag{ - Name: "cluster-cidr", - Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)", - Value: &ServerConfig.ClusterCIDR, - }, - cli.StringSliceFlag{ - Name: "service-cidr", - Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)", - Value: &ServerConfig.ServiceCIDR, - }, - cli.StringFlag{ - Name: "service-node-port-range", - Usage: "(networking) Port range to reserve for services with NodePort visibility", - Destination: &ServerConfig.ServiceNodePortRange, - Value: "30000-32767", - }, - cli.StringSliceFlag{ - Name: "cluster-dns", - Usage: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", - Value: &ServerConfig.ClusterDNS, - }, - cli.StringFlag{ - Name: "cluster-domain", - Usage: "(networking) Cluster Domain", - Destination: &ServerConfig.ClusterDomain, - Value: "cluster.local", - }, + ClusterCIDR, + ServiceCIDR, + ServiceNodePortRange, + ClusterDNS, + ClusterDomain, cli.StringFlag{ Name: "flannel-backend", Usage: "(networking) One of 'none', 'vxlan', 'ipsec', 'host-gw', or 'wireguard'", @@ -187,21 +215,10 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.KubeConfigMode, EnvVar: version.ProgramUpper + "_KUBECONFIG_MODE", }, - cli.StringSliceFlag{ - Name: "kube-apiserver-arg", - Usage: "(flags) Customized flag for kube-apiserver process", - Value: &ServerConfig.ExtraAPIArgs, - }, - cli.StringSliceFlag{ - Name: "kube-scheduler-arg", - Usage: "(flags) Customized flag for kube-scheduler process", - Value: &ServerConfig.ExtraSchedulerArgs, - }, - cli.StringSliceFlag{ - Name: "kube-controller-manager-arg", - Usage: "(flags) Customized flag for kube-controller-manager process", - Value: &ServerConfig.ExtraControllerArgs, - }, + ExtraAPIArgs, + ExtraEtcdArgs, + ExtraControllerArgs, + ExtraSchedulerArgs, cli.StringSliceFlag{ Name: "kube-cloud-controller-manager-arg", Usage: "(flags) Customized flag for kube-cloud-controller-manager process", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index ab8d32469a3e..092a06ae4762 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -117,6 +117,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs + serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 0c6339167eb7..e0a11a12e51a 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -74,7 +74,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error { ElectionTimeout: 5000, LogOutputs: []string{"stderr"}, } - configFile, err := args.ToConfigFile() + configFile, err := args.ToConfigFile(c.config.ExtraEtcdArgs) if err != nil { return err } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index c15d33d4b61d..f9546e137edd 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -132,6 +132,7 @@ type Control struct { ExtraAPIArgs []string ExtraControllerArgs []string ExtraCloudControllerArgs []string + ExtraEtcdArgs []string ExtraSchedulerAPIArgs []string NoLeaderElect bool JoinURL string diff --git a/pkg/daemons/executor/etcd.go b/pkg/daemons/executor/etcd.go index a22b957b6b66..ebda9ea608c8 100644 --- a/pkg/daemons/executor/etcd.go +++ b/pkg/daemons/executor/etcd.go @@ -18,8 +18,8 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) { return InitialOptions{}, nil } -func (e Embedded) ETCD(args ETCDConfig) error { - configFile, err := args.ToConfigFile() +func (e Embedded) ETCD(args ETCDConfig, extraArgs []string) error { + configFile, err := args.ToConfigFile(extraArgs) if err != nil { return err } diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index 1b4004dccca0..117da38859de 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -6,12 +6,15 @@ import ( "net/http" "os" "path/filepath" - - "sigs.k8s.io/yaml" + "strconv" + "strings" + "time" "github.com/rancher/k3s/pkg/cli/cmds" daemonconfig "github.com/rancher/k3s/pkg/daemons/config" + yaml2 "gopkg.in/yaml.v2" "k8s.io/apiserver/pkg/authentication/authenticator" + "sigs.k8s.io/yaml" ) var ( @@ -27,7 +30,7 @@ type Executor interface { Scheduler(apiReady <-chan struct{}, args []string) error ControllerManager(apiReady <-chan struct{}, args []string) error CurrentETCDOptions() (InitialOptions, error) - ETCD(args ETCDConfig) error + ETCD(args ETCDConfig, extraArgs []string) error CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error } @@ -69,13 +72,53 @@ type InitialOptions struct { State string `json:"initial-cluster-state,omitempty"` } -func (e ETCDConfig) ToConfigFile() (string, error) { +func (e ETCDConfig) ToConfigFile(extraArgs []string) (string, error) { confFile := filepath.Join(e.DataDir, "config") bytes, err := yaml.Marshal(&e) if err != nil { return "", err } + if len(extraArgs) > 0 { + var s map[string]interface{} + if err := yaml2.Unmarshal(bytes, &s); err != nil { + return "", err + } + + for _, v := range extraArgs { + extraArg := strings.SplitN(v, "=", 2) + // Depending on the argV, we have different types to handle. + // Source: https://github.com/etcd-io/etcd/blob/44b8ae145b505811775f5af915dd19198d556d55/server/config/config.go#L36-L190 and https://etcd.io/docs/v3.5/op-guide/configuration/#configuration-file + if len(extraArg) == 2 { + key := strings.TrimLeft(extraArg[0], "-") + lowerKey := strings.ToLower(key) + var stringArr []string + if i, err := strconv.Atoi(extraArg[1]); err == nil { + s[key] = i + } else if time, err := time.ParseDuration(extraArg[1]); err == nil && (strings.Contains(lowerKey, "time") || strings.Contains(lowerKey, "duration") || strings.Contains(lowerKey, "interval") || strings.Contains(lowerKey, "retention")) { + // auto-compaction-retention is either a time.Duration or int, depending on version. If it is an int, it will be caught above. + s[key] = time + } else if err := yaml.Unmarshal([]byte(extraArg[1]), &stringArr); err == nil { + s[key] = stringArr + } else { + switch strings.ToLower(extraArg[1]) { + case "true": + s[key] = true + case "false": + s[key] = false + default: + s[key] = extraArg[1] + } + } + } + } + + bytes, err = yaml2.Marshal(&s) + if err != nil { + return "", err + } + } + if err := os.MkdirAll(e.DataDir, 0700); err != nil { return "", err } @@ -118,8 +161,8 @@ func CurrentETCDOptions() (InitialOptions, error) { return executor.CurrentETCDOptions() } -func ETCD(args ETCDConfig) error { - return executor.ETCD(args) +func ETCD(args ETCDConfig, extraArgs []string) error { + return executor.ETCD(args, extraArgs) } func CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error { diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 536cf421ab1d..6d6745633ca0 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -237,7 +237,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error { return err } logrus.Infof("Retrieving etcd snapshot %s from S3", e.config.ClusterResetRestorePath) - if err := e.s3.download(ctx); err != nil { + if err := e.s3.Download(ctx); err != nil { return err } logrus.Infof("S3 download complete for %s", e.config.ClusterResetRestorePath) @@ -605,7 +605,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init HeartbeatInterval: 500, Logger: "zap", LogOutputs: []string{"stderr"}, - }) + }, e.config.ExtraEtcdArgs) } // RemovePeer removes a peer from the cluster. The peer name and IP address must both match. @@ -1024,7 +1024,7 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]Snapsho // if it hasn't yet been initialized. func (e *ETCD) initS3IfNil(ctx context.Context) error { if e.s3 == nil { - s3, err := newS3(ctx, e.config) + s3, err := NewS3(ctx, e.config) if err != nil { return err } diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 8cfbf2eee635..ad70c36cc389 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -31,7 +31,7 @@ type S3 struct { // newS3 creates a new value of type s3 pointer with a // copy of the config.Control pointer and initializes // a new Minio client. -func newS3(ctx context.Context, config *config.Control) (*S3, error) { +func NewS3(ctx context.Context, config *config.Control) (*S3, error) { tr := http.DefaultTransport switch { @@ -112,7 +112,7 @@ func (s *S3) upload(ctx context.Context, snapshot string) error { // download downloads the given snapshot from the configured S3 // compatible backend. -func (s *S3) download(ctx context.Context) error { +func (s *S3) Download(ctx context.Context) error { var remotePath string if s.config.EtcdS3Folder != "" { remotePath = filepath.Join(s.config.EtcdS3Folder, s.config.ClusterResetRestorePath)