From 9df1dfdffc3ae5cb131369a51e1f6ebe032aef74 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 15 Apr 2021 17:14:05 -0700 Subject: [PATCH] Add support for multiple CIDRs/ClusterDNS addresses Signed-off-by: Brad Davidson --- go.mod | 1 + pkg/agent/config/config.go | 64 +++++++++++-- pkg/agent/netpol/network_policy_controller.go | 2 +- .../netpol/network_policy_controller_test.go | 10 +- pkg/agent/syssetup/setup.go | 1 + pkg/cli/cmds/server.go | 33 +++---- pkg/cli/server/server.go | 92 ++++++++++++++++--- pkg/daemons/agent/agent.go | 5 +- pkg/daemons/config/types.go | 10 +- pkg/daemons/control/server.go | 7 +- pkg/server/server.go | 4 +- pkg/util/net.go | 22 +++++ vendor/modules.txt | 1 + 13 files changed, 199 insertions(+), 53 deletions(-) create mode 100644 pkg/util/net.go diff --git a/go.mod b/go.mod index 44b59e8d2419..11dd32d9494b 100644 --- a/go.mod +++ b/go.mod @@ -122,5 +122,6 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubectl v0.21.0 k8s.io/kubernetes v1.21.0 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 sigs.k8s.io/yaml v1.2.0 ) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 50ba532c30e9..a6c35b962204 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -31,6 +31,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/net" + utilsnet "k8s.io/utils/net" ) const ( @@ -488,27 +489,35 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } if controlConfig.ClusterIPRange != nil { - nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange + nodeConfig.AgentConfig.ClusterCIDR = controlConfig.ClusterIPRange + nodeConfig.AgentConfig.ClusterCIDRs = []*sysnet.IPNet{controlConfig.ClusterIPRange} + } + + if len(controlConfig.ClusterIPRanges) > 0 { + nodeConfig.AgentConfig.ClusterCIDRs = controlConfig.ClusterIPRanges } if controlConfig.ServiceIPRange != nil { - nodeConfig.AgentConfig.ServiceCIDR = *controlConfig.ServiceIPRange + nodeConfig.AgentConfig.ServiceCIDR = controlConfig.ServiceIPRange + nodeConfig.AgentConfig.ServiceCIDRs = []*sysnet.IPNet{controlConfig.ServiceIPRange} + } + + if len(controlConfig.ServiceIPRanges) > 0 { + nodeConfig.AgentConfig.ServiceCIDRs = controlConfig.ServiceIPRanges } if controlConfig.ServiceNodePortRange != nil { nodeConfig.AgentConfig.ServiceNodePortRange = *controlConfig.ServiceNodePortRange } - // Old versions of the server do not send enough information to correctly start the NPC. Users - // need to upgrade the server to at least the same version as the agent, or disable the NPC - // cluster-wide. - if controlConfig.DisableNPC == false && (controlConfig.ServiceIPRange == nil || controlConfig.ServiceNodePortRange == nil) { - return nil, fmt.Errorf("incompatible down-level server detected; servers must be upgraded to at least %s, or restarted with --disable-network-policy", version.Version) + if len(controlConfig.ClusterDNSs) == 0 { + nodeConfig.AgentConfig.ClusterDNSs = []sysnet.IP{controlConfig.ClusterDNS} + } else { + nodeConfig.AgentConfig.ClusterDNSs = controlConfig.ClusterDNSs } nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs - nodeConfig.AgentConfig.NodeTaints = envInfo.Taints nodeConfig.AgentConfig.NodeLabels = envInfo.Labels nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry @@ -520,6 +529,10 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.PodManifests = filepath.Join(envInfo.DataDir, "agent", DefaultPodManifestPath) nodeConfig.AgentConfig.ProtectKernelDefaults = envInfo.ProtectKernelDefaults + if err := validateNetworkConfig(nodeConfig); err != nil { + return nil, err + } + return nodeConfig, nil } @@ -532,3 +545,38 @@ func getConfig(info *clientaccess.Info) (*config.Control, error) { controlControl := &config.Control{} return controlControl, json.Unmarshal(data, controlControl) } + +// validateNetworkConfig ensures that the network configuration values provided by the server make sense. +func validateNetworkConfig(nodeConfig *config.Node) error { + // Old versions of the server do not send enough information to correctly start the NPC. Users + // need to upgrade the server to at least the same version as the agent, or disable the NPC + // cluster-wide. + if nodeConfig.AgentConfig.DisableNPC == false && (nodeConfig.AgentConfig.ServiceCIDR == nil || nodeConfig.AgentConfig.ServiceNodePortRange.Size == 0) { + return fmt.Errorf("incompatible down-level server detected; servers must be upgraded to at least %s, or restarted with --disable-network-policy", version.Version) + } + + // Dual-stack operation requires fairly extensive manual configuration at the moment - do some + // preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to + // enable dual-stack DNS on a single-stack cluster. + dualCluster, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ClusterCIDRs) + if err != nil { + return errors.Wrap(err, "failed to validate cluster-cidr") + } + dualService, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ServiceCIDRs) + if err != nil { + return errors.Wrap(err, "failed to validate service-cidr") + } + dualDNS, err := utilsnet.IsDualStackIPs(nodeConfig.AgentConfig.ClusterDNSs) + if err != nil { + return errors.Wrap(err, "failed to validate cluster-dns") + } + + if (nodeConfig.AgentConfig.DisableNPC == false || nodeConfig.NoFlannel == false) && (dualCluster || dualService) { + return errors.New("flannel CNI and network policy enforcement are not compatible with dual-stack operation; servers must be restarted with --flannel-backend=none --disable-network-policy and an alternative CNI plugin deployed") + } + if dualDNS == true && (dualCluster == false || dualService == false) { + return errors.New("dual-stack cluster-dns requires dual-stack cluster-cidr and service-cidr") + } + + return nil +} diff --git a/pkg/agent/netpol/network_policy_controller.go b/pkg/agent/netpol/network_policy_controller.go index 1859f41f6533..9cdb57ab0ba5 100644 --- a/pkg/agent/netpol/network_policy_controller.go +++ b/pkg/agent/netpol/network_policy_controller.go @@ -592,7 +592,7 @@ func NewNetworkPolicyController(clientset kubernetes.Interface, // be up to date with all of the policy changes from any enqueued request after that npc.fullSyncRequestChan = make(chan struct{}, 1) - npc.serviceClusterIPRange = config.AgentConfig.ServiceCIDR + npc.serviceClusterIPRange = *config.AgentConfig.ServiceCIDR npc.serviceNodePortRange = strings.ReplaceAll(config.AgentConfig.ServiceNodePortRange.String(), "-", ":") npc.syncPeriod = defaultSyncPeriod diff --git a/pkg/agent/netpol/network_policy_controller_test.go b/pkg/agent/netpol/network_policy_controller_test.go index d141e3c80bad..d6bde79b3866 100644 --- a/pkg/agent/netpol/network_policy_controller_test.go +++ b/pkg/agent/netpol/network_policy_controller_test.go @@ -253,15 +253,17 @@ func testForMissingOrUnwanted(t *testing.T, targetMsg string, got []podInfo, wan } } -func newMinimalNodeConfig(clusterIPCIDR string, nodePortRange string, hostNameOverride string, externalIPs []string) *config.Node { +func newMinimalNodeConfig(serviceIPCIDR string, nodePortRange string, hostNameOverride string, externalIPs []string) *config.Node { nodeConfig := &config.Node{AgentConfig: config.Agent{}} - if clusterIPCIDR != "" { - _, cidr, err := net.ParseCIDR(clusterIPCIDR) + if serviceIPCIDR != "" { + _, cidr, err := net.ParseCIDR(serviceIPCIDR) if err != nil { panic("failed to get parse --service-cluster-ip-range parameter: " + err.Error()) } - nodeConfig.AgentConfig.ClusterCIDR = *cidr + nodeConfig.AgentConfig.ServiceCIDR = cidr + } else { + nodeConfig.AgentConfig.ServiceCIDR = &net.IPNet{} } if nodePortRange != "" { portRange, err := utilnet.ParsePortRange(nodePortRange) diff --git a/pkg/agent/syssetup/setup.go b/pkg/agent/syssetup/setup.go index d19310fdbc26..599a30f0c210 100644 --- a/pkg/agent/syssetup/setup.go +++ b/pkg/agent/syssetup/setup.go @@ -32,6 +32,7 @@ func Configure() { loadKernelModule("nf_conntrack") loadKernelModule("br_netfilter") loadKernelModule("iptable_nat") + loadKernelModule("ip6table_nat") // Kernel is inconsistent about how devconf is configured for // new network namespaces between ipv4 and ipv6. Make sure to diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 42bf4fc10f0d..ba0809f9ec28 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -13,15 +13,15 @@ const ( ) type Server struct { - ClusterCIDR string + ClusterCIDR cli.StringSlice AgentToken string AgentTokenFile string Token string TokenFile string ClusterSecret string - ServiceCIDR string + ServiceCIDR cli.StringSlice ServiceNodePortRange string - ClusterDNS string + ClusterDNS cli.StringSlice ClusterDomain string // The port which kubectl clients can access k8s HTTPSPort int @@ -126,17 +126,15 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", Destination: &ServerConfig.DataDir, }, - cli.StringFlag{ - Name: "cluster-cidr", - Usage: "(networking) Network CIDR to use for pod IPs", - Destination: &ServerConfig.ClusterCIDR, - Value: "10.42.0.0/16", + cli.StringSliceFlag{ + Name: "cluster-cidr", + Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)", + Value: &ServerConfig.ClusterCIDR, }, - cli.StringFlag{ - Name: "service-cidr", - Usage: "(networking) Network CIDR to use for services IPs", - Destination: &ServerConfig.ServiceCIDR, - Value: "10.43.0.0/16", + cli.StringSliceFlag{ + Name: "service-cidr", + Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)", + Value: &ServerConfig.ServiceCIDR, }, cli.StringFlag{ Name: "service-node-port-range", @@ -144,11 +142,10 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.ServiceNodePortRange, Value: "30000-32767", }, - cli.StringFlag{ - Name: "cluster-dns", - Usage: "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", - Destination: &ServerConfig.ClusterDNS, - Value: "", + cli.StringSliceFlag{ + Name: "cluster-dns", + Usage: "(networking) IPv4/IPv6 Cluster IPs for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", + Value: &ServerConfig.ClusterDNS, }, cli.StringFlag{ Name: "cluster-domain", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index f9c7f2009b6d..28886b17a62d 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -190,18 +190,57 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP) } - _, serverConfig.ControlConfig.ClusterIPRange, err = net.ParseCIDR(cfg.ClusterCIDR) - if err != nil { - return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ClusterCIDR, err) + if len(cmds.ServerConfig.ClusterCIDR) == 0 { + cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16") + } + for _, cidr := range cmds.ServerConfig.ClusterCIDR { + for _, v := range strings.Split(cidr, ",") { + _, parsed, err := net.ParseCIDR(v) + if err != nil { + return errors.Wrapf(err, "invalid cluster-cidr %s", v) + } + serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed) + } } - _, serverConfig.ControlConfig.ServiceIPRange, err = net.ParseCIDR(cfg.ServiceCIDR) - if err != nil { - return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err) + + // Set ClusterIPRange to the first IPv4 block, for legacy clients + for _, ipRange := range serverConfig.ControlConfig.ClusterIPRanges { + if ipRange.IP.To4() != nil { + serverConfig.ControlConfig.ClusterIPRange = ipRange + break + } + } + if serverConfig.ControlConfig.ClusterIPRange == nil { + return errors.New("cannot configure legacy cluster-cidr: no IPv4 cluster-cidr ranges found") + } + + if len(cmds.ServerConfig.ServiceCIDR) == 0 { + cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16") + } + for _, cidr := range cmds.ServerConfig.ServiceCIDR { + for _, v := range strings.Split(cidr, ",") { + _, parsed, err := net.ParseCIDR(v) + if err != nil { + return errors.Wrapf(err, "invalid service-cidr %s", v) + } + serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed) + } + } + + // Set ServiceIPRange to the first IPv4 block, for legacy clients + for _, ipRange := range serverConfig.ControlConfig.ServiceIPRanges { + if ipRange.IP.To4() != nil { + serverConfig.ControlConfig.ServiceIPRange = ipRange + break + } + } + if serverConfig.ControlConfig.ServiceIPRange == nil { + return errors.New("cannot configure legacy service-cidr: no IPv4 service-cidr ranges found") } serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange) if err != nil { - return errors.Wrapf(err, "Invalid port range %s: %v", cfg.ServiceNodePortRange, err) + return errors.Wrapf(err, "invalid port range %s", cfg.ServiceNodePortRange) } _, apiServerServiceIP, err := controlplane.ServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange) @@ -210,14 +249,41 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String()) - // If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10, + // If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 - if cfg.ClusterDNS == "" { - serverConfig.ControlConfig.ClusterDNS = make(net.IP, 4) - copy(serverConfig.ControlConfig.ClusterDNS, serverConfig.ControlConfig.ServiceIPRange.IP.To4()) - serverConfig.ControlConfig.ClusterDNS[3] = 10 + // If there are no IPv4 ServiceCIDRs, an error will be raised. + if len(cmds.ServerConfig.ClusterDNS) == 0 { + for _, ipRange := range serverConfig.ControlConfig.ServiceIPRanges { + if ipRange.IP.To4() != nil { + serverConfig.ControlConfig.ClusterDNS = ipRange.IP.To4() + serverConfig.ControlConfig.ClusterDNS[3] = 10 + serverConfig.ControlConfig.ClusterDNSs = []net.IP{serverConfig.ControlConfig.ClusterDNS} + break + } + } + if serverConfig.ControlConfig.ClusterDNS == nil { + return errors.New("cannot autoconfigure cluster-dns address: no IPv4 service-cidr ranges found") + } } else { - serverConfig.ControlConfig.ClusterDNS = net.ParseIP(cfg.ClusterDNS) + for _, ip := range cmds.ServerConfig.ClusterDNS { + for _, v := range strings.Split(ip, ",") { + parsed := net.ParseIP(v) + if parsed == nil { + return fmt.Errorf("invalid cluster-dns address %s", v) + } + serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, parsed) + } + } + // Set ClusterDNS to the first IPv4 address, for legacy clients + for _, ip := range serverConfig.ControlConfig.ClusterDNSs { + if ip.To4() != nil { + serverConfig.ControlConfig.ClusterDNS = ip + break + } + } + if serverConfig.ControlConfig.ClusterDNS == nil { + return errors.New("cannot configure legacy cluster-dns address: no IPv4 cluster-dns addresses found") + } } if cfg.DefaultLocalStoragePath == "" { diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 6ac0ec57a3bb..d968f1a6f52f 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -13,6 +13,7 @@ import ( "github.com/opencontainers/runc/libcontainer/system" "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/executor" + "github.com/rancher/k3s/pkg/util" "github.com/rancher/k3s/pkg/version" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -47,7 +48,7 @@ func startKubeProxy(cfg *config.Agent) error { "proxy-mode": "iptables", "healthz-bind-address": "127.0.0.1", "kubeconfig": cfg.KubeConfigKubeProxy, - "cluster-cidr": cfg.ClusterCIDR.String(), + "cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs), } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName @@ -94,7 +95,7 @@ func startKubelet(cfg *config.Agent) error { argsMap["network-plugin"] = "cni" } if len(cfg.ClusterDNS) > 0 { - argsMap["cluster-dns"] = cfg.ClusterDNS.String() + argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs) } if cfg.ResolvConf != "" { argsMap["resolv-conf"] = cfg.ResolvConf diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index eb1a2f34b708..630e61b54c31 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -58,10 +58,13 @@ type Agent struct { NodeConfigPath string ServingKubeletCert string ServingKubeletKey string - ServiceCIDR net.IPNet + ServiceCIDR *net.IPNet + ServiceCIDRs []*net.IPNet ServiceNodePortRange utilnet.PortRange - ClusterCIDR net.IPNet + ClusterCIDR *net.IPNet + ClusterCIDRs []*net.IPNet ClusterDNS net.IP + ClusterDNSs []net.IP ClusterDomain string ResolvConf string RootDir string @@ -106,9 +109,12 @@ type Control struct { AgentToken string `json:"-"` Token string `json:"-"` ClusterIPRange *net.IPNet + ClusterIPRanges []*net.IPNet ServiceIPRange *net.IPNet + ServiceIPRanges []*net.IPNet ServiceNodePortRange *utilnet.PortRange ClusterDNS net.IP + ClusterDNSs []net.IP ClusterDomain string NoCoreDNS bool KubeConfigOutput string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index e20e8955379b..f3067dbc4830 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -16,6 +16,7 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/control/deps" "github.com/rancher/k3s/pkg/daemons/executor" + util2 "github.com/rancher/k3s/pkg/util" "github.com/rancher/k3s/pkg/version" "github.com/rancher/wrangler-api/pkg/generated/controllers/rbac" "github.com/sirupsen/logrus" @@ -100,7 +101,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) erro "kubeconfig": runtime.KubeConfigController, "service-account-private-key-file": runtime.ServiceKey, "allocate-node-cidrs": "true", - "cluster-cidr": cfg.ClusterIPRange.String(), + "cluster-cidr": util2.JoinIPNets(cfg.ClusterIPRanges), "root-ca-file": runtime.ServerCA, "port": "10252", "profiling": "false", @@ -155,7 +156,7 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["allow-privileged"] = "true" argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") argsMap["service-account-signing-key-file"] = runtime.ServiceKey - argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String() + argsMap["service-cluster-ip-range"] = util2.JoinIPNets(cfg.ServiceIPRanges) argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) if cfg.AdvertiseIP != "" { @@ -360,7 +361,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *c ccmOptions.KubeCloudShared.AllocateNodeCIDRs = true ccmOptions.KubeCloudShared.CloudProvider.Name = version.Program - ccmOptions.KubeCloudShared.ClusterCIDR = cfg.ClusterIPRange.String() + ccmOptions.KubeCloudShared.ClusterCIDR = util2.JoinIPNets(cfg.ClusterIPRanges) ccmOptions.KubeCloudShared.ConfigureCloudRoutes = false ccmOptions.Kubeconfig = runtime.KubeConfigCloudController ccmOptions.NodeStatusUpdateFrequency = metav1.Duration{Duration: 1 * time.Minute} diff --git a/pkg/server/server.go b/pkg/server/server.go index 4aa34c78fd4c..84638450f3a0 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -435,8 +435,8 @@ func setNoProxyEnv(config *config.Control) error { envList = append(envList, ".svc", "."+config.ClusterDomain, - config.ClusterIPRange.String(), - config.ServiceIPRange.String(), + util.JoinIPNets(config.ClusterIPRanges), + util.JoinIPNets(config.ServiceIPRanges), ) os.Unsetenv("no_proxy") return os.Setenv("NO_PROXY", strings.Join(envList, ",")) diff --git a/pkg/util/net.go b/pkg/util/net.go new file mode 100644 index 000000000000..b73529107141 --- /dev/null +++ b/pkg/util/net.go @@ -0,0 +1,22 @@ +package util + +import ( + "net" + "strings" +) + +func JoinIPs(elems []net.IP) string { + var strs []string + for _, elem := range elems { + strs = append(strs, elem.String()) + } + return strings.Join(strs, ",") +} + +func JoinIPNets(elems []*net.IPNet) string { + var strs []string + for _, elem := range elems { + strs = append(strs, elem.String()) + } + return strings.Join(strs, ",") +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 26ca61f0a017..a13589abcb69 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -3053,6 +3053,7 @@ k8s.io/metrics/pkg/client/external_metrics # k8s.io/mount-utils v0.0.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.21.0-k3s1 k8s.io/mount-utils # k8s.io/utils v0.0.0-20201110183641-67b214c5f920 +## explicit k8s.io/utils/buffer k8s.io/utils/clock k8s.io/utils/exec