Add support for multiple CIDRs/ClusterDNS addresses

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

go.mod

@@ -122,5 +122,6 @@ require (
 	k8s.io/klog v1.0.0
 	k8s.io/kubectl v0.21.0
 	k8s.io/kubernetes v1.21.0
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 	sigs.k8s.io/yaml v1.2.0
 )

pkg/agent/config/config.go

@@ -31,6 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/net"
+	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -488,27 +489,35 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 
 	if controlConfig.ClusterIPRange != nil {
-		nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange
+		nodeConfig.AgentConfig.ClusterCIDR = controlConfig.ClusterIPRange
+		nodeConfig.AgentConfig.ClusterCIDRs = []*sysnet.IPNet{controlConfig.ClusterIPRange}
+	}
+
+	if len(controlConfig.ClusterIPRanges) > 0 {
+		nodeConfig.AgentConfig.ClusterCIDRs = controlConfig.ClusterIPRanges
 	}
 
 	if controlConfig.ServiceIPRange != nil {
-		nodeConfig.AgentConfig.ServiceCIDR = *controlConfig.ServiceIPRange
+		nodeConfig.AgentConfig.ServiceCIDR = controlConfig.ServiceIPRange
+		nodeConfig.AgentConfig.ServiceCIDRs = []*sysnet.IPNet{controlConfig.ServiceIPRange}
+	}
+
+	if len(controlConfig.ServiceIPRanges) > 0 {
+		nodeConfig.AgentConfig.ServiceCIDRs = controlConfig.ServiceIPRanges
 	}
 
 	if controlConfig.ServiceNodePortRange != nil {
 		nodeConfig.AgentConfig.ServiceNodePortRange = *controlConfig.ServiceNodePortRange
 	}
 
-	// Old versions of the server do not send enough information to correctly start the NPC. Users
-	// need to upgrade the server to at least the same version as the agent, or disable the NPC
-	// cluster-wide.
-	if controlConfig.DisableNPC == false && (controlConfig.ServiceIPRange == nil || controlConfig.ServiceNodePortRange == nil) {
-		return nil, fmt.Errorf("incompatible down-level server detected; servers must be upgraded to at least %s, or restarted with --disable-network-policy", version.Version)
+	if len(controlConfig.ClusterDNSs) == 0 {
+		nodeConfig.AgentConfig.ClusterDNSs = []sysnet.IP{controlConfig.ClusterDNS}
+	} else {
+		nodeConfig.AgentConfig.ClusterDNSs = controlConfig.ClusterDNSs
 	}
 
 	nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs
 	nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs
-
 	nodeConfig.AgentConfig.NodeTaints = envInfo.Taints
 	nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
 	nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry
@@ -520,6 +529,10 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.PodManifests = filepath.Join(envInfo.DataDir, "agent", DefaultPodManifestPath)
 	nodeConfig.AgentConfig.ProtectKernelDefaults = envInfo.ProtectKernelDefaults
 
+	if err := validateNetworkConfig(nodeConfig); err != nil {
+		return nil, err
+	}
+
 	return nodeConfig, nil
 }
 
@@ -532,3 +545,38 @@ func getConfig(info *clientaccess.Info) (*config.Control, error) {
 	controlControl := &config.Control{}
 	return controlControl, json.Unmarshal(data, controlControl)
 }
+
+// validateNetworkConfig ensures that the network configuration values provided by the server make sense.
+func validateNetworkConfig(nodeConfig *config.Node) error {
+	// Old versions of the server do not send enough information to correctly start the NPC. Users
+	// need to upgrade the server to at least the same version as the agent, or disable the NPC
+	// cluster-wide.
+	if nodeConfig.AgentConfig.DisableNPC == false && (nodeConfig.AgentConfig.ServiceCIDR == nil || nodeConfig.AgentConfig.ServiceNodePortRange.Size == 0) {
+		return fmt.Errorf("incompatible down-level server detected; servers must be upgraded to at least %s, or restarted with --disable-network-policy", version.Version)
+	}
+
+	// Dual-stack operation requires fairly extensive manual configuration at the moment - do some
+	// preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to
+	// enable dual-stack DNS on a single-stack cluster.
+	dualCluster, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ClusterCIDRs)
+	if err != nil {
+		return errors.Wrap(err, "failed to validate cluster-cidr")
+	}
+	dualService, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ServiceCIDRs)
+	if err != nil {
+		return errors.Wrap(err, "failed to validate service-cidr")
+	}
+	dualDNS, err := utilsnet.IsDualStackIPs(nodeConfig.AgentConfig.ClusterDNSs)
+	if err != nil {
+		return errors.Wrap(err, "failed to validate cluster-dns")
+	}
+
+	if (nodeConfig.AgentConfig.DisableNPC == false || nodeConfig.NoFlannel == false) && (dualCluster || dualService) {
+		return errors.New("flannel CNI and network policy enforcement are not compatible with dual-stack operation; servers must be restarted with --flannel-backend=none --disable-network-policy and an alternative CNI plugin deployed")
+	}
+	if dualDNS == true && (dualCluster == false || dualService == false) {
+		return errors.New("dual-stack cluster-dns requires dual-stack cluster-cidr and service-cidr")
+	}
+
+	return nil
+}

pkg/agent/netpol/network_policy_controller.go

@@ -592,7 +592,7 @@ func NewNetworkPolicyController(clientset kubernetes.Interface,
 	// be up to date with all of the policy changes from any enqueued request after that
 	npc.fullSyncRequestChan = make(chan struct{}, 1)
 
-	npc.serviceClusterIPRange = config.AgentConfig.ServiceCIDR
+	npc.serviceClusterIPRange = *config.AgentConfig.ServiceCIDR
 	npc.serviceNodePortRange = strings.ReplaceAll(config.AgentConfig.ServiceNodePortRange.String(), "-", ":")
 	npc.syncPeriod = defaultSyncPeriod
 

pkg/agent/netpol/network_policy_controller_test.go

@@ -253,15 +253,17 @@ func testForMissingOrUnwanted(t *testing.T, targetMsg string, got []podInfo, wan
 	}
 }
 
-func newMinimalNodeConfig(clusterIPCIDR string, nodePortRange string, hostNameOverride string, externalIPs []string) *config.Node {
+func newMinimalNodeConfig(serviceIPCIDR string, nodePortRange string, hostNameOverride string, externalIPs []string) *config.Node {
 	nodeConfig := &config.Node{AgentConfig: config.Agent{}}
 
-	if clusterIPCIDR != "" {
-		_, cidr, err := net.ParseCIDR(clusterIPCIDR)
+	if serviceIPCIDR != "" {
+		_, cidr, err := net.ParseCIDR(serviceIPCIDR)
 		if err != nil {
 			panic("failed to get parse --service-cluster-ip-range parameter: " + err.Error())
 		}
-		nodeConfig.AgentConfig.ClusterCIDR = *cidr
+		nodeConfig.AgentConfig.ServiceCIDR = cidr
+	} else {
+		nodeConfig.AgentConfig.ServiceCIDR = &net.IPNet{}
 	}
 	if nodePortRange != "" {
 		portRange, err := utilnet.ParsePortRange(nodePortRange)

pkg/agent/syssetup/setup.go

@@ -32,6 +32,7 @@ func Configure() {
 	loadKernelModule("nf_conntrack")
 	loadKernelModule("br_netfilter")
 	loadKernelModule("iptable_nat")
+	loadKernelModule("ip6table_nat")
 
 	// Kernel is inconsistent about how devconf is configured for
 	// new network namespaces between ipv4 and ipv6. Make sure to

pkg/cli/cmds/server.go

@@ -13,15 +13,15 @@ const (
 )
 
 type Server struct {
-	ClusterCIDR          string
+	ClusterCIDR          cli.StringSlice
 	AgentToken           string
 	AgentTokenFile       string
 	Token                string
 	TokenFile            string
 	ClusterSecret        string
-	ServiceCIDR          string
+	ServiceCIDR          cli.StringSlice
 	ServiceNodePortRange string
-	ClusterDNS           string
+	ClusterDNS           cli.StringSlice
 	ClusterDomain        string
 	// The port which kubectl clients can access k8s
 	HTTPSPort int
@@ -126,29 +126,26 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
 				Destination: &ServerConfig.DataDir,
 			},
-			cli.StringFlag{
-				Name:        "cluster-cidr",
-				Usage:       "(networking) Network CIDR to use for pod IPs",
-				Destination: &ServerConfig.ClusterCIDR,
-				Value:       "10.42.0.0/16",
+			cli.StringSliceFlag{
+				Name:  "cluster-cidr",
+				Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
+				Value: &ServerConfig.ClusterCIDR,
 			},
-			cli.StringFlag{
-				Name:        "service-cidr",
-				Usage:       "(networking) Network CIDR to use for services IPs",
-				Destination: &ServerConfig.ServiceCIDR,
-				Value:       "10.43.0.0/16",
+			cli.StringSliceFlag{
+				Name:  "service-cidr",
+				Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)",
+				Value: &ServerConfig.ServiceCIDR,
 			},
 			cli.StringFlag{
 				Name:        "service-node-port-range",
 				Usage:       "(networking) Port range to reserve for services with NodePort visibility",
 				Destination: &ServerConfig.ServiceNodePortRange,
 				Value:       "30000-32767",
 			},
-			cli.StringFlag{
-				Name:        "cluster-dns",
-				Usage:       "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
-				Destination: &ServerConfig.ClusterDNS,
-				Value:       "",
+			cli.StringSliceFlag{
+				Name:  "cluster-dns",
+				Usage: "(networking) IPv4/IPv6 Cluster IPs for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
+				Value: &ServerConfig.ClusterDNS,
 			},
 			cli.StringFlag{
 				Name:        "cluster-domain",

pkg/cli/server/server.go

@@ -190,18 +190,57 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
 	}
 
-	_, serverConfig.ControlConfig.ClusterIPRange, err = net.ParseCIDR(cfg.ClusterCIDR)
-	if err != nil {
-		return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ClusterCIDR, err)
+	if len(cmds.ServerConfig.ClusterCIDR) == 0 {
+		cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
+	}
+	for _, cidr := range cmds.ServerConfig.ClusterCIDR {
+		for _, v := range strings.Split(cidr, ",") {
+			_, parsed, err := net.ParseCIDR(v)
+			if err != nil {
+				return errors.Wrapf(err, "invalid cluster-cidr %s", v)
+			}
+			serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed)
+		}
 	}
-	_, serverConfig.ControlConfig.ServiceIPRange, err = net.ParseCIDR(cfg.ServiceCIDR)
-	if err != nil {
-		return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err)
+
+	// Set ClusterIPRange to the first IPv4 block, for legacy clients
+	for _, ipRange := range serverConfig.ControlConfig.ClusterIPRanges {
+		if ipRange.IP.To4() != nil {
+			serverConfig.ControlConfig.ClusterIPRange = ipRange
+			break
+		}
+	}
+	if serverConfig.ControlConfig.ClusterIPRange == nil {
+		return errors.New("cannot configure legacy cluster-cidr: no IPv4 cluster-cidr ranges found")
+	}
+
+	if len(cmds.ServerConfig.ServiceCIDR) == 0 {
+		cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
+	}
+	for _, cidr := range cmds.ServerConfig.ServiceCIDR {
+		for _, v := range strings.Split(cidr, ",") {
+			_, parsed, err := net.ParseCIDR(v)
+			if err != nil {
+				return errors.Wrapf(err, "invalid service-cidr %s", v)
+			}
+			serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed)
+		}
+	}
+
+	// Set ServiceIPRange to the first IPv4 block, for legacy clients
+	for _, ipRange := range serverConfig.ControlConfig.ServiceIPRanges {
+		if ipRange.IP.To4() != nil {
+			serverConfig.ControlConfig.ServiceIPRange = ipRange
+			break
+		}
+	}
+	if serverConfig.ControlConfig.ServiceIPRange == nil {
+		return errors.New("cannot configure legacy service-cidr: no IPv4 service-cidr ranges found")
 	}
 
 	serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange)
 	if err != nil {
-		return errors.Wrapf(err, "Invalid port range %s: %v", cfg.ServiceNodePortRange, err)
+		return errors.Wrapf(err, "invalid port range %s", cfg.ServiceNodePortRange)
 	}
 
 	_, apiServerServiceIP, err := controlplane.ServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange)
@@ -210,14 +249,41 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	}
 	serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String())
 
-	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
+	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
-	if cfg.ClusterDNS == "" {
-		serverConfig.ControlConfig.ClusterDNS = make(net.IP, 4)
-		copy(serverConfig.ControlConfig.ClusterDNS, serverConfig.ControlConfig.ServiceIPRange.IP.To4())
-		serverConfig.ControlConfig.ClusterDNS[3] = 10
+	// If there are no IPv4 ServiceCIDRs, an error will be raised.
+	if len(cmds.ServerConfig.ClusterDNS) == 0 {
+		for _, ipRange := range serverConfig.ControlConfig.ServiceIPRanges {
+			if ipRange.IP.To4() != nil {
+				serverConfig.ControlConfig.ClusterDNS = ipRange.IP.To4()
+				serverConfig.ControlConfig.ClusterDNS[3] = 10
+				serverConfig.ControlConfig.ClusterDNSs = []net.IP{serverConfig.ControlConfig.ClusterDNS}
+				break
+			}
+		}
+		if serverConfig.ControlConfig.ClusterDNS == nil {
+			return errors.New("cannot autoconfigure cluster-dns address: no IPv4 service-cidr ranges found")
+		}
 	} else {
-		serverConfig.ControlConfig.ClusterDNS = net.ParseIP(cfg.ClusterDNS)
+		for _, ip := range cmds.ServerConfig.ClusterDNS {
+			for _, v := range strings.Split(ip, ",") {
+				parsed := net.ParseIP(v)
+				if parsed == nil {
+					return fmt.Errorf("invalid cluster-dns address %s", v)
+				}
+				serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, parsed)
+			}
+		}
+		// Set ClusterDNS to the first IPv4 address, for legacy clients
+		for _, ip := range serverConfig.ControlConfig.ClusterDNSs {
+			if ip.To4() != nil {
+				serverConfig.ControlConfig.ClusterDNS = ip
+				break
+			}
+		}
+		if serverConfig.ControlConfig.ClusterDNS == nil {
+			return errors.New("cannot configure legacy cluster-dns address: no IPv4 cluster-dns addresses found")
+		}
 	}
 
 	if cfg.DefaultLocalStoragePath == "" {

pkg/daemons/agent/agent.go

@@ -13,6 +13,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/executor"
+	"github.com/rancher/k3s/pkg/util"
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -47,7 +48,7 @@ func startKubeProxy(cfg *config.Agent) error {
 		"proxy-mode":           "iptables",
 		"healthz-bind-address": "127.0.0.1",
 		"kubeconfig":           cfg.KubeConfigKubeProxy,
-		"cluster-cidr":         cfg.ClusterCIDR.String(),
+		"cluster-cidr":         util.JoinIPNets(cfg.ClusterCIDRs),
 	}
 	if cfg.NodeName != "" {
 		argsMap["hostname-override"] = cfg.NodeName
@@ -94,7 +95,7 @@ func startKubelet(cfg *config.Agent) error {
 		argsMap["network-plugin"] = "cni"
 	}
 	if len(cfg.ClusterDNS) > 0 {
-		argsMap["cluster-dns"] = cfg.ClusterDNS.String()
+		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
 	}
 	if cfg.ResolvConf != "" {
 		argsMap["resolv-conf"] = cfg.ResolvConf

pkg/daemons/config/types.go

@@ -58,10 +58,13 @@ type Agent struct {
 	NodeConfigPath          string
 	ServingKubeletCert      string
 	ServingKubeletKey       string
-	ServiceCIDR             net.IPNet
+	ServiceCIDR             *net.IPNet
+	ServiceCIDRs            []*net.IPNet
 	ServiceNodePortRange    utilnet.PortRange
-	ClusterCIDR             net.IPNet
+	ClusterCIDR             *net.IPNet
+	ClusterCIDRs            []*net.IPNet
 	ClusterDNS              net.IP
+	ClusterDNSs             []net.IP
 	ClusterDomain           string
 	ResolvConf              string
 	RootDir                 string
@@ -106,9 +109,12 @@ type Control struct {
 	AgentToken               string `json:"-"`
 	Token                    string `json:"-"`
 	ClusterIPRange           *net.IPNet
+	ClusterIPRanges          []*net.IPNet
 	ServiceIPRange           *net.IPNet
+	ServiceIPRanges          []*net.IPNet
 	ServiceNodePortRange     *utilnet.PortRange
 	ClusterDNS               net.IP
+	ClusterDNSs              []net.IP
 	ClusterDomain            string
 	NoCoreDNS                bool
 	KubeConfigOutput         string