Move VXLAN L2 programming to its own object.

- Remove special-case L2 logic in the RouteTable. - Add new VXLANFDB object to manage it.
projectcalico · Jan 25, 2024 · 5987494 · 5987494
1 parent c3d3768
commit 5987494
Show file tree

Hide file tree

Showing 14 changed files with 488 additions and 230 deletions.
diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go
@@ -495,7 +495,6 @@ func newBPFEndpointManager(
 		m.routeTable = routetable.New(
 			[]string{bpfInDev},
 			family,
-			false, // vxlan
 			config.NetlinkTimeout,
 			nil, // deviceRouteSourceAddress
 			config.DeviceRouteProtocol,

diff --git a/felix/dataplane/linux/endpoint_mgr_test.go b/felix/dataplane/linux/endpoint_mgr_test.go
@@ -586,8 +586,7 @@ func chainsForIfaces(ifaceMetadata []string,
 }
 
 type mockRouteTable struct {
-	currentRoutes   map[string][]routetable.Target
-	currentL2Routes map[string][]routetable.L2Target
+	currentRoutes map[string][]routetable.Target
 }
 
 func (t *mockRouteTable) SetRoutes(ifaceName string, targets []routetable.Target) {
@@ -598,14 +597,6 @@ func (t *mockRouteTable) SetRoutes(ifaceName string, targets []routetable.Target
 	t.currentRoutes[ifaceName] = targets
 }
 
-func (t *mockRouteTable) SetL2Routes(ifaceName string, targets []routetable.L2Target) {
-	log.WithFields(log.Fields{
-		"ifaceName": ifaceName,
-		"targets":   targets,
-	}).Debug("SetL2Routes")
-	t.currentL2Routes[ifaceName] = targets
-}
-
 func (t *mockRouteTable) RouteRemove(_ string, _ ip.CIDR) {
 }
 

diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go
@@ -16,6 +16,7 @@ package intdataplane
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -26,7 +27,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
@@ -35,32 +35,31 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"k8s.io/client-go/kubernetes"
 
-	"github.com/projectcalico/calico/felix/bpf/bpfmap"
-	"github.com/projectcalico/calico/felix/bpf/conntrack"
-	"github.com/projectcalico/calico/felix/bpf/failsafes"
-	bpfmaps "github.com/projectcalico/calico/felix/bpf/maps"
-	"github.com/projectcalico/calico/felix/bpf/nat"
-	tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs"
-	"github.com/projectcalico/calico/felix/environment"
-	"github.com/projectcalico/calico/felix/iptables/cmdshim"
-
+	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
 	"github.com/projectcalico/api/pkg/lib/numorstring"
 
 	"github.com/projectcalico/calico/felix/bpf"
+	"github.com/projectcalico/calico/felix/bpf/bpfmap"
+	"github.com/projectcalico/calico/felix/bpf/conntrack"
 	bpfconntrack "github.com/projectcalico/calico/felix/bpf/conntrack"
+	"github.com/projectcalico/calico/felix/bpf/failsafes"
 	bpfifstate "github.com/projectcalico/calico/felix/bpf/ifstate"
 	bpfipsets "github.com/projectcalico/calico/felix/bpf/ipsets"
+	bpfmaps "github.com/projectcalico/calico/felix/bpf/maps"
+	"github.com/projectcalico/calico/felix/bpf/nat"
 	bpfnat "github.com/projectcalico/calico/felix/bpf/nat"
 	bpfproxy "github.com/projectcalico/calico/felix/bpf/proxy"
 	bpfroutes "github.com/projectcalico/calico/felix/bpf/routes"
-
 	"github.com/projectcalico/calico/felix/bpf/tc"
+	tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs"
 	"github.com/projectcalico/calico/felix/config"
 	"github.com/projectcalico/calico/felix/dataplane/common"
+	"github.com/projectcalico/calico/felix/environment"
 	"github.com/projectcalico/calico/felix/idalloc"
 	"github.com/projectcalico/calico/felix/ifacemonitor"
 	"github.com/projectcalico/calico/felix/ipsets"
 	"github.com/projectcalico/calico/felix/iptables"
+	"github.com/projectcalico/calico/felix/iptables/cmdshim"
 	"github.com/projectcalico/calico/felix/jitter"
 	"github.com/projectcalico/calico/felix/labelindex"
 	"github.com/projectcalico/calico/felix/logutils"
@@ -69,6 +68,7 @@ import (
 	"github.com/projectcalico/calico/felix/routetable"
 	"github.com/projectcalico/calico/felix/rules"
 	"github.com/projectcalico/calico/felix/throttle"
+	"github.com/projectcalico/calico/felix/vxlanfdb"
 	"github.com/projectcalico/calico/felix/wireguard"
 	"github.com/projectcalico/calico/libcalico-go/lib/health"
 	lclogutils "github.com/projectcalico/calico/libcalico-go/lib/logutils"
@@ -291,6 +291,7 @@ type InternalDataplane struct {
 	vxlanParentC   chan string
 	vxlanManagerV6 *vxlanManager
 	vxlanParentCV6 chan string
+	vxlanFDBs      []*vxlanfdb.VXLANFDB
 
 	wireguardManager   *wireguardManager
 	wireguardManagerV6 *wireguardManager
@@ -506,18 +507,22 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 		var routeTableVXLAN routetable.RouteTableInterface
 		if !config.RouteSyncDisabled {
 			log.Debug("RouteSyncDisabled is false.")
-			routeTableVXLAN = routetable.New([]string{"^vxlan.calico$"}, 4, true, config.NetlinkTimeout,
+			routeTableVXLAN = routetable.New([]string{"^vxlan.calico$"}, 4, config.NetlinkTimeout,
 				config.DeviceRouteSourceAddress, config.DeviceRouteProtocol, true, unix.RT_TABLE_MAIN,
 				dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth))
 		} else {
 			log.Info("RouteSyncDisabled is true, using DummyTable.")
 			routeTableVXLAN = &routetable.DummyTable{}
 		}
 
+		vxlanFDB := vxlanfdb.New(netlink.FAMILY_V4, VXLANIfaceNameV4, featureDetector, config.NetlinkTimeout)
+		dp.vxlanFDBs = append(dp.vxlanFDBs, vxlanFDB)
+
 		dp.vxlanManager = newVXLANManager(
 			ipSetsV4,
 			routeTableVXLAN,
-			"vxlan.calico",
+			vxlanFDB,
+			VXLANIfaceNameV4,
 			config,
 			dp.loopSummarizer,
 			4,
@@ -528,7 +533,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 		dp.RegisterManager(dp.vxlanManager)
 	} else {
 		// Start a cleanup goroutine not to block felix if it needs to retry
-		go cleanUpVXLANDevice("vxlan.calico")
+		go cleanUpVXLANDevice(VXLANIfaceNameV4)
 	}
 
 	dp.endpointStatusCombiner = newEndpointStatusCombiner(dp.fromDataplane, config.IPv6Enabled)
@@ -865,7 +870,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 
 	if !config.RouteSyncDisabled {
 		log.Debug("RouteSyncDisabled is false.")
-		routeTableV4 = routetable.New(interfaceRegexes, 4, false, config.NetlinkTimeout,
+		routeTableV4 = routetable.New(interfaceRegexes, 4, config.NetlinkTimeout,
 			config.DeviceRouteSourceAddress, config.DeviceRouteProtocol, config.RemoveExternalRoutes, unix.RT_TABLE_MAIN,
 			dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth),
 			routetable.WithRouteCleanupGracePeriod(routeCleanupGracePeriod))
@@ -964,18 +969,22 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 			var routeTableVXLANV6 routetable.RouteTableInterface
 			if !config.RouteSyncDisabled {
 				log.Debug("RouteSyncDisabled is false.")
-				routeTableVXLANV6 = routetable.New([]string{"^vxlan-v6.calico$"}, 6, true, config.NetlinkTimeout,
+				routeTableVXLANV6 = routetable.New([]string{"^vxlan-v6.calico$"}, 6, config.NetlinkTimeout,
 					config.DeviceRouteSourceAddressIPv6, config.DeviceRouteProtocol, true, unix.RT_TABLE_MAIN,
 					dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth))
 			} else {
 				log.Debug("RouteSyncDisabled is true, using DummyTable for routeTableVXLANV6.")
 				routeTableVXLANV6 = &routetable.DummyTable{}
 			}
 
+			vxlanFDBV6 := vxlanfdb.New(netlink.FAMILY_V6, VXLANIfaceNameV6, featureDetector, config.NetlinkTimeout)
+			dp.vxlanFDBs = append(dp.vxlanFDBs, vxlanFDBV6)
+
 			dp.vxlanManagerV6 = newVXLANManager(
 				ipSetsV6,
 				routeTableVXLANV6,
-				"vxlan-v6.calico",
+				vxlanFDBV6,
+				VXLANIfaceNameV6,
 				config,
 				dp.loopSummarizer,
 				6,
@@ -986,14 +995,14 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane {
 			dp.RegisterManager(dp.vxlanManagerV6)
 		} else {
 			// Start a cleanup goroutine not to block felix if it needs to retry
-			go cleanUpVXLANDevice("vxlan-v6.calico")
+			go cleanUpVXLANDevice(VXLANIfaceNameV6)
 		}
 
 		var routeTableV6 routetable.RouteTableInterface
 		if !config.RouteSyncDisabled {
 			log.Debug("RouteSyncDisabled is false.")
 			routeTableV6 = routetable.New(
-				interfaceRegexes, 6, false, config.NetlinkTimeout,
+				interfaceRegexes, 6, config.NetlinkTimeout,
 				config.DeviceRouteSourceAddressIPv6, config.DeviceRouteProtocol, config.RemoveExternalRoutes,
 				unix.RT_TABLE_MAIN, dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth),
 				routetable.WithRouteCleanupGracePeriod(routeCleanupGracePeriod))
@@ -2030,6 +2039,10 @@ func (d *InternalDataplane) processIfaceStateUpdate(ifaceUpdate *ifaceStateUpdat
 		mgr.OnUpdate(ifaceUpdate)
 	}
 
+	for _, fdb := range d.vxlanFDBs {
+		fdb.OnIfaceStateChanged(ifaceUpdate.Name, ifaceUpdate.State)
+	}
+
 	for _, mgr := range d.managersWithRouteTables {
 		for _, routeTable := range mgr.GetRouteTableSyncers() {
 			routeTable.OnIfaceStateChanged(ifaceUpdate.Name, ifaceUpdate.State)
@@ -2179,6 +2192,9 @@ func (d *InternalDataplane) apply() {
 			// Queue a resync on the next Apply().
 			r.QueueResync()
 		}
+		for _, fdb := range d.vxlanFDBs {
+			fdb.QueueResync()
+		}
 		d.forceRouteRefresh = false
 	}
 
@@ -2203,6 +2219,20 @@ func (d *InternalDataplane) apply() {
 		}(ipSets)
 	}
 
+	// Update any VXLAN FDB entries.
+	for _, fdb := range d.vxlanFDBs {
+		err := fdb.Apply()
+		if err != nil {
+			var lnf netlink.LinkNotFoundError
+			if errors.As(err, &lnf) || errors.Is(err, vxlanfdb.ErrLinkDown) {
+				log.Debug("VXLAN interface not ready yet, can't sync FDB entries.")
+			} else {
+				log.WithError(err).Warn("Failed to synchronize VXLAN FDB entries, will retry...")
+				d.dataplaneNeedsSync = true
+			}
+		}
+	}
+
 	// Update the routing table in parallel with the other updates.  We'll wait for it to finish
 	// before we return.
 	var routesWG sync.WaitGroup