From 59874943b854b6058bed86aae9b14628b4408038 Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Thu, 25 Jan 2024 10:57:06 +0000 Subject: [PATCH] Move VXLAN L2 programming to its own object. - Remove special-case L2 logic in the RouteTable. - Add new VXLANFDB object to manage it. --- felix/dataplane/linux/bpf_ep_mgr.go | 1 - felix/dataplane/linux/endpoint_mgr_test.go | 11 +- felix/dataplane/linux/int_dataplane.go | 68 +++- felix/dataplane/linux/vxlan_mgr.go | 54 ++- felix/dataplane/linux/vxlan_mgr_test.go | 47 ++- felix/ifacemonitor/iface_monitor.go | 4 +- felix/ifacemonitor/netlink_real.go | 2 +- felix/ifacemonitor/update_filter.go | 2 +- felix/netlinkshim/handlemgr/handle_manager.go | 18 +- felix/routetable/interface.go | 1 - felix/routetable/route_table.go | 147 +------- felix/routetable/route_table_test.go | 7 - felix/vxlanfdb/vxlan_fdb.go | 355 ++++++++++++++++++ felix/wireguard/wireguard.go | 1 - 14 files changed, 488 insertions(+), 230 deletions(-) create mode 100644 felix/vxlanfdb/vxlan_fdb.go diff --git a/felix/dataplane/linux/bpf_ep_mgr.go b/felix/dataplane/linux/bpf_ep_mgr.go index 13420995264..2bf2ff931d6 100644 --- a/felix/dataplane/linux/bpf_ep_mgr.go +++ b/felix/dataplane/linux/bpf_ep_mgr.go @@ -495,7 +495,6 @@ func newBPFEndpointManager( m.routeTable = routetable.New( []string{bpfInDev}, family, - false, // vxlan config.NetlinkTimeout, nil, // deviceRouteSourceAddress config.DeviceRouteProtocol, diff --git a/felix/dataplane/linux/endpoint_mgr_test.go b/felix/dataplane/linux/endpoint_mgr_test.go index fdf71bab0a3..c52e5c08bad 100644 --- a/felix/dataplane/linux/endpoint_mgr_test.go +++ b/felix/dataplane/linux/endpoint_mgr_test.go @@ -586,8 +586,7 @@ func chainsForIfaces(ifaceMetadata []string, } type mockRouteTable struct { - currentRoutes map[string][]routetable.Target - currentL2Routes map[string][]routetable.L2Target + currentRoutes map[string][]routetable.Target } func (t *mockRouteTable) SetRoutes(ifaceName string, targets []routetable.Target) { @@ -598,14 +597,6 @@ func (t *mockRouteTable) SetRoutes(ifaceName string, targets []routetable.Target t.currentRoutes[ifaceName] = targets } -func (t *mockRouteTable) SetL2Routes(ifaceName string, targets []routetable.L2Target) { - log.WithFields(log.Fields{ - "ifaceName": ifaceName, - "targets": targets, - }).Debug("SetL2Routes") - t.currentL2Routes[ifaceName] = targets -} - func (t *mockRouteTable) RouteRemove(_ string, _ ip.CIDR) { } diff --git a/felix/dataplane/linux/int_dataplane.go b/felix/dataplane/linux/int_dataplane.go index 25956cfa2dd..79d1c3f8ca3 100644 --- a/felix/dataplane/linux/int_dataplane.go +++ b/felix/dataplane/linux/int_dataplane.go @@ -16,6 +16,7 @@ package intdataplane import ( "context" + "errors" "fmt" "net" "os" @@ -26,7 +27,6 @@ import ( "sync/atomic" "time" - apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" @@ -35,32 +35,31 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "k8s.io/client-go/kubernetes" - "github.com/projectcalico/calico/felix/bpf/bpfmap" - "github.com/projectcalico/calico/felix/bpf/conntrack" - "github.com/projectcalico/calico/felix/bpf/failsafes" - bpfmaps "github.com/projectcalico/calico/felix/bpf/maps" - "github.com/projectcalico/calico/felix/bpf/nat" - tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" - "github.com/projectcalico/calico/felix/environment" - "github.com/projectcalico/calico/felix/iptables/cmdshim" - + apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/projectcalico/calico/felix/bpf" + "github.com/projectcalico/calico/felix/bpf/bpfmap" + "github.com/projectcalico/calico/felix/bpf/conntrack" bpfconntrack "github.com/projectcalico/calico/felix/bpf/conntrack" + "github.com/projectcalico/calico/felix/bpf/failsafes" bpfifstate "github.com/projectcalico/calico/felix/bpf/ifstate" bpfipsets "github.com/projectcalico/calico/felix/bpf/ipsets" + bpfmaps "github.com/projectcalico/calico/felix/bpf/maps" + "github.com/projectcalico/calico/felix/bpf/nat" bpfnat "github.com/projectcalico/calico/felix/bpf/nat" bpfproxy "github.com/projectcalico/calico/felix/bpf/proxy" bpfroutes "github.com/projectcalico/calico/felix/bpf/routes" - "github.com/projectcalico/calico/felix/bpf/tc" + tcdefs "github.com/projectcalico/calico/felix/bpf/tc/defs" "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/dataplane/common" + "github.com/projectcalico/calico/felix/environment" "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/ifacemonitor" "github.com/projectcalico/calico/felix/ipsets" "github.com/projectcalico/calico/felix/iptables" + "github.com/projectcalico/calico/felix/iptables/cmdshim" "github.com/projectcalico/calico/felix/jitter" "github.com/projectcalico/calico/felix/labelindex" "github.com/projectcalico/calico/felix/logutils" @@ -69,6 +68,7 @@ import ( "github.com/projectcalico/calico/felix/routetable" "github.com/projectcalico/calico/felix/rules" "github.com/projectcalico/calico/felix/throttle" + "github.com/projectcalico/calico/felix/vxlanfdb" "github.com/projectcalico/calico/felix/wireguard" "github.com/projectcalico/calico/libcalico-go/lib/health" lclogutils "github.com/projectcalico/calico/libcalico-go/lib/logutils" @@ -291,6 +291,7 @@ type InternalDataplane struct { vxlanParentC chan string vxlanManagerV6 *vxlanManager vxlanParentCV6 chan string + vxlanFDBs []*vxlanfdb.VXLANFDB wireguardManager *wireguardManager wireguardManagerV6 *wireguardManager @@ -506,7 +507,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { var routeTableVXLAN routetable.RouteTableInterface if !config.RouteSyncDisabled { log.Debug("RouteSyncDisabled is false.") - routeTableVXLAN = routetable.New([]string{"^vxlan.calico$"}, 4, true, config.NetlinkTimeout, + routeTableVXLAN = routetable.New([]string{"^vxlan.calico$"}, 4, config.NetlinkTimeout, config.DeviceRouteSourceAddress, config.DeviceRouteProtocol, true, unix.RT_TABLE_MAIN, dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth)) } else { @@ -514,10 +515,14 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { routeTableVXLAN = &routetable.DummyTable{} } + vxlanFDB := vxlanfdb.New(netlink.FAMILY_V4, VXLANIfaceNameV4, featureDetector, config.NetlinkTimeout) + dp.vxlanFDBs = append(dp.vxlanFDBs, vxlanFDB) + dp.vxlanManager = newVXLANManager( ipSetsV4, routeTableVXLAN, - "vxlan.calico", + vxlanFDB, + VXLANIfaceNameV4, config, dp.loopSummarizer, 4, @@ -528,7 +533,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { dp.RegisterManager(dp.vxlanManager) } else { // Start a cleanup goroutine not to block felix if it needs to retry - go cleanUpVXLANDevice("vxlan.calico") + go cleanUpVXLANDevice(VXLANIfaceNameV4) } dp.endpointStatusCombiner = newEndpointStatusCombiner(dp.fromDataplane, config.IPv6Enabled) @@ -865,7 +870,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { if !config.RouteSyncDisabled { log.Debug("RouteSyncDisabled is false.") - routeTableV4 = routetable.New(interfaceRegexes, 4, false, config.NetlinkTimeout, + routeTableV4 = routetable.New(interfaceRegexes, 4, config.NetlinkTimeout, config.DeviceRouteSourceAddress, config.DeviceRouteProtocol, config.RemoveExternalRoutes, unix.RT_TABLE_MAIN, dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth), routetable.WithRouteCleanupGracePeriod(routeCleanupGracePeriod)) @@ -964,7 +969,7 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { var routeTableVXLANV6 routetable.RouteTableInterface if !config.RouteSyncDisabled { log.Debug("RouteSyncDisabled is false.") - routeTableVXLANV6 = routetable.New([]string{"^vxlan-v6.calico$"}, 6, true, config.NetlinkTimeout, + routeTableVXLANV6 = routetable.New([]string{"^vxlan-v6.calico$"}, 6, config.NetlinkTimeout, config.DeviceRouteSourceAddressIPv6, config.DeviceRouteProtocol, true, unix.RT_TABLE_MAIN, dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth)) } else { @@ -972,10 +977,14 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { routeTableVXLANV6 = &routetable.DummyTable{} } + vxlanFDBV6 := vxlanfdb.New(netlink.FAMILY_V6, VXLANIfaceNameV6, featureDetector, config.NetlinkTimeout) + dp.vxlanFDBs = append(dp.vxlanFDBs, vxlanFDBV6) + dp.vxlanManagerV6 = newVXLANManager( ipSetsV6, routeTableVXLANV6, - "vxlan-v6.calico", + vxlanFDBV6, + VXLANIfaceNameV6, config, dp.loopSummarizer, 6, @@ -986,14 +995,14 @@ func NewIntDataplaneDriver(config Config) *InternalDataplane { dp.RegisterManager(dp.vxlanManagerV6) } else { // Start a cleanup goroutine not to block felix if it needs to retry - go cleanUpVXLANDevice("vxlan-v6.calico") + go cleanUpVXLANDevice(VXLANIfaceNameV6) } var routeTableV6 routetable.RouteTableInterface if !config.RouteSyncDisabled { log.Debug("RouteSyncDisabled is false.") routeTableV6 = routetable.New( - interfaceRegexes, 6, false, config.NetlinkTimeout, + interfaceRegexes, 6, config.NetlinkTimeout, config.DeviceRouteSourceAddressIPv6, config.DeviceRouteProtocol, config.RemoveExternalRoutes, unix.RT_TABLE_MAIN, dp.loopSummarizer, featureDetector, routetable.WithLivenessCB(dp.reportHealth), routetable.WithRouteCleanupGracePeriod(routeCleanupGracePeriod)) @@ -2030,6 +2039,10 @@ func (d *InternalDataplane) processIfaceStateUpdate(ifaceUpdate *ifaceStateUpdat mgr.OnUpdate(ifaceUpdate) } + for _, fdb := range d.vxlanFDBs { + fdb.OnIfaceStateChanged(ifaceUpdate.Name, ifaceUpdate.State) + } + for _, mgr := range d.managersWithRouteTables { for _, routeTable := range mgr.GetRouteTableSyncers() { routeTable.OnIfaceStateChanged(ifaceUpdate.Name, ifaceUpdate.State) @@ -2179,6 +2192,9 @@ func (d *InternalDataplane) apply() { // Queue a resync on the next Apply(). r.QueueResync() } + for _, fdb := range d.vxlanFDBs { + fdb.QueueResync() + } d.forceRouteRefresh = false } @@ -2203,6 +2219,20 @@ func (d *InternalDataplane) apply() { }(ipSets) } + // Update any VXLAN FDB entries. + for _, fdb := range d.vxlanFDBs { + err := fdb.Apply() + if err != nil { + var lnf netlink.LinkNotFoundError + if errors.As(err, &lnf) || errors.Is(err, vxlanfdb.ErrLinkDown) { + log.Debug("VXLAN interface not ready yet, can't sync FDB entries.") + } else { + log.WithError(err).Warn("Failed to synchronize VXLAN FDB entries, will retry...") + d.dataplaneNeedsSync = true + } + } + } + // Update the routing table in parallel with the other updates. We'll wait for it to finish // before we return. var routesWG sync.WaitGroup diff --git a/felix/dataplane/linux/vxlan_mgr.go b/felix/dataplane/linux/vxlan_mgr.go index 49a50b4f311..0f34bfafe93 100644 --- a/felix/dataplane/linux/vxlan_mgr.go +++ b/felix/dataplane/linux/vxlan_mgr.go @@ -40,9 +40,15 @@ import ( "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/felix/routetable" "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/felix/vxlanfdb" "github.com/projectcalico/calico/libcalico-go/lib/set" ) +const ( + VXLANIfaceNameV4 = "vxlan.calico" + VXLANIfaceNameV6 = "vxlan-v6.calico" +) + // added so that we can shim netlink for tests type netlinkHandle interface { LinkByName(name string) (netlink.Link, error) @@ -65,6 +71,7 @@ type vxlanManager struct { parentIfaceName string lastParentDevUpdate time.Time previouslyUsedParentNames set.Set[string] + fdb VXLANFDB // Hold pending updates. routesByDest map[string]*proto.RouteUpdate @@ -93,7 +100,7 @@ type vxlanManager struct { noEncapProtocol netlink.RouteProtocol // Used so that we can shim the no encap route table for the tests noEncapRTConstruct func( - interfacePrefixes []string, ipVersion uint8, vxlan bool, netlinkTimeout time.Duration, + interfacePrefixes []string, ipVersion uint8, netlinkTimeout time.Duration, deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool, ) routetable.RouteTableInterface @@ -105,9 +112,14 @@ const ( defaultVXLANProto netlink.RouteProtocol = 80 ) +type VXLANFDB interface { + SetVTEPs(vteps []vxlanfdb.VTEP) +} + func newVXLANManager( ipsetsDataplane common.IPSetsDataplane, rt routetable.RouteTableInterface, + fdb VXLANFDB, deviceName string, dpConfig Config, opRecorder logutils.OpRecorder, @@ -127,7 +139,6 @@ func newVXLANManager( brt = routetable.New( []string{routetable.InterfaceNone}, 4, - false, dpConfig.NetlinkTimeout, dpConfig.DeviceRouteSourceAddress, blackHoleProto, @@ -140,7 +151,6 @@ func newVXLANManager( brt = routetable.New( []string{routetable.InterfaceNone}, ipVersion, - false, dpConfig.NetlinkTimeout, dpConfig.DeviceRouteSourceAddressIPv6, blackHoleProto, @@ -160,13 +170,19 @@ func newVXLANManager( return newVXLANManagerWithShims( ipsetsDataplane, rt, brt, + fdb, deviceName, dpConfig, nlHandle, ipVersion, - func(interfaceRegexes []string, ipVersion uint8, vxlan bool, netlinkTimeout time.Duration, - deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool) routetable.RouteTableInterface { - return routetable.New(interfaceRegexes, ipVersion, vxlan, netlinkTimeout, + func(interfaceRegexes []string, + ipVersion uint8, + netlinkTimeout time.Duration, + deviceRouteSourceAddress net.IP, + deviceRouteProtocol netlink.RouteProtocol, + removeExternalRoutes bool, + ) routetable.RouteTableInterface { + return routetable.New(interfaceRegexes, ipVersion, netlinkTimeout, deviceRouteSourceAddress, deviceRouteProtocol, removeExternalRoutes, unix.RT_TABLE_MAIN, opRecorder, featureDetector, ) @@ -177,11 +193,12 @@ func newVXLANManager( func newVXLANManagerWithShims( ipsetsDataplane common.IPSetsDataplane, rt, brt routetable.RouteTableInterface, + fdb VXLANFDB, deviceName string, dpConfig Config, nlHandle netlinkHandle, ipVersion uint8, - noEncapRTConstruct func(interfacePrefixes []string, ipVersion uint8, vxlan bool, netlinkTimeout time.Duration, + noEncapRTConstruct func(interfacePrefixes []string, ipVersion uint8, netlinkTimeout time.Duration, deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool) routetable.RouteTableInterface, ) *vxlanManager { noEncapProtocol := defaultVXLANProto @@ -201,6 +218,7 @@ func newVXLANManagerWithShims( routeTable: rt, blackholeRouteTable: brt, previouslyUsedParentNames: set.New[string](), + fdb: fdb, routesByDest: map[string]*proto.RouteUpdate{}, localIPAMBlocks: map[string]*proto.RouteUpdate{}, vtepsByNode: map[string]*proto.VXLANTunnelEndpointUpdate{}, @@ -424,7 +442,7 @@ func (m *vxlanManager) CompleteDeferredWork() error { // The route table accepts the desired state. Start by setting the desired L2 "routes" by iterating // known VTEPs. - var l2routes []routetable.L2Target + var l2routes []vxlanfdb.VTEP for _, u := range m.vtepsByNode { mac, err := parseMacForIPVersion(u, m.ipVersion) if err != nil { @@ -438,15 +456,15 @@ func (m *vxlanManager) CompleteDeferredWork() error { addr = u.Ipv6Addr parentDeviceIP = u.ParentDeviceIpv6 } - l2routes = append(l2routes, routetable.L2Target{ - VTEPMAC: mac, - GW: ip.FromString(addr), - IP: ip.FromString(parentDeviceIP), + l2routes = append(l2routes, vxlanfdb.VTEP{ + TunnelMAC: mac, + TunnelIP: ip.FromString(addr), + HostIP: ip.FromString(parentDeviceIP), }) allowedVXLANSources = append(allowedVXLANSources, parentDeviceIP) } m.logCtx.WithField("l2routes", l2routes).Debug("VXLAN manager sending L2 updates") - m.routeTable.SetL2Routes(m.vxlanDevice, l2routes) + m.fdb.SetVTEPs(l2routes) m.ipsetsDataplane.AddOrReplaceIPSet(m.ipSetMetadata, allowedVXLANSources) m.vtepsDirty = false } @@ -507,14 +525,10 @@ func (m *vxlanManager) CompleteDeferredWork() error { m.blackholeRouteTable.SetRoutes(routetable.InterfaceNone, m.blackholeRoutes()) - // only set the noEncapRouteTable table if it's nil, as you will lose the routes that are being managed already - // and the new table will probably delete routes that were put in there by the previous table if m.noEncapRouteTable != nil { - m.logCtx.WithField("link", m.parentIfaceName).WithField("routes", noEncapRoutes).Debug( "VXLAN manager sending unencapsulated L3 updates") m.noEncapRouteTable.SetRoutes(m.parentIfaceName, noEncapRoutes) - } else { return errors.New("no encap route table not set, will defer adding routes") } @@ -555,7 +569,6 @@ func (m *vxlanManager) OnParentNameUpdate(name string) { m.noEncapRouteTable = m.noEncapRTConstruct( []string{"^" + strings.Join(names, "|") + "$"}, m.ipVersion, - false, m.dpConfig.NetlinkTimeout, devRouteSrcAddr, m.noEncapProtocol, @@ -792,7 +805,10 @@ func (m *vxlanManager) ensureAddressOnLink(ipStr string, link netlink.Link) erro addrPresent = true continue } - m.logCtx.WithFields(logrus.Fields{"address": existing, "link": link.Attrs().Name}).Warn("Removing unwanted IP from VXLAN device") + m.logCtx.WithFields(logrus.Fields{ + "address": existing, + "link": link.Attrs().Name, + }).Warn("Removing unwanted IP from VXLAN device") if err := m.nlHandle.AddrDel(link, &existing); err != nil { return fmt.Errorf("failed to remove IP address %s", existing) } diff --git a/felix/dataplane/linux/vxlan_mgr_test.go b/felix/dataplane/linux/vxlan_mgr_test.go index 6a6a3a52581..b0486dc1af4 100644 --- a/felix/dataplane/linux/vxlan_mgr_test.go +++ b/felix/dataplane/linux/vxlan_mgr_test.go @@ -19,11 +19,14 @@ import ( "net" "time" + log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/felix/dataplane/common" "github.com/projectcalico/calico/felix/ip" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/calico/felix/routetable" "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/felix/vxlanfdb" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -73,16 +76,14 @@ func (m *mockVXLANDataplane) AddrList(link netlink.Link, family int) ([]netlink. IPNet: &net.IPNet{ IP: net.IPv4(172, 0, 0, 2), }, - }, - } + }} if m.ipVersion == 6 { l = []netlink.Addr{{ IPNet: &net.IPNet{ IP: net.ParseIP("fc00:10:96::2"), }, - }, - } + }} } return l, nil } @@ -106,29 +107,42 @@ func (m *mockVXLANDataplane) LinkDel(netlink.Link) error { return nil } +type mockVXLANFDB struct { + currentVTEPs []vxlanfdb.VTEP // FIXME move to separate mock. +} + +func (t *mockVXLANFDB) SetVTEPs(targets []vxlanfdb.VTEP) { + log.WithFields(log.Fields{ + "targets": targets, + }).Debug("SetL2Routes") + t.currentVTEPs = targets +} + var _ = Describe("VXLANManager", func() { var manager, managerV6 *vxlanManager var rt, brt, prt *mockRouteTable + var fdb *mockVXLANFDB BeforeEach(func() { rt = &mockRouteTable{ - currentRoutes: map[string][]routetable.Target{}, - currentL2Routes: map[string][]routetable.L2Target{}, + currentRoutes: map[string][]routetable.Target{}, } brt = &mockRouteTable{ - currentRoutes: map[string][]routetable.Target{}, - currentL2Routes: map[string][]routetable.L2Target{}, + currentRoutes: map[string][]routetable.Target{}, } prt = &mockRouteTable{ - currentRoutes: map[string][]routetable.Target{}, - currentL2Routes: map[string][]routetable.L2Target{}, + currentRoutes: map[string][]routetable.Target{}, } + // FIXME actually assert on the FDB contents. + fdb = &mockVXLANFDB{} + la := netlink.NewLinkAttrs() la.Name = "eth0" manager = newVXLANManagerWithShims( common.NewMockIPSets(), rt, brt, + fdb, "vxlan.calico", Config{ MaxIPSetSize: 5, @@ -144,8 +158,10 @@ var _ = Describe("VXLANManager", func() { ipVersion: 4, }, 4, - func(interfacePrefixes []string, ipVersion uint8, vxlan bool, netlinkTimeout time.Duration, - deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool) routetable.RouteTableInterface { + func( + interfacePrefixes []string, ipVersion uint8, netlinkTimeout time.Duration, + deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool, + ) routetable.RouteTableInterface { return prt }, ) @@ -153,6 +169,7 @@ var _ = Describe("VXLANManager", func() { managerV6 = newVXLANManagerWithShims( common.NewMockIPSets(), rt, brt, + fdb, "vxlan-v6.calico", Config{ MaxIPSetSize: 5, @@ -168,8 +185,10 @@ var _ = Describe("VXLANManager", func() { ipVersion: 6, }, 6, - func(interfacePrefixes []string, ipVersion uint8, vxlan bool, netlinkTimeout time.Duration, - deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool) routetable.RouteTableInterface { + func( + interfacePrefixes []string, ipVersion uint8, netlinkTimeout time.Duration, + deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, removeExternalRoutes bool, + ) routetable.RouteTableInterface { return prt }, ) diff --git a/felix/ifacemonitor/iface_monitor.go b/felix/ifacemonitor/iface_monitor.go index 817baac356c..59e4b7e2e2a 100644 --- a/felix/ifacemonitor/iface_monitor.go +++ b/felix/ifacemonitor/iface_monitor.go @@ -285,7 +285,7 @@ func (m *InterfaceMonitor) storeAndNotifyLink(ifaceExists bool, link netlink.Lin m.storeAndNotifyLinkInner(ifaceExists, newName, link) } -func linkIsOperUp(link netlink.Link) bool { +func LinkIsOperUp(link netlink.Link) bool { // We need the operstate of the interface; this is carried in the IFF_RUNNING flag. The // IFF_UP flag contains the admin state, which doesn't tell us whether we can program routes // etc. @@ -315,7 +315,7 @@ func (m *InterfaceMonitor) storeAndNotifyLinkInner(ifaceExists bool, ifaceName s } newState := StateNotPresent if ifaceExists { - if linkIsOperUp(link) { + if LinkIsOperUp(link) { newState = StateUp } else { newState = StateDown diff --git a/felix/ifacemonitor/netlink_real.go b/felix/ifacemonitor/netlink_real.go index b9464b454b2..eb39bbba203 100644 --- a/felix/ifacemonitor/netlink_real.go +++ b/felix/ifacemonitor/netlink_real.go @@ -32,7 +32,7 @@ type netlinkReal struct { func newRealNetlink(featureDetector environment.FeatureDetectorIface, timeout time.Duration) *netlinkReal { return &netlinkReal{ - handleMgr: handlemgr.NewHandleManager(netlink.FAMILY_ALL, featureDetector, handlemgr.WithSocketTimeout(timeout)), + handleMgr: handlemgr.NewHandleManager(featureDetector, handlemgr.WithSocketTimeout(timeout)), } } diff --git a/felix/ifacemonitor/update_filter.go b/felix/ifacemonitor/update_filter.go index f01f9bb82c2..f731a495312 100644 --- a/felix/ifacemonitor/update_filter.go +++ b/felix/ifacemonitor/update_filter.go @@ -86,7 +86,7 @@ mainLoop: return } idx := int(linkUpd.Index) - linkIsUp := linkUpd.Header.Type == syscall.RTM_NEWLINK && linkIsOperUp(linkUpd.Link) + linkIsUp := linkUpd.Header.Type == syscall.RTM_NEWLINK && LinkIsOperUp(linkUpd.Link) var delay time.Duration if linkIsUp { if len(updatesByIfaceIdx[idx]) == 0 { diff --git a/felix/netlinkshim/handlemgr/handle_manager.go b/felix/netlinkshim/handlemgr/handle_manager.go index 2680521bf7b..e9bd05256f0 100644 --- a/felix/netlinkshim/handlemgr/handle_manager.go +++ b/felix/netlinkshim/handlemgr/handle_manager.go @@ -35,7 +35,7 @@ type HandleManager struct { // reset on successful connection. numRepeatFailures int - family int + strictEnabled bool socketTimeout time.Duration featureDetector environment.FeatureDetectorIface @@ -56,16 +56,18 @@ func WithNewHandleOverride(newNetlinkHandle func() (netlinkshim.Interface, error } } -func NewHandleManager( - netlinkFamily int, - featureDetector environment.FeatureDetectorIface, - opts ...NetlinkHandleManagerOpt, -) *HandleManager { +func WithStrictModeOverride(strictEnabled bool) NetlinkHandleManagerOpt { + return func(manager *HandleManager) { + manager.strictEnabled = strictEnabled + } +} + +func NewHandleManager(featureDetector environment.FeatureDetectorIface, opts ...NetlinkHandleManagerOpt) *HandleManager { nlm := &HandleManager{ - family: netlinkFamily, socketTimeout: defaultSocketTimeout, featureDetector: featureDetector, newNetlinkHandle: netlinkshim.NewRealNetlink, + strictEnabled: true, } for _, o := range opts { o(nlm) @@ -111,7 +113,7 @@ func (r *HandleManager) newHandle() (netlinkshim.Interface, error) { nlHandle.Delete() return nil, err } - if r.featureDetector.GetFeatures().KernelSideRouteFiltering { + if r.strictEnabled && r.featureDetector.GetFeatures().KernelSideRouteFiltering { logrus.Debug("Kernel supports route filtering, enabling 'strict' netlink mode.") err = nlHandle.SetStrictCheck(true) if err != nil { diff --git a/felix/routetable/interface.go b/felix/routetable/interface.go index a6f7ebc95c8..debd32911cb 100644 --- a/felix/routetable/interface.go +++ b/felix/routetable/interface.go @@ -31,7 +31,6 @@ type RouteTableSyncer interface { type RouteTableInterface interface { RouteTableSyncer SetRoutes(ifaceName string, targets []Target) - SetL2Routes(ifaceName string, targets []L2Target) RouteRemove(ifaceName string, cidr ip.CIDR) RouteUpdate(ifaceName string, target Target) } diff --git a/felix/routetable/route_table.go b/felix/routetable/route_table.go index cb390593ecc..8c719ed5007 100644 --- a/felix/routetable/route_table.go +++ b/felix/routetable/route_table.go @@ -197,9 +197,6 @@ type RouteTable struct { pendingConntrackCleanups map[ip.Addr]chan struct{} - // Whether this route table is managing vxlan routes. - vxlan bool - deviceRouteSourceAddress net.IP deviceRouteProtocol netlink.RouteProtocol @@ -240,7 +237,6 @@ func WithRouteCleanupGracePeriod(routeCleanupGracePeriod time.Duration) RouteTab func New( interfaceRegexes []string, ipVersion uint8, - vxlan bool, netlinkTimeout time.Duration, deviceRouteSourceAddress net.IP, deviceRouteProtocol netlink.RouteProtocol, @@ -254,7 +250,6 @@ func New( interfaceRegexes, ipVersion, netlinkshim.NewRealNetlink, - vxlan, netlinkTimeout, addStaticARPEntry, conntrack.New(), @@ -274,7 +269,6 @@ func NewWithShims( interfaceRegexes []string, ipVersion uint8, newNetlinkHandle func() (netlinkshim.Interface, error), - vxlan bool, netlinkTimeout time.Duration, addStaticARPEntry func(cidr ip.CIDR, destMAC net.HardwareAddr, ifaceName string) error, conntrack conntrackIface, @@ -349,7 +343,6 @@ func NewWithShims( addStaticARPEntry: addStaticARPEntry, conntrack: conntrack, time: timeShim, - vxlan: vxlan, deviceRouteSourceAddress: deviceRouteSourceAddress, deviceRouteProtocol: deviceRouteProtocol, removeExternalRoutes: removeExternalRoutes, @@ -357,7 +350,6 @@ func NewWithShims( opReporter: opReporter, livenessCallback: func() {}, nl: handlemgr.NewHandleManager( - family, featureDetector, handlemgr.WithNewHandleOverride(newNetlinkHandle), handlemgr.WithSocketTimeout(netlinkTimeout), @@ -571,15 +563,7 @@ func (r *RouteTable) Apply() error { firstTry := retry == 0 lastTry := retry == maxApplyRetries-1 fullResync := ia == updateTypeFullResync || lastTry - var err error - if r.vxlan { - // Sync L2 routes first. - err = r.syncL2RoutesForLink(ifaceName) - } - if err == nil { - // No errors syncing L2, sync L3 routes. - err = r.syncRoutesForLink(ifaceName, fullResync, firstTry) - } + err := r.syncRoutesForLink(ifaceName, fullResync, firstTry) // Handle errors from syncing either L2 or L3 routes. switch err { @@ -1010,135 +994,6 @@ func (r *RouteTable) readProgrammedRoutes(logCxt *log.Entry, ifaceName string) ( return programmedRoutes, nil } -func (r *RouteTable) syncL2RoutesForLink(ifaceName string) error { - logCxt := r.logCxt.WithField("ifaceName", ifaceName) - logCxt.Debug("Syncing interface L2 routes") - if updatedTargets, ok := r.pendingIfaceNameToL2Targets[ifaceName]; ok { - logCxt.Debug("Have updated targets.") - if updatedTargets == nil { - delete(r.ifaceNameToL2Targets, ifaceName) - } else { - r.ifaceNameToL2Targets[ifaceName] = updatedTargets - } - delete(r.pendingIfaceNameToL2Targets, ifaceName) - } - expectedTargets := r.ifaceNameToL2Targets[ifaceName] - - // Try to get the link attributes. This may fail if it's been deleted out from under us. - linkAttrs, err := r.getLinkAttributes(ifaceName) - if err != nil { - r.logCxt.WithError(err).Error("Failed to get link attributes") - return err - } - - // Build a map of expected targets by hwaddr for easier lookup, - // so we can compare the expected L2 targets against the programmed ones - // for this link. - expected := map[string]bool{} - for _, target := range expectedTargets { - expected[target.VTEPMAC.String()] = true - } - - // Get the current set of neighbors on this interface. - existingNeigh, err := netlink.NeighList(linkAttrs.Index, netlink.FAMILY_V4) - if err != nil { - return err - } - - // For each existing neighbor, if we don't expect an entry for its MAC address to be programmed - // on this link, then delete it. - var updatesFailed bool - - for _, existing := range existingNeigh { - if existing.HardwareAddr == nil { - log.WithField("neighbor", existing).Debug("Ignoring existing ARP entry with no hardware addr") - continue - } - if _, ok := expected[existing.HardwareAddr.String()]; !ok { - logCxt.WithField("neighbor", existing).Debug("Neighbor should no longer be programmed") - - // Remove the FDB entry for this neighbor. - n := netlink.Neigh{ - LinkIndex: existing.LinkIndex, - State: netlink.NUD_PERMANENT, - Family: syscall.AF_BRIDGE, - Flags: netlink.NTF_SELF, - IP: existing.IP, - HardwareAddr: existing.HardwareAddr, - } - if err := netlink.NeighDel(&n); err != nil { - if !strings.Contains(err.Error(), "no such file or directory") { - logCxt.WithError(err).Warnf("Failed to delete neighbor FDB entry %+v", n) - updatesFailed = true - } - } else { - logCxt.WithField("neighbor", existing).Info("Removed old neighbor FDB entry") - } - - // Delete the ARP entry. - if err := netlink.NeighDel(&existing); err != nil { - if !strings.Contains(err.Error(), "no such file or directory") { - logCxt.WithError(err).Warnf("Failed to delete neighbor ARP entry %+v", existing) - updatesFailed = true - } - } else { - logCxt.WithField("neighbor", existing).Info("Removed old neighbor ARP entry") - } - } - } - - // For each expected target, ensure that it is programmed. If the value has changed since last programming, this - // will update it. - for _, target := range expectedTargets { - if err = r.ensureL2Dataplane(linkAttrs, target); err != nil { - logCxt.WithError(err).Warnf("Failed to sync L2 dataplane for interface") - updatesFailed = true - continue - } - } - - if updatesFailed { - r.nl.CloseHandle() // Defensive: force a netlink reconnection next time. - - // Recheck whether the interface exists so we don't produce spammy logs during - // interface removal. - return r.filterErrorByIfaceState(ifaceName, UpdateFailed, UpdateFailed, false) - } - - return nil -} - -func (r *RouteTable) ensureL2Dataplane(linkAttrs *netlink.LinkAttrs, target L2Target) error { - // For each L2 entry we need to program, program it. - // Add a static ARP entry. - a := &netlink.Neigh{ - LinkIndex: linkAttrs.Index, - State: netlink.NUD_PERMANENT, - Type: syscall.RTN_UNICAST, - IP: target.GW.AsNetIP(), - HardwareAddr: target.VTEPMAC, - } - if err := netlink.NeighSet(a); err != nil { - return err - } - log.WithField("entry", a).Debug("Programmed ARP") - - // Add a FDB entry for this neighbor. - n := &netlink.Neigh{ - LinkIndex: linkAttrs.Index, - State: netlink.NUD_PERMANENT, - Family: syscall.AF_BRIDGE, - Flags: netlink.NTF_SELF, - IP: target.IP.AsNetIP(), - HardwareAddr: target.VTEPMAC, - } - if err := netlink.NeighSet(n); err != nil { - return err - } - log.WithField("entry", n).Debug("Programmed FDB") - return nil -} - // startConntrackDeletion starts the deletion of conntrack entries for the given CIDR in the background. Pending // deletions are tracked in the pendingConntrackCleanups map so we can block waiting for them later. // diff --git a/felix/routetable/route_table_test.go b/felix/routetable/route_table_test.go index c2496fc3dc1..c97b0980ab6 100644 --- a/felix/routetable/route_table_test.go +++ b/felix/routetable/route_table_test.go @@ -61,7 +61,6 @@ var _ = Describe("RouteTable v6", func() { []string{"^cali.*"}, 6, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -130,7 +129,6 @@ var _ = Describe("RouteTable", func() { []string{"^cali.*"}, 4, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -277,7 +275,6 @@ var _ = Describe("RouteTable", func() { []string{"^cali.*"}, 4, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -405,7 +402,6 @@ var _ = Describe("RouteTable", func() { []string{"^cali.*"}, 4, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -1062,7 +1058,6 @@ var _ = Describe("RouteTable (main table)", func() { []string{"^cali.*"}, 4, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -1165,7 +1160,6 @@ var _ = Describe("RouteTable (table 100)", func() { []string{"^cali$", InterfaceNone}, // exact interface match 4, dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, @@ -1458,7 +1452,6 @@ var _ = Describe("Tests to verify ip version is policed", func() { []string{"^cali$", InterfaceNone}, 5, // invalid IP version dataplane.NewMockNetlink, - false, 10*time.Second, dataplane.AddStaticArpEntry, dataplane, diff --git a/felix/vxlanfdb/vxlan_fdb.go b/felix/vxlanfdb/vxlan_fdb.go new file mode 100644 index 00000000000..3b4e8bd1eb7 --- /dev/null +++ b/felix/vxlanfdb/vxlan_fdb.go @@ -0,0 +1,355 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vxlanfdb + +import ( + "fmt" + "net" + "time" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/projectcalico/calico/felix/deltatracker" + "github.com/projectcalico/calico/felix/environment" + "github.com/projectcalico/calico/felix/ifacemonitor" + "github.com/projectcalico/calico/felix/ip" + "github.com/projectcalico/calico/felix/netlinkshim" + "github.com/projectcalico/calico/felix/netlinkshim/handlemgr" +) + +type VTEP struct { + // HostIP is the node's real IP address; the IP that we send the + // VXLAN packets to. + HostIP ip.Addr + // TunnelIP is the IP of the remote tunnel device, which we use as + // a gateway for the remote workloads.. + TunnelIP ip.Addr + // TunnelMAC is the MAC address of the remote tunnel device. + TunnelMAC net.HardwareAddr +} + +type VXLANFDB struct { + ifaceName string + ifIndex int + arpEntries *deltatracker.DeltaTracker[string, ipMACMapping] + fdbEntries *deltatracker.DeltaTracker[string, ipMACMapping] + logCxt *log.Entry + resyncPending bool + logNextSuccess bool + nl *handlemgr.HandleManager + family int +} + +type ipMACMapping struct { + IP ip.Addr + MAC net.HardwareAddr +} + +func New(family int, ifaceName string, featureDetector environment.FeatureDetectorIface, netlinkTimeout time.Duration) *VXLANFDB { + switch family { + case unix.AF_INET, unix.AF_INET6: + default: + log.WithField("family", family).Panic("Unknown family") + } + f := VXLANFDB{ + family: family, + ifaceName: ifaceName, + arpEntries: deltatracker.New[string, ipMACMapping](), + fdbEntries: deltatracker.New[string, ipMACMapping](), + logCxt: log.WithFields(log.Fields{ + "iface": ifaceName, + "family": family, + }), + resyncPending: true, + logNextSuccess: true, + nl: handlemgr.NewHandleManager( + featureDetector, + handlemgr.WithSocketTimeout(netlinkTimeout), + // The Netlink library doesn't seem to be able to list + // both types of neighbors in strict mode. + handlemgr.WithStrictModeOverride(false), + ), + } + return &f +} + +func (r *VXLANFDB) OnIfaceStateChanged(ifaceName string, state ifacemonitor.State) { + if ifaceName != r.ifaceName { + return + } + if state == ifacemonitor.StateUp { + r.logCxt.Debug("VXLAN device came up, doing a resync.") + r.resyncPending = true + } else { + r.logCxt.WithField("state", state).Debug("VXLAN device changed state.") + } +} + +func (r *VXLANFDB) QueueResync() { + r.resyncPending = true +} + +func (r *VXLANFDB) SetVTEPs(vteps []VTEP) { + r.arpEntries.Desired().DeleteAll() + r.fdbEntries.Desired().DeleteAll() + for _, t := range vteps { + macStr := t.TunnelMAC.String() + // Add an ARP entry, for the remote tunnel IP. This allows the + // kernel to calculate the inner ethernet header without doing a + // broadcast ARP to all VXLAN peers. + r.arpEntries.Desired().Set(macStr, ipMACMapping{ + IP: t.TunnelIP, + MAC: t.TunnelMAC, + }) + // Add an FDB entry. While this is also a MAC/IP tuple, it tells + // the kernel something very different! The FDB entry tells the + // kernel that, if it needs to send traffic to the VTEP MAC, it + // should send the VXLAN packet to a particular host's real IP + // address. + r.fdbEntries.Desired().Set(macStr, ipMACMapping{ + MAC: t.TunnelMAC, + IP: t.HostIP, + }) + } +} + +func (r *VXLANFDB) Apply() error { + debug := log.IsLevelEnabled(log.DebugLevel) + nl, err := r.nl.Handle() + if err != nil { + return fmt.Errorf("failed to connect to netlink") + } + + if r.resyncPending { + if err := r.resync(nl); err != nil { + return err + } + r.resyncPending = false + } + defer func() { + if r.resyncPending { + r.logNextSuccess = true + } + }() + + if r.ifIndex == 0 { + return ErrLinkDown + } + + errs := map[string]error{} + r.arpEntries.PendingUpdates().Iter(func(macStr string, entry ipMACMapping) deltatracker.IterAction { + if debug { + log.WithField("entry", entry).Debug("Adding ARP/NDP entry.") + } + a := &netlink.Neigh{ + Family: r.family, + LinkIndex: r.ifIndex, + State: netlink.NUD_PERMANENT, + Type: unix.RTN_UNICAST, + IP: entry.IP.AsNetIP(), + HardwareAddr: entry.MAC, + } + if err := nl.NeighSet(a); err != nil { + if len(errs) == 0 { + log.WithError(err).WithField("entry", entry).Warn("Failed to add ARP entry, only logging first instance.") + } + errs[macStr] = err + return deltatracker.IterActionNoOp + } + return deltatracker.IterActionUpdateDataplane + }) + if len(errs) > 0 { + log.WithField("numErrors", len(errs)).Warn("Failed to add some ARP entries") + r.resyncPending = true + r.nl.CloseHandle() // Defensive: force a netlink reconnection next time. + clear(errs) + } + + r.arpEntries.PendingDeletions().Iter(func(macStr string) deltatracker.IterAction { + entry, _ := r.arpEntries.Dataplane().Get(macStr) + if debug { + log.WithField("entry", entry).Debug("Deleting ARP entry.") + } + a := &netlink.Neigh{ + Family: r.family, + LinkIndex: r.ifIndex, + Type: unix.RTN_UNICAST, + IP: entry.IP.AsNetIP(), + HardwareAddr: entry.MAC, + } + if err := nl.NeighDel(a); err != nil && !errors.Is(err, unix.ENOENT) { + if len(errs) == 0 { + log.WithError(err).WithField("entry", entry).Warn("Failed to delete ARP entry, only logging first instance.") + } + errs[macStr] = err + } + return deltatracker.IterActionUpdateDataplane + }) + if len(errs) > 0 { + log.WithField("numErrors", len(errs)).Warn("Failed to remove some ARP entries") + r.resyncPending = true + r.nl.CloseHandle() // Defensive: force a netlink reconnection next time. + clear(errs) + } + + r.fdbEntries.PendingUpdates().Iter(func(macStr string, entry ipMACMapping) deltatracker.IterAction { + if debug { + log.WithField("entry", entry).Debug("Adding FDB entry.") + } + a := &netlink.Neigh{ + LinkIndex: r.ifIndex, + State: netlink.NUD_PERMANENT, + Family: unix.AF_BRIDGE, + Flags: netlink.NTF_SELF, + IP: entry.IP.AsNetIP(), + HardwareAddr: entry.MAC, + } + if err := nl.NeighSet(a); err != nil { + if len(errs) == 0 { + log.WithError(err).WithField("entry", entry).Warn("Failed to add FDB entry, only logging first instance.") + } + errs[macStr] = err + return deltatracker.IterActionNoOp + } + return deltatracker.IterActionUpdateDataplane + }) + if len(errs) > 0 { + log.WithField("numErrors", len(errs)).Warn("Failed to add some FDB entries") + r.resyncPending = true + r.nl.CloseHandle() // Defensive: force a netlink reconnection next time. + clear(errs) + } + + r.fdbEntries.PendingDeletions().Iter(func(macStr string) deltatracker.IterAction { + entry, _ := r.fdbEntries.Dataplane().Get(macStr) + if log.IsLevelEnabled(log.DebugLevel) { + log.WithField("entry", entry).Debug("Deleting FDB entry.") + } + a := &netlink.Neigh{ + LinkIndex: r.ifIndex, + State: netlink.NUD_PERMANENT, + Family: unix.AF_BRIDGE, + Flags: netlink.NTF_SELF, + IP: entry.IP.AsNetIP(), + HardwareAddr: entry.MAC, + } + if err := nl.NeighDel(a); err != nil && !errors.Is(err, unix.ENOENT) { + if len(errs) == 0 { + log.WithError(err).WithField("entry", entry).Warn("Failed to delete FDB entry, only logging first instance.") + } + errs[macStr] = err + } + return deltatracker.IterActionUpdateDataplane + }) + if len(errs) > 0 { + log.WithField("numErrors", len(errs)).Warn("Failed to remove some ARP entries") + r.resyncPending = true + r.nl.CloseHandle() // Defensive: force a netlink reconnection next time. + clear(errs) + } + + if !r.resyncPending && r.logNextSuccess { + r.logCxt.Info("VXLAN FDB now in sync.") + r.logNextSuccess = false + } + + return nil +} + +var ErrLinkDown = fmt.Errorf("VXLAN device is down") + +func (r *VXLANFDB) resync(nl netlinkshim.Interface) error { + // Refresh the link ID. If the VXLAN device gets recreated then + // this can change. + link, err := nl.LinkByName(r.ifaceName) + if err != nil { + r.resyncPending = false // OnIfaceStateChanged will trigger a resync when iface appears. + return fmt.Errorf("failed to get interface: %w", err) + } + if !ifacemonitor.LinkIsOperUp(link) { + r.resyncPending = false // OnIfaceStateChanged will trigger a resync when iface appears. + return ErrLinkDown + } + r.ifIndex = link.Attrs().Index + + // Refresh the neighbours. + existingNeigh, err := nl.NeighList(r.ifIndex, unix.AF_INET) + if err != nil { + r.logCxt.WithError(err).Error("Failed to list neighbors") + return fmt.Errorf("failed to list neighbors: %w", err) + } + existingFDB, err := nl.NeighList(r.ifIndex, unix.AF_BRIDGE) + if err != nil { + r.logCxt.WithError(err).Error("Failed to list FDB entries") + return fmt.Errorf("failed to list FDB entries: %w", err) + } + err = r.arpEntries.Dataplane().ReplaceAllIter(func(f func(macStr string, v ipMACMapping)) error { + for _, n := range existingNeigh { + if n.HardwareAddr == nil { + continue + } + if n.State&unix.NUD_PERMANENT == 0 { + // We only manage static entries so ignore this one. + continue + } + hwAddrStr := n.HardwareAddr.String() + if log.IsLevelEnabled(log.DebugLevel) { + log.WithFields(log.Fields{ + "mac": hwAddrStr, + "ip": n.IP.String(), + }).Debug("Loaded ARP entry from kernel.") + } + f(hwAddrStr, ipMACMapping{ + IP: ip.FromNetIP(n.IP), + MAC: n.HardwareAddr, + }) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to update arpEntries: %w", err) + } + err = r.fdbEntries.Dataplane().ReplaceAllIter(func(f func(k string, v ipMACMapping)) error { + for _, n := range existingFDB { + if n.HardwareAddr == nil { + continue + } + if n.State&unix.NUD_PERMANENT == 0 { + // We only manage static entries so ignore this one. + continue + } + hwAddrStr := n.HardwareAddr.String() + if log.IsLevelEnabled(log.DebugLevel) { + log.WithFields(log.Fields{ + "mac": hwAddrStr, + "ip": n.IP.String(), + }).Debug("Loaded FDB entry from kernel.") + } + f(hwAddrStr, ipMACMapping{ + IP: ip.FromNetIP(n.IP), + MAC: n.HardwareAddr, + }) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to update fdbEntries: %w", err) + } + + return nil +} diff --git a/felix/wireguard/wireguard.go b/felix/wireguard/wireguard.go index 9db5f8b7360..a7185c82b4f 100644 --- a/felix/wireguard/wireguard.go +++ b/felix/wireguard/wireguard.go @@ -233,7 +233,6 @@ func NewWithShims( []string{"^" + interfaceName + "$", routetable.InterfaceNone}, ipVersion, newRoutetableNetlink, - false, // vxlan netlinkTimeout, func(cidr ip.CIDR, destMAC net.HardwareAddr, ifaceName string) error { return nil }, // addStaticARPEntry &noOpConnTrack{},