Send IPv4 GARP and IPv6 Unsolicited NA in "cmdAdd"

In "cmdAdd", SRIOV-CNI would construct and send IPv4 Gratuitous ARP
and/or Unsolicited Neighbor Advertisement depending on the IP
addresses configured by IPAM. The reason why this change is needed
is for the scenario when an IP address is reused by IPAM with
different interfaces (with different link-layer addresses). This can
occur when pods are deleted and created. For performance reasons,
sending of GARP and/or Unsolicited NA would update invalid ARP/Neighbor
caches in other neighbors/nodes.

Also we set IPv4 ARP Notify and IPv6 Neighbor Discovery Notify in sysfs
for each interface. This will send GARP and/or Unsolicited NA when the
interface is either brought up or the link-layer address changes. This
is useful in cases where an application reenables the interface or the
MAC address configuration is changed.

Some new packages were added, thus go.mod and go.sum were modified
accordingly.

Mocked PciUtils for sriov tests since sriov.go would call PciUtils to
set IPv4 ARP Notify and IPv6 Neighbor Discovery.

Fixes #177

Signed-off-by: William Zhao <wizhao@redhat.com>

cmd/sriov/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"fmt"
+	"net"
 	"runtime"
 
 	"github.com/containernetworking/cni/pkg/skel"
@@ -144,6 +145,54 @@ func cmdAdd(args *skel.CmdArgs) error {
 		result = newResult
 	}
 
+	/* After IPAM configuration is done, the following needs to handle the case of an IP address being reused by a different pods.
+	 * This is achieved by sending Gratuitous ARPs and/or Unsolicited Neighbor Advertisements unconditionally.
+	 * Although we set arp_notify and ndisc_notify unconditionally on the interface (please see EnableArpAndNdiscNotify()), the kernel
+	 * only sends GARPs/Unsolicited NA when the interface goes from down to up, or when the link-layer address changes on the interfaces.
+	 * These scenarios are perfectly valid and recommended to be enabled for optimal network performance.
+	 * However for our specific case, which the kernel is unaware of, is the reuse of IP addresses across pods where each pod has a different
+	 * link-layer address for it's SRIOV interface. The ARP/Neighbor cache residing in neighbors would be invalid if an IP address is reused.
+	 * In order to update the cache, the GARP/Unsolicited NA packets should be sent for performance reasons. Otherwise, the neighbors
+	 * may be sending packets with the incorrect link-layer address. Eventually, most network stacks would send ARPs and/or Neighbor
+	 * Solicitation packets when the connection is unreachable. This would correct the invalid cache; however this may take a significant
+	 * amount of time to complete.
+	 */
+	if !netConf.DPDKMode {
+		err = netns.Do(func(_ ns.NetNS) error {
+			// Retrieve the interface name in the container.
+			contVeth, err := net.InterfaceByName(args.IfName)
+			if err != nil {
+				return fmt.Errorf("failed to look up interface %q: %v", args.IfName, err)
+			}
+
+			// Check that we have a valid hardware MAC address.
+			if len(contVeth.HardwareAddr) != 6 {
+				return fmt.Errorf("error invalid Ethernet MAC address: %q", contVeth.HardwareAddr)
+			}
+
+			// For all the IP addresses assigned by IPAM, we will sent either a GARP (IPv4) or Unsolicited NA (IPv6).
+			for _, ipc := range result.IPs {
+				var err error
+				if ipc.Address.IP.To4() == nil { // IPv6
+					/* As per RFC 4861, sending unsolicited neighbor advertisements should be considered as a performance
+					 * optimization. It does not reliably update caches in all nodes. The Neighbor Unreachability Detection
+					 * algorithm is more reliable although it may take slightly longer to update.
+					 */
+					err = utils.SendUnsolicitedNeighborAdvertisement(ipc.Address.IP, *contVeth)
+				} else { // IPv4
+					err = utils.SendGratuitousArp(ipc.Address.IP, *contVeth)
+				}
+				if err != nil {
+					return fmt.Errorf("error sending messages for ip %s on interface %q: %v", ipc.Address.IP.String(), args.IfName, err)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+
 	// Cache NetConf for CmdDel
 	if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
 		return fmt.Errorf("error saving NetConf %q", err)

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a
 	github.com/stretchr/testify v1.4.0
 	github.com/vishvananda/netlink v1.0.1-0.20190924205540-07ace697bea4
+	golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b
 )
 
 require (
@@ -18,6 +19,6 @@ require (
 	github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 // indirect
 	github.com/stretchr/objx v0.1.0 // indirect
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

go.sum

@@ -43,9 +43,15 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrB
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b h1:3ogNYyK4oIQdIKzTu68hQrr4iuVxF3AxKl9Aj/eDrw0=
+golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/sriov/sriov.go

@@ -105,6 +105,7 @@ type pciUtils interface {
 	getSriovNumVfs(ifName string) (int, error)
 	getVFLinkNamesFromVFID(pfName string, vfID int) ([]string, error)
 	getPciAddress(ifName string, vf int) (string, error)
+	enableArpAndNdiscNotify(ifName string) error
 }
 
 type pciUtilsImpl struct{}
@@ -121,6 +122,10 @@ func (p *pciUtilsImpl) getPciAddress(ifName string, vf int) (string, error) {
 	return utils.GetPciAddress(ifName, vf)
 }
 
+func (p *pciUtilsImpl) enableArpAndNdiscNotify(ifName string) error {
+	return utils.EnableArpAndNdiscNotify(ifName)
+}
+
 // Manager provides interface invoke sriov nic related operations
 type Manager interface {
 	SetupVF(conf *sriovtypes.NetConf, podifName string, cid string, netns ns.NetNS) (string, error)
@@ -192,7 +197,12 @@ func (s *sriovManager) SetupVF(conf *sriovtypes.NetConf, podifName string, cid s
 			return fmt.Errorf("error setting container interface name %s for %s", linkName, tempName)
 		}
 
-		// 6. Bring IF up in Pod netns
+		// 6. Enable IPv4 ARP notify and IPv6 Network Discovery notify
+		if err := s.utils.enableArpAndNdiscNotify(podifName); err != nil {
+			return fmt.Errorf("failed to enable arp_notify for interface name: %s, %q", podifName, err)
+		}
+
+		// 7. Bring IF up in Pod netns
 		if err := s.nLink.LinkSetUp(linkObj); err != nil {
 			return fmt.Errorf("error bringing interface up in container ns: %q", err)
 		}