[12.0 stable] Enhance LinuxCollector to support detecting multiple app VIF IPs

docs/APP-CONNECTIVITY.md

@@ -207,6 +207,25 @@ to retrieve cloud-init configuration, obtain information from EVE (e.g. device U
 hostname, uplink IP address) or to download [patch envelopes](PATCH-ENVELOPES.md).
 More information about metadata server can be found in [ECO-METADATA.md](ECO-METADATA.md).
 
+#### IPAM
+
+Every Local Network Instance must be configured with an IPv4 network subnet and an IP
+range within this subnet for automatic IP allocations. Host IP addresses from this subnet
+that do not fall within the IP range are available for manual assignment.
+
+Whether an IP address is selected manually or dynamically assigned by EVE from the configured
+IP range, an internal DHCP server is used to distribute these IP addresses to applications.
+Container applications are deployed inside a "shim VM", which EVE prepares, ensuring that
+a DHCP client is running for every virtual interface connected to a network instance
+This guarantees that the IP address is received and applied before the application starts.
+In contrast, VM applications are responsible for starting their own DHCP client and applying
+the received IP addresses.
+
+Regardless of the application type, EVE does not automatically assume that the allocated
+IP address is actually in use. Instead, it monitors the set of IP leases granted by the internal
+DHCP server and updates the set of application IP addresses in the published info messages
+accordingly.
+
 ### Switch Network Instance
 
 Switch Network Instance is a simple L2-only bridge between connected applications and
@@ -232,6 +251,52 @@ inbound ACL rules are that much more important.
 Metadata HTTP server is run for switch network instance only if it has uplink port with
 an IP address.
 
+#### IP address detection
+
+Unlike a Local Network Instance, a switch network instance is configured without any IP
+configuration, and EVE does not run an internal DHCP server. Instead, if IP connectivity
+is required, IP addresses must be assigned statically within the connected applications
+or provided by an external DHCP server or another application offering DHCP services.
+
+Since EVE is not in control of IP address allocations and leases, it must monitor application
+traffic to learn which IP addresses are being used and report this information to the controller.
+
+In the case of an external DHCP server (IPv4), EVE captures the DHCPACK packet from the server,
+which confirms the leased IP address. Because EVE manages MAC address allocations, it knows
+the MAC address of every application's virtual interface (VIF). It can then map the CHADDR
+(Client Hardware Address) attribute to the corresponding application VIF and learn the assigned
+IP address from the YIADDR (your, i.e. client, IP Address) attribute. Additionally, EVE reads
+the DHCP option 51 (Lease Time), if available, to determine how long the leased IP address
+is valid. If EVE does not observe an IP renewal within this period, it assumes that the IP address
+is no longer in use and reports this change to the controller.
+
+For statically assigned IPv4 addresses, EVE captures both ARP reply and request packets to learn
+the application VIF IP assignment from either Sender IP + MAC or Target IP + MAC attribute
+pairs. Since ARP cache entries have a limited lifetime — typically around 2 minutes — EVE expects
+to see at least one ARP packet for every assigned IP within a 10-minute window (this is not
+configurable). If no ARP packet is observed within this period for a previously detected IP
+assignment, EVE assumes that the IP address has been removed and reports this change to
+the controller. EVE also captures ARP packets for IP addresses configured via DHCP, but these
+are ignored as the information from the previously captured DHCPACK takes precedence.
+Note that ARP-based IP detection is enabled by default but can be disabled by setting
+the configuration item `network.switch.enable.arpsnoop` to `false`. Change in this config
+options will apply to already deployed switch network instances.
+
+For an external DHCPv6 server, EVE captures DHCPv6 REPLY messages. It learns the target MAC
+address from the DUID option (Client Identifier, option code 1), while the IPv6 address
+and its valid lifetime are provided by the IA Address (option code 5).
+
+To learn IPv6 addresses assigned using SLAAC (Stateless Address Auto Configuration),
+EVE captures unicast ICMPv6 Neighbor Solicitation messages. These are sent from the interface
+with the assigned IPv6 address to check if the address is free or already in use by another
+host — a process known as Duplicate Address Detection (DAD). The ICMPv6 packet sent to detect
+IP duplicates for a particular VIF IP will have the VIF MAC address as the source address
+in the Ethernet header. EVE uses this, along with the "Target Address" field from the ICMPv6
+header, to identify the assigned IPv6 address.
+
+EVE is capable of detecting multiple IPs assigned to the same VIF MAC address. This is commonly
+seen when applications use VLAN sub-interfaces, which share the parent interface's MAC address.
+
 ### Uplink Port
 
 Network instances can be configured with an "uplink" network adapter, which will be used to provide

docs/CONFIG-PROPERTIES.md

@@ -53,7 +53,7 @@
 | netdump.topic.maxcount | integer | 10 | maximum number of netdumps that can be published for each topic. The oldest netdump is unpublished should a new netdump exceed the limit.
 | netdump.downloader.with.pcap | boolean | false | include packet captures inside netdumps for download requests. However, even if enabled, TCP segments carrying non-empty payload (i.e. content which is being downloaded) are excluded and the overall PCAP size is limited to 64MB. |
 | netdump.downloader.http.with.fieldvalue | boolean | false | include HTTP header field values in captured network traces for download requests (beware: may contain secrets, such as datastore credentials). |
-| network.switch.enable.arpsnoop | boolean | true | enable ARP Snooping on switch Network Instance, may need a device reboot to take effect |
+| network.switch.enable.arpsnoop | boolean | true | enable ARP Snooping on switch Network Instances |
 | wwan.query.visible.providers | bool | false | enable to periodically (once per hour) query the set of visible cellular service providers and publish them under WirelessStatus (for every modem) |
 | network.local.legacy.mac.address | bool | false | enables legacy MAC address generation for local network instances for those EVE nodes where changing MAC addresses in applications will lead to incorrect network configuration |
 

pkg/pillar/cmd/zedagent/handleconfig.go

@@ -1200,8 +1200,22 @@ func updateLocalServerMap(getconfigCtx *getconfigContext, localServerURL string)
 				continue
 			}
 			if localServerIP != nil {
-				// check if the defined IP of localServer equals the allocated IP of the app
-				if adapterStatus.AllocatedIPv4Addr.Equal(localServerIP) {
+				// Check if the defined IP of localServer equals one of the IPs
+				// allocated to the app.
+				var matchesApp bool
+				for _, ip := range adapterStatus.AssignedAddresses.IPv4Addrs {
+					if ip.Address.Equal(localServerIP) {
+						matchesApp = true
+						break
+					}
+				}
+				for _, ip := range adapterStatus.AssignedAddresses.IPv6Addrs {
+					if ip.Address.Equal(localServerIP) {
+						matchesApp = true
+						break
+					}
+				}
+				if matchesApp {
 					srvAddr := localServerAddr{
 						localServerAddr: localServerURL,
 						bridgeIP:        adapterStatus.BridgeIPAddr,

pkg/pillar/cmd/zedagent/handlemetrics.go

@@ -1108,16 +1108,18 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string,
 		for _, ifname := range ifNames {
 			networkInfo := new(info.ZInfoNetwork)
 			networkInfo.LocalName = *proto.String(ifname)
-			ipv4Addr, ipv6Addrs, allocated, macAddr, ipAddrMismatch :=
-				getAppIP(ctx, aiStatus, ifname)
-			if ipv4Addr != nil {
-				networkInfo.IPAddrs = append(networkInfo.IPAddrs, ipv4Addr.String())
+			addrs, hasIPv4Addr, macAddr, ipAddrMismatch :=
+				getAppIPs(ctx, aiStatus, ifname)
+			for _, ipv4Addr := range addrs.IPv4Addrs {
+				networkInfo.IPAddrs = append(networkInfo.IPAddrs,
+					ipv4Addr.Address.String())
 			}
-			for _, ipv6Addr := range ipv6Addrs {
-				networkInfo.IPAddrs = append(networkInfo.IPAddrs, ipv6Addr.String())
+			for _, ipv6Addr := range addrs.IPv6Addrs {
+				networkInfo.IPAddrs = append(networkInfo.IPAddrs,
+					ipv6Addr.Address.String())
 			}
 			networkInfo.MacAddr = *proto.String(macAddr.String())
-			networkInfo.Ipv4Up = allocated
+			networkInfo.Ipv4Up = hasIPv4Addr
 			networkInfo.IpAddrMisMatch = ipAddrMismatch
 			name := appIfnameToName(aiStatus, ifname)
 			log.Tracef("app %s/%s localName %s devName %s",
@@ -1563,21 +1565,22 @@ func sendMetricsProtobuf(ctx *getconfigContext,
 
 // Use the ifname/vifname to find the AppNetAdapter status
 // and from there the (ip, allocated, mac) addresses for the app
-func getAppIP(ctx *zedagentContext, aiStatus *types.AppInstanceStatus,
-	vifname string) (net.IP, []net.IP, bool, net.HardwareAddr, bool) {
+func getAppIPs(ctx *zedagentContext, aiStatus *types.AppInstanceStatus,
+	vifname string) (types.AssignedAddrs, bool, net.HardwareAddr, bool) {
 
 	log.Tracef("getAppIP(%s, %s)", aiStatus.Key(), vifname)
 	for _, adapterStatus := range aiStatus.AppNetAdapters {
 		if adapterStatus.VifUsed != vifname {
 			continue
 		}
-		log.Tracef("getAppIP(%s, %s) found AppIP v4: %s, v6: %s, ipv4 assigned %v mac %s",
-			aiStatus.Key(), vifname, adapterStatus.AllocatedIPv4Addr,
-			adapterStatus.AllocatedIPv6List, adapterStatus.IPv4Assigned, adapterStatus.Mac)
-		return adapterStatus.AllocatedIPv4Addr, adapterStatus.AllocatedIPv6List, adapterStatus.IPv4Assigned,
+		log.Tracef("getAppIP(%s, %s) found AppIPs v4: %v, v6: %v, ipv4 assigned %v mac %s",
+			aiStatus.Key(), vifname, adapterStatus.AssignedAddresses.IPv4Addrs,
+			adapterStatus.AssignedAddresses.IPv6Addrs, adapterStatus.IPv4Assigned,
+			adapterStatus.Mac)
+		return adapterStatus.AssignedAddresses, adapterStatus.IPv4Assigned,
 			adapterStatus.Mac, adapterStatus.IPAddrMisMatch
 	}
-	return nil, nil, false, nil, false
+	return types.AssignedAddrs{}, false, nil, false
 }
 
 func createVolumeInstanceMetrics(ctx *zedagentContext, reportMetrics *metrics.ZMetricMsg) {

pkg/pillar/cmd/zedagent/handlenetworkinstance.go

@@ -8,7 +8,6 @@ package zedagent
 import (
 	"bytes"
 	"fmt"
-	"net"
 	"time"
 
 	"github.com/golang/protobuf/ptypes/timestamp"
@@ -109,11 +108,13 @@ func prepareAndPublishNetworkInstanceInfoMsg(ctx *zedagentContext,
 		for mac, addrs := range status.IPAssignments {
 			assignment := new(zinfo.ZmetIPAssignmentEntry)
 			assignment.MacAddress = mac
-			if !addrs.IPv4Addr.Equal(net.IP{}) {
-				assignment.IpAddress = append(assignment.IpAddress, addrs.IPv4Addr.String())
+			for _, assignedIP := range addrs.IPv4Addrs {
+				assignment.IpAddress = append(assignment.IpAddress,
+					assignedIP.Address.String())
 			}
-			for _, ip := range addrs.IPv6Addrs {
-				assignment.IpAddress = append(assignment.IpAddress, ip.String())
+			for _, assignedIP := range addrs.IPv6Addrs {
+				assignment.IpAddress = append(assignment.IpAddress,
+					assignedIP.Address.String())
 			}
 			info.IpAssignments = append(info.IpAssignments, assignment)
 		}

pkg/pillar/cmd/zedrouter/appnetwork.go

@@ -62,7 +62,8 @@ func (z *zedrouter) updateVIFsForStateCollecting(
 		}
 		_, vifs, err := z.getArgsForNIStateCollecting(network)
 		if err == nil {
-			err = z.niStateCollector.UpdateCollectingForNI(*netConfig, vifs)
+			err = z.niStateCollector.UpdateCollectingForNI(*netConfig, vifs,
+				z.enableArpSnooping)
 		}
 		if err != nil {
 			z.log.Error(err)
@@ -168,10 +169,7 @@ func (z *zedrouter) updateNIStatusAfterAppNetworkActivate(status *types.AppNetwo
 		netInstStatus.AddVif(z.log, adapterStatus.Vif, adapterStatus.Mac,
 			status.UUIDandVersion.UUID)
 		netInstStatus.IPAssignments[adapterStatus.Mac.String()] =
-			types.AssignedAddrs{
-				IPv4Addr:  adapterStatus.AllocatedIPv4Addr,
-				IPv6Addrs: adapterStatus.AllocatedIPv6List,
-			}
+			adapterStatus.AssignedAddresses
 		z.publishNetworkInstanceStatus(netInstStatus)
 	}
 }

pkg/pillar/cmd/zedrouter/ipam.go

@@ -120,13 +120,15 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 	// Lookup to see if it is already allocated.
 	if ipAddr == nil {
 		addrs := niStatus.IPAssignments[adapterStatus.Mac.String()]
-		if !isEmptyIP(addrs.IPv4Addr) {
-			z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): found IP %v for MAC %v",
-				networkID, appID, addrs.IPv4Addr, adapterStatus.Mac)
-			ipAddr = addrs.IPv4Addr
+		ipAddr = addrs.GetInternallyLeasedIPv4Addr()
+		if ipAddr != nil {
+			z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): "+
+				"found EVE-allocated IP %v for MAC %v", networkID, appID, ipAddr,
+				adapterStatus.Mac)
 		}
 	}
 
+	var newlyAllocated bool
 	if ipAddr == nil {
 		// Allocate IP address dynamically.
 		// Get the app number for the AppNetAdapter entry.
@@ -140,6 +142,7 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 		}
 		// Pick an IP address from the subnet.
 		ipAddr = netutils.AddToIP(niStatus.DhcpRange.Start, appNum)
+		newlyAllocated = true
 		// Check if the address falls into the Dhcp Range.
 		if !niStatus.DhcpRange.Contains(ipAddr) {
 			err := fmt.Errorf("no free IP addresses in DHCP range(%v, %v)",
@@ -162,12 +165,19 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 		}
 	}
 	// Later will be overwritten with addresses received from nistate.Collector,
-	// which snoops DHCP traffic and watches DNS server leases to learn the *actual*
+	// which snoops DHCP traffic and watches DHCP server leases to learn the *actual*
 	// IP address assignments.
-	addrs := niStatus.IPAssignments[adapterStatus.Mac.String()] // preserve IPv6 addresses
-	addrs.IPv4Addr = ipAddr
-	niStatus.IPAssignments[adapterStatus.Mac.String()] = addrs
-	z.publishNetworkInstanceStatus(niStatus)
+	if newlyAllocated {
+		// Preserve other IPv4 and IPv6 addresses.
+		addrs := niStatus.IPAssignments[adapterStatus.Mac.String()]
+		addrs.IPv4Addrs = append(addrs.IPv4Addrs,
+			types.AssignedAddr{
+				Address:    ipAddr,
+				AssignedBy: types.AddressSourceInternalDHCP,
+			})
+		niStatus.IPAssignments[adapterStatus.Mac.String()] = addrs
+		z.publishNetworkInstanceStatus(niStatus)
+	}
 	z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): allocated IP %v for MAC %v",
 		networkID, appID, ipAddr, adapterStatus.Mac)
 	return ipAddr, nil
@@ -188,16 +198,16 @@ func (z *zedrouter) recordAssignedIPsToAdapterStatus(adapter *types.AppNetAdapte
 		z.removeAssignedIPsFromAdapterStatus(adapter)
 		return
 	}
-	adapter.AllocatedIPv4Addr = vifAddrs.IPv4Addr
-	if !isEmptyIP(adapter.AppIPAddr) &&
-		!adapter.AppIPAddr.Equal(adapter.AllocatedIPv4Addr) {
-		// Config and status do not match.
-		adapter.IPAddrMisMatch = true
-	} else {
-		adapter.IPAddrMisMatch = false
+	adapter.AssignedAddresses = vifAddrs.AssignedAddrs
+	adapter.IPAddrMisMatch = false
+	if !isEmptyIP(adapter.AppIPAddr) {
+		leasedIP := adapter.AssignedAddresses.GetInternallyLeasedIPv4Addr()
+		if !adapter.AppIPAddr.Equal(leasedIP) {
+			// Config and status do not match.
+			adapter.IPAddrMisMatch = true
+		}
 	}
-	adapter.AllocatedIPv6List = vifAddrs.IPv6Addrs
-	adapter.IPv4Assigned = !isEmptyIP(vifAddrs.IPv4Addr)
+	adapter.IPv4Assigned = len(vifAddrs.IPv4Addrs) > 0
 }
 
 func (z *zedrouter) removeAssignedIPsFromAppNetStatus(status *types.AppNetworkStatus) {
@@ -208,8 +218,8 @@ func (z *zedrouter) removeAssignedIPsFromAppNetStatus(status *types.AppNetworkSt
 }
 
 func (z *zedrouter) removeAssignedIPsFromAdapterStatus(adapterStatus *types.AppNetAdapterStatus) {
-	adapterStatus.AllocatedIPv6List = nil
-	adapterStatus.AllocatedIPv4Addr = nil
+	adapterStatus.AssignedAddresses.IPv4Addrs = nil
+	adapterStatus.AssignedAddresses.IPv6Addrs = nil
 	adapterStatus.IPAddrMisMatch = false
 	adapterStatus.IPv4Assigned = false
 }

pkg/pillar/cmd/zedrouter/networkinstance.go

@@ -230,7 +230,7 @@ func (z *zedrouter) doUpdateActivatedNetworkInstance(config types.NetworkInstanc
 	z.processNIReconcileStatus(niRecStatus, status)
 	_, vifs, err := z.getArgsForNIStateCollecting(config.UUID)
 	if err == nil {
-		err = z.niStateCollector.UpdateCollectingForNI(config, vifs)
+		err = z.niStateCollector.UpdateCollectingForNI(config, vifs, z.enableArpSnooping)
 	}
 	if err != nil {
 		z.log.Error(err)

pkg/pillar/cmd/zedrouter/pubsubhandlers.go

@@ -51,7 +51,33 @@ func (z *zedrouter) handleGlobalConfigImpl(ctxArg interface{}, key string,
 			}
 			z.metricInterval = metricInterval
 		}
-		z.enableArpSnooping = gcp.GlobalValueBool(types.EnableARPSnoop)
+		enableArpSnooping := gcp.GlobalValueBool(types.EnableARPSnoop)
+		if z.enableArpSnooping != enableArpSnooping {
+			z.enableArpSnooping = enableArpSnooping
+			// Start/Stop ARP snooping in every activated Switch NI.
+			for _, item := range z.pubNetworkInstanceStatus.GetAll() {
+				niStatus := item.(types.NetworkInstanceStatus)
+				if !niStatus.Activated {
+					continue
+				}
+				if niStatus.Type != types.NetworkInstanceTypeSwitch {
+					// ARP snooping is only used in Switch NIs.
+					continue
+				}
+				niConfig := z.lookupNetworkInstanceConfig(niStatus.Key())
+				if niConfig == nil {
+					continue
+				}
+				_, vifs, err := z.getArgsForNIStateCollecting(niConfig.UUID)
+				if err == nil {
+					err = z.niStateCollector.UpdateCollectingForNI(
+						*niConfig, vifs, z.enableArpSnooping)
+				}
+				if err != nil {
+					z.log.Error(err)
+				}
+			}
+		}
 		z.localLegacyMACAddr = gcp.GlobalValueBool(types.NetworkLocalLegacyMACAddress)
 		z.niReconciler.ApplyUpdatedGCP(z.runCtx, *gcp)
 	}
@@ -223,7 +249,13 @@ func (z *zedrouter) handleNetworkInstanceCreate(ctxArg interface{}, key string,
 
 	// Set bridge IP address.
 	if status.Gateway != nil {
-		addrs := types.AssignedAddrs{IPv4Addr: status.Gateway}
+		addrs := types.AssignedAddrs{
+			IPv4Addrs: []types.AssignedAddr{
+				{
+					Address:    status.Gateway,
+					AssignedBy: types.AddressSourceEVEInternal,
+				}},
+		}
 		status.IPAssignments[status.BridgeMac.String()] = addrs
 		status.BridgeIPAddr = status.Gateway
 	}
@@ -359,8 +391,14 @@ func (z *zedrouter) handleNetworkInstanceModify(ctxArg interface{}, key string,
 	if status.BridgeMac != nil {
 		delete(status.IPAssignments, status.BridgeMac.String())
 	}
-	if status.Gateway != nil {
-		addrs := types.AssignedAddrs{IPv4Addr: status.Gateway}
+	if status.Gateway != nil && status.BridgeMac != nil {
+		addrs := types.AssignedAddrs{
+			IPv4Addrs: []types.AssignedAddr{
+				{
+					Address:    status.Gateway,
+					AssignedBy: types.AddressSourceEVEInternal,
+				}},
+		}
 		status.IPAssignments[status.BridgeMac.String()] = addrs
 		status.BridgeIPAddr = status.Gateway
 	}