Enhance LinuxCollector to support detecting multiple app VIF IPs

Application VIF may have multiple IP address assigned. They can be
either assigned directly on the same interface, or used on separate
VLAN sub-interfaces which share the parent interface MAC address.

LinuxCollector should detect and publish all of them, instead of
flapping between them and generating many IP address change
notifications, which trigger a flood of NI and App info messages
published to the controller.

For DHCP assigned IP addresses, we use lease time to determine if
a previously detected IP address is still valid. For statically
assigned IP, we expect to see at least one associated ARP packet
every 10 minutes. Otherwise, we consider the IP address to be removed
or simply not used anymore.

This commit also enhances LinuxCollector to support enabling or
disabling ARP snooping in runtime, without requiring to recreate all
switch network instances or rebooting device.

Signed-off-by: Milan Lenco <[email protected]>
(cherry picked from commit 5b2acf4)

docs/APP-CONNECTIVITY.md

@@ -207,6 +207,25 @@ to retrieve cloud-init configuration, obtain information from EVE (e.g. device U
 hostname, uplink IP address) or to download [patch envelopes](PATCH-ENVELOPES.md).
 More information about metadata server can be found in [ECO-METADATA.md](ECO-METADATA.md).
 
+#### IPAM
+
+Every Local Network Instance must be configured with an IPv4 network subnet and an IP
+range within this subnet for automatic IP allocations. Host IP addresses from this subnet
+that do not fall within the IP range are available for manual assignment.
+
+Whether an IP address is selected manually or dynamically assigned by EVE from the configured
+IP range, an internal DHCP server is used to distribute these IP addresses to applications.
+Container applications are deployed inside a "shim VM", which EVE prepares, ensuring that
+a DHCP client is running for every virtual interface connected to a network instance
+This guarantees that the IP address is received and applied before the application starts.
+In contrast, VM applications are responsible for starting their own DHCP client and applying
+the received IP addresses.
+
+Regardless of the application type, EVE does not automatically assume that the allocated
+IP address is actually in use. Instead, it monitors the set of IP leases granted by the internal
+DHCP server and updates the set of application IP addresses in the published info messages
+accordingly.
+
 ### Switch Network Instance
 
 Switch Network Instance is a simple L2-only bridge between connected applications and
@@ -232,6 +251,52 @@ inbound ACL rules are that much more important.
 Metadata HTTP server is run for switch network instance only if it has uplink port with
 an IP address.
 
+#### IP address detection
+
+Unlike a Local Network Instance, a switch network instance is configured without any IP
+configuration, and EVE does not run an internal DHCP server. Instead, if IP connectivity
+is required, IP addresses must be assigned statically within the connected applications
+or provided by an external DHCP server or another application offering DHCP services.
+
+Since EVE is not in control of IP address allocations and leases, it must monitor application
+traffic to learn which IP addresses are being used and report this information to the controller.
+
+In the case of an external DHCP server (IPv4), EVE captures the DHCPACK packet from the server,
+which confirms the leased IP address. Because EVE manages MAC address allocations, it knows
+the MAC address of every application's virtual interface (VIF). It can then map the CHADDR
+(Client Hardware Address) attribute to the corresponding application VIF and learn the assigned
+IP address from the YIADDR (your, i.e. client, IP Address) attribute. Additionally, EVE reads
+the DHCP option 51 (Lease Time), if available, to determine how long the leased IP address
+is valid. If EVE does not observe an IP renewal within this period, it assumes that the IP address
+is no longer in use and reports this change to the controller.
+
+For statically assigned IPv4 addresses, EVE captures both ARP reply and request packets to learn
+the application VIF IP assignment from either Sender IP + MAC or Target IP + MAC attribute
+pairs. Since ARP cache entries have a limited lifetime — typically around 2 minutes — EVE expects
+to see at least one ARP packet for every assigned IP within a 10-minute window (this is not
+configurable). If no ARP packet is observed within this period for a previously detected IP
+assignment, EVE assumes that the IP address has been removed and reports this change to
+the controller. EVE also captures ARP packets for IP addresses configured via DHCP, but these
+are ignored as the information from the previously captured DHCPACK takes precedence.
+Note that ARP-based IP detection is enabled by default but can be disabled by setting
+the configuration item `network.switch.enable.arpsnoop` to `false`. Change in this config
+options will apply to already deployed switch network instances.
+
+For an external DHCPv6 server, EVE captures DHCPv6 REPLY messages. It learns the target MAC
+address from the DUID option (Client Identifier, option code 1), while the IPv6 address
+and its valid lifetime are provided by the IA Address (option code 5).
+
+To learn IPv6 addresses assigned using SLAAC (Stateless Address Auto Configuration),
+EVE captures unicast ICMPv6 Neighbor Solicitation messages. These are sent from the interface
+with the assigned IPv6 address to check if the address is free or already in use by another
+host — a process known as Duplicate Address Detection (DAD). The ICMPv6 packet sent to detect
+IP duplicates for a particular VIF IP will have the VIF MAC address as the source address
+in the Ethernet header. EVE uses this, along with the "Target Address" field from the ICMPv6
+header, to identify the assigned IPv6 address.
+
+EVE is capable of detecting multiple IPs assigned to the same VIF MAC address. This is commonly
+seen when applications use VLAN sub-interfaces, which share the parent interface's MAC address.
+
 ### Uplink Port
 
 Network instances can be configured with an "uplink" network adapter, which will be used to provide

docs/CONFIG-PROPERTIES.md

@@ -53,7 +53,7 @@
 | netdump.topic.maxcount | integer | 10 | maximum number of netdumps that can be published for each topic. The oldest netdump is unpublished should a new netdump exceed the limit.
 | netdump.downloader.with.pcap | boolean | false | include packet captures inside netdumps for download requests. However, even if enabled, TCP segments carrying non-empty payload (i.e. content which is being downloaded) are excluded and the overall PCAP size is limited to 64MB. |
 | netdump.downloader.http.with.fieldvalue | boolean | false | include HTTP header field values in captured network traces for download requests (beware: may contain secrets, such as datastore credentials). |
-| network.switch.enable.arpsnoop | boolean | true | enable ARP Snooping on switch Network Instance, may need a device reboot to take effect |
+| network.switch.enable.arpsnoop | boolean | true | enable ARP Snooping on switch Network Instances |
 | wwan.query.visible.providers | bool | false | enable to periodically (once per hour) query the set of visible cellular service providers and publish them under WirelessStatus (for every modem) |
 | network.local.legacy.mac.address | bool | false | enables legacy MAC address generation for local network instances for those EVE nodes where changing MAC addresses in applications will lead to incorrect network configuration |
 

pkg/pillar/cmd/zedagent/handleconfig.go

@@ -1200,8 +1200,22 @@ func updateLocalServerMap(getconfigCtx *getconfigContext, localServerURL string)
 				continue
 			}
 			if localServerIP != nil {
-				// check if the defined IP of localServer equals the allocated IP of the app
-				if adapterStatus.AllocatedIPv4Addr.Equal(localServerIP) {
+				// Check if the defined IP of localServer equals one of the IPs
+				// allocated to the app.
+				var matchesApp bool
+				for _, ip := range adapterStatus.AssignedAddresses.IPv4Addrs {
+					if ip.Address.Equal(localServerIP) {
+						matchesApp = true
+						break
+					}
+				}
+				for _, ip := range adapterStatus.AssignedAddresses.IPv6Addrs {
+					if ip.Address.Equal(localServerIP) {
+						matchesApp = true
+						break
+					}
+				}
+				if matchesApp {
 					srvAddr := localServerAddr{
 						localServerAddr: localServerURL,
 						bridgeIP:        adapterStatus.BridgeIPAddr,

pkg/pillar/cmd/zedagent/handlemetrics.go

@@ -1108,16 +1108,18 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string,
 		for _, ifname := range ifNames {
 			networkInfo := new(info.ZInfoNetwork)
 			networkInfo.LocalName = *proto.String(ifname)
-			ipv4Addr, ipv6Addrs, allocated, macAddr, ipAddrMismatch :=
-				getAppIP(ctx, aiStatus, ifname)
-			if ipv4Addr != nil {
-				networkInfo.IPAddrs = append(networkInfo.IPAddrs, ipv4Addr.String())
+			addrs, hasIPv4Addr, macAddr, ipAddrMismatch :=
+				getAppIPs(ctx, aiStatus, ifname)
+			for _, ipv4Addr := range addrs.IPv4Addrs {
+				networkInfo.IPAddrs = append(networkInfo.IPAddrs,
+					ipv4Addr.Address.String())
 			}
-			for _, ipv6Addr := range ipv6Addrs {
-				networkInfo.IPAddrs = append(networkInfo.IPAddrs, ipv6Addr.String())
+			for _, ipv6Addr := range addrs.IPv6Addrs {
+				networkInfo.IPAddrs = append(networkInfo.IPAddrs,
+					ipv6Addr.Address.String())
 			}
 			networkInfo.MacAddr = *proto.String(macAddr.String())
-			networkInfo.Ipv4Up = allocated
+			networkInfo.Ipv4Up = hasIPv4Addr
 			networkInfo.IpAddrMisMatch = ipAddrMismatch
 			name := appIfnameToName(aiStatus, ifname)
 			log.Tracef("app %s/%s localName %s devName %s",
@@ -1563,21 +1565,22 @@ func sendMetricsProtobuf(ctx *getconfigContext,
 
 // Use the ifname/vifname to find the AppNetAdapter status
 // and from there the (ip, allocated, mac) addresses for the app
-func getAppIP(ctx *zedagentContext, aiStatus *types.AppInstanceStatus,
-	vifname string) (net.IP, []net.IP, bool, net.HardwareAddr, bool) {
+func getAppIPs(ctx *zedagentContext, aiStatus *types.AppInstanceStatus,
+	vifname string) (types.AssignedAddrs, bool, net.HardwareAddr, bool) {
 
 	log.Tracef("getAppIP(%s, %s)", aiStatus.Key(), vifname)
 	for _, adapterStatus := range aiStatus.AppNetAdapters {
 		if adapterStatus.VifUsed != vifname {
 			continue
 		}
-		log.Tracef("getAppIP(%s, %s) found AppIP v4: %s, v6: %s, ipv4 assigned %v mac %s",
-			aiStatus.Key(), vifname, adapterStatus.AllocatedIPv4Addr,
-			adapterStatus.AllocatedIPv6List, adapterStatus.IPv4Assigned, adapterStatus.Mac)
-		return adapterStatus.AllocatedIPv4Addr, adapterStatus.AllocatedIPv6List, adapterStatus.IPv4Assigned,
+		log.Tracef("getAppIP(%s, %s) found AppIPs v4: %v, v6: %v, ipv4 assigned %v mac %s",
+			aiStatus.Key(), vifname, adapterStatus.AssignedAddresses.IPv4Addrs,
+			adapterStatus.AssignedAddresses.IPv6Addrs, adapterStatus.IPv4Assigned,
+			adapterStatus.Mac)
+		return adapterStatus.AssignedAddresses, adapterStatus.IPv4Assigned,
 			adapterStatus.Mac, adapterStatus.IPAddrMisMatch
 	}
-	return nil, nil, false, nil, false
+	return types.AssignedAddrs{}, false, nil, false
 }
 
 func createVolumeInstanceMetrics(ctx *zedagentContext, reportMetrics *metrics.ZMetricMsg) {

pkg/pillar/cmd/zedagent/handlenetworkinstance.go

@@ -8,7 +8,6 @@ package zedagent
 import (
 	"bytes"
 	"fmt"
-	"net"
 	"time"
 
 	"github.com/golang/protobuf/ptypes/timestamp"
@@ -109,11 +108,13 @@ func prepareAndPublishNetworkInstanceInfoMsg(ctx *zedagentContext,
 		for mac, addrs := range status.IPAssignments {
 			assignment := new(zinfo.ZmetIPAssignmentEntry)
 			assignment.MacAddress = mac
-			if !addrs.IPv4Addr.Equal(net.IP{}) {
-				assignment.IpAddress = append(assignment.IpAddress, addrs.IPv4Addr.String())
+			for _, assignedIP := range addrs.IPv4Addrs {
+				assignment.IpAddress = append(assignment.IpAddress,
+					assignedIP.Address.String())
 			}
-			for _, ip := range addrs.IPv6Addrs {
-				assignment.IpAddress = append(assignment.IpAddress, ip.String())
+			for _, assignedIP := range addrs.IPv6Addrs {
+				assignment.IpAddress = append(assignment.IpAddress,
+					assignedIP.Address.String())
 			}
 			info.IpAssignments = append(info.IpAssignments, assignment)
 		}

pkg/pillar/cmd/zedrouter/appnetwork.go

@@ -62,7 +62,8 @@ func (z *zedrouter) updateVIFsForStateCollecting(
 		}
 		_, vifs, err := z.getArgsForNIStateCollecting(network)
 		if err == nil {
-			err = z.niStateCollector.UpdateCollectingForNI(*netConfig, vifs)
+			err = z.niStateCollector.UpdateCollectingForNI(*netConfig, vifs,
+				z.enableArpSnooping)
 		}
 		if err != nil {
 			z.log.Error(err)
@@ -168,10 +169,7 @@ func (z *zedrouter) updateNIStatusAfterAppNetworkActivate(status *types.AppNetwo
 		netInstStatus.AddVif(z.log, adapterStatus.Vif, adapterStatus.Mac,
 			status.UUIDandVersion.UUID)
 		netInstStatus.IPAssignments[adapterStatus.Mac.String()] =
-			types.AssignedAddrs{
-				IPv4Addr:  adapterStatus.AllocatedIPv4Addr,
-				IPv6Addrs: adapterStatus.AllocatedIPv6List,
-			}
+			adapterStatus.AssignedAddresses
 		z.publishNetworkInstanceStatus(netInstStatus)
 	}
 }

pkg/pillar/cmd/zedrouter/ipam.go

@@ -120,13 +120,15 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 	// Lookup to see if it is already allocated.
 	if ipAddr == nil {
 		addrs := niStatus.IPAssignments[adapterStatus.Mac.String()]
-		if !isEmptyIP(addrs.IPv4Addr) {
-			z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): found IP %v for MAC %v",
-				networkID, appID, addrs.IPv4Addr, adapterStatus.Mac)
-			ipAddr = addrs.IPv4Addr
+		ipAddr = addrs.GetInternallyLeasedIPv4Addr()
+		if ipAddr != nil {
+			z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): "+
+				"found EVE-allocated IP %v for MAC %v", networkID, appID, ipAddr,
+				adapterStatus.Mac)
 		}
 	}
 
+	var newlyAllocated bool
 	if ipAddr == nil {
 		// Allocate IP address dynamically.
 		// Get the app number for the AppNetAdapter entry.
@@ -140,6 +142,7 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 		}
 		// Pick an IP address from the subnet.
 		ipAddr = netutils.AddToIP(niStatus.DhcpRange.Start, appNum)
+		newlyAllocated = true
 		// Check if the address falls into the Dhcp Range.
 		if !niStatus.DhcpRange.Contains(ipAddr) {
 			err := fmt.Errorf("no free IP addresses in DHCP range(%v, %v)",
@@ -162,12 +165,19 @@ func (z *zedrouter) lookupOrAllocateIPv4ForVIF(niStatus *types.NetworkInstanceSt
 		}
 	}
 	// Later will be overwritten with addresses received from nistate.Collector,
-	// which snoops DHCP traffic and watches DNS server leases to learn the *actual*
+	// which snoops DHCP traffic and watches DHCP server leases to learn the *actual*
 	// IP address assignments.
-	addrs := niStatus.IPAssignments[adapterStatus.Mac.String()] // preserve IPv6 addresses
-	addrs.IPv4Addr = ipAddr
-	niStatus.IPAssignments[adapterStatus.Mac.String()] = addrs
-	z.publishNetworkInstanceStatus(niStatus)
+	if newlyAllocated {
+		// Preserve other IPv4 and IPv6 addresses.
+		addrs := niStatus.IPAssignments[adapterStatus.Mac.String()]
+		addrs.IPv4Addrs = append(addrs.IPv4Addrs,
+			types.AssignedAddr{
+				Address:    ipAddr,
+				AssignedBy: types.AddressSourceInternalDHCP,
+			})
+		niStatus.IPAssignments[adapterStatus.Mac.String()] = addrs
+		z.publishNetworkInstanceStatus(niStatus)
+	}
 	z.log.Functionf("lookupOrAllocateIPv4(NI:%v, app:%v): allocated IP %v for MAC %v",
 		networkID, appID, ipAddr, adapterStatus.Mac)
 	return ipAddr, nil
@@ -188,16 +198,16 @@ func (z *zedrouter) recordAssignedIPsToAdapterStatus(adapter *types.AppNetAdapte
 		z.removeAssignedIPsFromAdapterStatus(adapter)
 		return
 	}
-	adapter.AllocatedIPv4Addr = vifAddrs.IPv4Addr
-	if !isEmptyIP(adapter.AppIPAddr) &&
-		!adapter.AppIPAddr.Equal(adapter.AllocatedIPv4Addr) {
-		// Config and status do not match.
-		adapter.IPAddrMisMatch = true
-	} else {
-		adapter.IPAddrMisMatch = false
+	adapter.AssignedAddresses = vifAddrs.AssignedAddrs
+	adapter.IPAddrMisMatch = false
+	if !isEmptyIP(adapter.AppIPAddr) {
+		leasedIP := adapter.AssignedAddresses.GetInternallyLeasedIPv4Addr()
+		if !adapter.AppIPAddr.Equal(leasedIP) {
+			// Config and status do not match.
+			adapter.IPAddrMisMatch = true
+		}
 	}
-	adapter.AllocatedIPv6List = vifAddrs.IPv6Addrs
-	adapter.IPv4Assigned = !isEmptyIP(vifAddrs.IPv4Addr)
+	adapter.IPv4Assigned = len(vifAddrs.IPv4Addrs) > 0
 }
 
 func (z *zedrouter) removeAssignedIPsFromAppNetStatus(status *types.AppNetworkStatus) {
@@ -208,8 +218,8 @@ func (z *zedrouter) removeAssignedIPsFromAppNetStatus(status *types.AppNetworkSt
 }
 
 func (z *zedrouter) removeAssignedIPsFromAdapterStatus(adapterStatus *types.AppNetAdapterStatus) {
-	adapterStatus.AllocatedIPv6List = nil
-	adapterStatus.AllocatedIPv4Addr = nil
+	adapterStatus.AssignedAddresses.IPv4Addrs = nil
+	adapterStatus.AssignedAddresses.IPv6Addrs = nil
 	adapterStatus.IPAddrMisMatch = false
 	adapterStatus.IPv4Assigned = false
 }

pkg/pillar/cmd/zedrouter/networkinstance.go

@@ -230,7 +230,7 @@ func (z *zedrouter) doUpdateActivatedNetworkInstance(config types.NetworkInstanc
 	z.processNIReconcileStatus(niRecStatus, status)
 	_, vifs, err := z.getArgsForNIStateCollecting(config.UUID)
 	if err == nil {
-		err = z.niStateCollector.UpdateCollectingForNI(config, vifs)
+		err = z.niStateCollector.UpdateCollectingForNI(config, vifs, z.enableArpSnooping)
 	}
 	if err != nil {
 		z.log.Error(err)

pkg/pillar/cmd/zedrouter/pubsubhandlers.go

@@ -51,7 +51,33 @@ func (z *zedrouter) handleGlobalConfigImpl(ctxArg interface{}, key string,
 			}
 			z.metricInterval = metricInterval
 		}
-		z.enableArpSnooping = gcp.GlobalValueBool(types.EnableARPSnoop)
+		enableArpSnooping := gcp.GlobalValueBool(types.EnableARPSnoop)
+		if z.enableArpSnooping != enableArpSnooping {
+			z.enableArpSnooping = enableArpSnooping
+			// Start/Stop ARP snooping in every activated Switch NI.
+			for _, item := range z.pubNetworkInstanceStatus.GetAll() {
+				niStatus := item.(types.NetworkInstanceStatus)
+				if !niStatus.Activated {
+					continue
+				}
+				if niStatus.Type != types.NetworkInstanceTypeSwitch {
+					// ARP snooping is only used in Switch NIs.
+					continue
+				}
+				niConfig := z.lookupNetworkInstanceConfig(niStatus.Key())
+				if niConfig == nil {
+					continue
+				}
+				_, vifs, err := z.getArgsForNIStateCollecting(niConfig.UUID)
+				if err == nil {
+					err = z.niStateCollector.UpdateCollectingForNI(
+						*niConfig, vifs, z.enableArpSnooping)
+				}
+				if err != nil {
+					z.log.Error(err)
+				}
+			}
+		}
 		z.localLegacyMACAddr = gcp.GlobalValueBool(types.NetworkLocalLegacyMACAddress)
 		z.niReconciler.ApplyUpdatedGCP(z.runCtx, *gcp)
 	}
@@ -223,7 +249,13 @@ func (z *zedrouter) handleNetworkInstanceCreate(ctxArg interface{}, key string,
 
 	// Set bridge IP address.
 	if status.Gateway != nil {
-		addrs := types.AssignedAddrs{IPv4Addr: status.Gateway}
+		addrs := types.AssignedAddrs{
+			IPv4Addrs: []types.AssignedAddr{
+				{
+					Address:    status.Gateway,
+					AssignedBy: types.AddressSourceEVEInternal,
+				}},
+		}
 		status.IPAssignments[status.BridgeMac.String()] = addrs
 		status.BridgeIPAddr = status.Gateway
 	}
@@ -359,8 +391,14 @@ func (z *zedrouter) handleNetworkInstanceModify(ctxArg interface{}, key string,
 	if status.BridgeMac != nil {
 		delete(status.IPAssignments, status.BridgeMac.String())
 	}
-	if status.Gateway != nil {
-		addrs := types.AssignedAddrs{IPv4Addr: status.Gateway}
+	if status.Gateway != nil && status.BridgeMac != nil {
+		addrs := types.AssignedAddrs{
+			IPv4Addrs: []types.AssignedAddr{
+				{
+					Address:    status.Gateway,
+					AssignedBy: types.AddressSourceEVEInternal,
+				}},
+		}
 		status.IPAssignments[status.BridgeMac.String()] = addrs
 		status.BridgeIPAddr = status.Gateway
 	}