Add support for NI with multiple ports

This commit implements support for Local NI with multiple ports attached.
API changes with detailed description can be found here:
lf-edge/eve-api#53

In summary, network instance can be now configured with "shared" port
label, potentially matching multiple device ports. The NI routing table
will contain routes from all the selected ports.

Shared labels can be also used to restrict port-forwarding to a subset
of NI ports and to create multipath static routes (routes with multiple
possible next-hops). For every multipath route, zedrouter will use
recently added portprober to select the best port at a given time (based
on the connectivity status, cost, etc.) and also to failover to another
port when the currently used port looses connectivity.

Signed-off-by: Milan Lenco <milan@zededa.com>

pkg/pillar/cmd/diag/diag.go

@@ -880,7 +880,7 @@ func printOutput(ctx *diagContext, caller string) {
 		// fields are set and Dhcp type; proxy info order
 		ifname := port.IfName
 		isMgmt := types.IsMgmtPort(*ctx.DeviceNetworkStatus, ifname)
-		priority := types.GetPortCost(*ctx.DeviceNetworkStatus,
+		cost := types.GetPortCost(*ctx.DeviceNetworkStatus,
 			ifname)
 		if isMgmt {
 			mgmtPorts++
@@ -892,12 +892,10 @@ func printOutput(ctx *diagContext, caller string) {
 		}
 		typeStr := "use: app-shared "
 		if isMgmt {
-			if priority == types.PortCostMin {
-				typeStr = "use: mgmt "
-			} else {
-				typeStr = fmt.Sprintf("use: mgmt (cost %d) ",
-					priority)
-			}
+			typeStr = "use: mgmt "
+		}
+		if cost > types.PortCostMin {
+			typeStr += fmt.Sprintf("(cost %d) ", cost)
 		}
 		macStr := ""
 		if len(port.MacAddr) != 0 {

pkg/pillar/cmd/msrv/handlers.go

@@ -19,10 +19,10 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/lf-edge/eve/pkg/pillar/types"
-	"github.com/lf-edge/eve/pkg/pillar/zedcloud"
-
 	"github.com/lf-edge/eve/pkg/pillar/utils"
 	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
+	"github.com/lf-edge/eve/pkg/pillar/utils/netutils"
+	"github.com/lf-edge/eve/pkg/pillar/zedcloud"
 )
 
 type middlewareKeys int
@@ -32,20 +32,19 @@ const (
 	appUUIDContextKey
 )
 
-func isEmptyIP(ip net.IP) bool {
-	return ip == nil || ip.Equal(net.IP{})
-}
-
 func (msrv *Msrv) handleNetwork() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		msrv.Log.Tracef("networkHandler.ServeHTTP")
 		remoteIP := net.ParseIP(strings.Split(r.RemoteAddr, ":")[0])
-		externalIP, code := msrv.getExternalIPForApp(remoteIP)
-		var ipStr string
+		externalIPs, code := msrv.getExternalIPsForApp(remoteIP)
+		var ipsStr []string
 		var hostname string
-		// Avoid returning the string <nil>
-		if !isEmptyIP(externalIP) {
-			ipStr = externalIP.String()
+		for _, ip := range externalIPs {
+			// Avoid returning the string <nil>
+			if netutils.IsEmptyIP(ip) {
+				continue
+			}
+			ipsStr = append(ipsStr, ip.String())
 		}
 		anStatus := msrv.lookupAppNetworkStatusByAppIP(remoteIP)
 		if anStatus != nil {
@@ -64,7 +63,7 @@ func (msrv *Msrv) handleNetwork() func(http.ResponseWriter, *http.Request) {
 		w.WriteHeader(code)
 		resp, _ := json.Marshal(map[string]interface{}{
 			"caller-ip":         r.RemoteAddr,
-			"external-ipv4":     ipStr,
+			"external-ipv4":     strings.Join(ipsStr, ","),
 			"hostname":          hostname, // Do not delete this line for backward compatibility
 			"app-instance-uuid": hostname,
 			"device-uuid":       enInfo.DeviceID,
@@ -83,13 +82,15 @@ func (msrv *Msrv) handleExternalIP() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		msrv.Log.Tracef("externalIPHandler.ServeHTTP")
 		remoteIP := net.ParseIP(strings.Split(r.RemoteAddr, ":")[0])
-		externalIP, code := msrv.getExternalIPForApp(remoteIP)
+		externalIPs, code := msrv.getExternalIPsForApp(remoteIP)
 		w.WriteHeader(code)
 		w.Header().Add("Content-Type", "text/plain")
-		// Avoid returning the string <nil>
-		if !isEmptyIP(externalIP) {
-			resp := []byte(externalIP.String() + "\n")
-			w.Write(resp)
+		for _, externalIP := range externalIPs {
+			// Avoid returning the string <nil>
+			if !netutils.IsEmptyIP(externalIP) {
+				resp := []byte(externalIP.String() + "\n")
+				w.Write(resp)
+			}
 		}
 	}
 }

pkg/pillar/cmd/msrv/msrv_test.go

@@ -86,7 +86,6 @@ func TestPostKubeconfig(t *testing.T) {
 				IPv4Addr: net.ParseIP("192.168.1.1"),
 			}},
 		},
-		SelectedUplinkIntfName: "eth0",
 	}
 	err = netInstance.Publish("6ba7b810-9dad-11d1-80b4-000000000003", niStatus)
 	g.Expect(err).ToNot(gomega.HaveOccurred())

pkg/pillar/cmd/msrv/pubsub.go

@@ -32,33 +32,45 @@ func (srv *Msrv) lookupAppNetworkStatusByAppIP(ip net.IP) *types.AppNetworkStatu
 	return nil
 }
 
-func (srv *Msrv) getExternalIPForApp(remoteIP net.IP) (net.IP, int) {
+func (srv *Msrv) getExternalIPsForApp(remoteIP net.IP) ([]net.IP, int) {
 	netstatus := srv.lookupNetworkInstanceStatusByAppIP(remoteIP)
 	if netstatus == nil {
-		srv.Log.Errorf("getExternalIPForApp: No NetworkInstanceStatus for %v", remoteIP)
+		srv.Log.Errorf("getExternalIPsForApp: No NetworkInstanceStatus for %v", remoteIP)
 		return nil, http.StatusNotFound
 	}
-	if netstatus.SelectedUplinkIntfName == "" {
-		srv.Log.Warnf("getExternalIPForApp: No SelectedUplinkIntfName for %v", remoteIP)
+	if len(netstatus.Ports) == 0 {
+		srv.Log.Warnf("getExternalIPsForApp: No Port for %v", remoteIP)
 		// Nothing to report */
 		return nil, http.StatusNoContent
 	}
-
 	dnStatus, err := srv.subDeviceNetworkStatus.Get("global")
 	if err != nil {
 		srv.Log.Error(fmt.Sprintf("cannot fetch device network status: %s", err))
 		return nil, http.StatusInternalServerError
 	}
 
+	var ips []net.IP
 	dns := dnStatus.(types.DeviceNetworkStatus)
-	ip, err := types.GetLocalAddrAnyNoLinkLocal(dns,
-		0, netstatus.SelectedUplinkIntfName)
-	if err != nil {
-		srv.Log.Errorf("getExternalIPForApp: No externalIP for %s: %s",
+	for _, port := range dns.Ports {
+		if generics.ContainsItem(netstatus.Ports, port.Logicallabel) {
+			for _, addr := range port.AddrInfoList {
+				ip := addr.Addr.To4()
+				if ip == nil {
+					continue
+				}
+				if ip.IsLinkLocalUnicast() {
+					continue
+				}
+				ips = append(ips, ip)
+			}
+		}
+	}
+	if len(ips) == 0 {
+		srv.Log.Errorf("getExternalIPsForApp: No externalIP for %s: %s",
 			remoteIP.String(), err)
 		return nil, http.StatusNoContent
 	}
-	return ip, http.StatusOK
+	return ips, http.StatusOK
 }
 
 func (srv *Msrv) lookupNetworkInstanceStatusByAppIP(

pkg/pillar/cmd/zedagent/handlemetrics.go

@@ -1130,25 +1130,18 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string,
 			networkInfo.DevName = *proto.String(name)
 			niStatus := appIfnameToNetworkInstance(ctx, aiStatus, ifname)
 			if niStatus != nil {
-				networkInfo.NtpServers = []string{}
-				if niStatus.NtpServer != nil {
+				for _, ntpServer := range niStatus.NTPServers {
 					networkInfo.NtpServers = append(networkInfo.NtpServers,
-						niStatus.NtpServer.String())
-				} else if niStatus.SelectedUplinkIntfName != "" {
-					ntpServers := types.GetNTPServers(*deviceNetworkStatus,
-						niStatus.SelectedUplinkIntfName)
-					for _, server := range ntpServers {
-						networkInfo.NtpServers = append(networkInfo.NtpServers, server.String())
-					}
+						ntpServer.String())
 				}
-
 				networkInfo.DefaultRouters = []string{niStatus.Gateway.String()}
 				networkInfo.Dns = &info.ZInfoDNS{
 					DNSservers: []string{},
 				}
 				networkInfo.Dns.DNSservers = []string{}
 				for _, dnsServer := range niStatus.DnsServers {
-					networkInfo.Dns.DNSservers = append(networkInfo.Dns.DNSservers, dnsServer.String())
+					networkInfo.Dns.DNSservers = append(networkInfo.Dns.DNSservers,
+						dnsServer.String())
 				}
 			}
 			ReportAppInfo.Network = append(ReportAppInfo.Network,
@@ -1675,12 +1668,16 @@ func protoEncodeNetworkInstanceMetricProto(status types.NetworkInstanceMetrics)
 	vlanInfo.NumTrunkPorts = status.VlanMetrics.NumTrunkPorts
 	vlanInfo.VlanCounts = status.VlanMetrics.VlanCounts
 	protoEncodeGenericInstanceMetric(status, metric)
-	metric.ProbeMetric = protoEncodeProbeMetrics(status.ProbeMetrics)
+	for _, pm := range status.ProbeMetrics {
+		metric.ProbeMetrics = append(metric.ProbeMetrics, protoEncodeProbeMetrics(pm))
+	}
 	return metric
 }
 
 func protoEncodeProbeMetrics(probeMetrics types.ProbeMetrics) *metrics.ZProbeNIMetrics {
 	protoMetrics := &metrics.ZProbeNIMetrics{
+		DstNetwork:     probeMetrics.DstNetwork,
+		CurrentPort:    probeMetrics.SelectedPort,
 		CurrentIntf:    probeMetrics.SelectedPortIfName,
 		RemoteEndpoint: strings.Join(probeMetrics.RemoteEndpoints, ", "),
 		PingIntv:       probeMetrics.LocalPingIntvl,

pkg/pillar/cmd/zedagent/handlenetworkinstance.go

@@ -81,9 +81,24 @@ func prepareAndPublishNetworkInstanceInfoMsg(ctx *zedagentContext,
 	if !deleted {
 		info.Displayname = status.DisplayName
 		info.InstType = uint32(status.Type)
-		info.CurrentUplinkIntf = status.SelectedUplinkIntfName
+		// info.Ports is set for new controllers (that support NI with multiple ports),
+		// while we also continue to set info.CurrentUplinkIntf for backward-compatibility
+		// with older controllers.
+		info.Ports = status.Ports
+		if len(status.Ports) > 0 {
+			// Just report the first port from the list.
+			// Typically, there will be at most one port anyway.
+			info.CurrentUplinkIntf = status.Ports[0]
+		}
 		info.Mtu = uint32(status.MTU)
-
+		for _, route := range status.CurrentRoutes {
+			info.IpRoutes = append(info.IpRoutes, &zinfo.IPRoute{
+				DestinationNetwork: route.DstNetwork.String(),
+				Gateway:            route.Gateway.String(),
+				Port:               route.OutputPort,
+				GatewayApp:         route.GatewayApp.String(),
+			})
+		}
 		if !status.ErrorTime.IsZero() {
 			errInfo := new(zinfo.ErrorInfo)
 			errInfo.Description = status.Error
@@ -125,11 +140,10 @@ func prepareAndPublishNetworkInstanceInfoMsg(ctx *zedagentContext,
 			vi.AppID = v.AppID.String()
 			info.Vifs = append(info.Vifs, vi)
 		}
-		if status.SelectedUplinkIntfName != "" {
-			ifname := status.SelectedUplinkIntfName
-			ia := ctx.assignableAdapters.LookupIoBundleIfName(ifname)
+		for _, port := range info.Ports {
+			ia := ctx.assignableAdapters.LookupIoBundleLogicallabel(port)
 			if ia == nil {
-				log.Warnf("Missing adapter for ifname %s", ifname)
+				log.Warnf("Missing IoBundle for port %s", port)
 			} else {
 				reportAA := new(zinfo.ZioBundle)
 				reportAA.Type = zcommon.PhyIoType(ia.Type)