Persist MAC generator ID used by a given app

Persisting MAC address generator ID avoids changing MAC addresses for
already deployed apps, even when the option network.local.legacy.mac.address
is changed by the user. Only newly deployed apps will apply the latest config
change and use the most recently selected MAC generator.

Signed-off-by: Milan Lenco <milan@zededa.com>
(cherry picked from commit 7c742f2)

pkg/pillar/base/logobjecttypes.go

@@ -166,6 +166,8 @@ const (
 	EncryptedVaultKeyFromControllerLogType LogObjectType = "encrypted_vault_key_from_controller"
 	// CachedResolvedIPsLogType:
 	CachedResolvedIPsLogType LogObjectType = "cached_resolved_ips"
+	// AppMACGeneratorLogType : type for AppMACGenerator log entries
+	AppMACGeneratorLogType LogObjectType = "app_mac_generator"
 )
 
 // RelationObjectType :

pkg/pillar/cmd/zedrouter/appnetwork.go

@@ -91,8 +91,7 @@ func (z *zedrouter) prepareConfigForVIFs(config types.AppNetworkConfig,
 			// User-configured static MAC address.
 			ulStatus.Mac = ulStatus.AppMacAddr
 		} else {
-			ulStatus.Mac = z.generateAppMac(config.UUIDandVersion.UUID, ulNum,
-				status.AppNum, netInstStatus)
+			adapterStatus.Mac = z.generateAppMac(ulNum, status, netInstStatus)
 		}
 		ulStatus.HostName = config.Key()
 		guestIP, err := z.lookupOrAllocateIPv4ForVIF(

pkg/pillar/cmd/zedrouter/ipam.go

@@ -41,32 +41,37 @@ func (z *zedrouter) generateBridgeMAC(brNum int) net.HardwareAddr {
 // Since these MAC addresses will not appear on external Ethernet networks, we can also
 // use OUI octets for randomness. Only I/G and U/L bits need to stay constant and set
 // appropriately.
-func (z *zedrouter) generateAppMac(appUUID uuid.UUID, ulNum int, appNum int,
+func (z *zedrouter) generateAppMac(ulNum int, appStatus *types.AppNetworkStatus,
 	netInstStatus *types.NetworkInstanceStatus) net.HardwareAddr {
 	h := sha256.New()
-	h.Write(appUUID[:])
+	h.Write(appStatus.UUIDandVersion.UUID[:])
 	h.Write(netInstStatus.UUIDandVersion.UUID[:])
 	nums := make([]byte, 2)
 	nums[0] = byte(ulNum)
-	nums[1] = byte(appNum)
+	nums[1] = byte(appStatus.AppNum)
 	h.Write(nums)
 	hash := h.Sum(nil)
 	switch netInstStatus.Type {
 	case types.NetworkInstanceTypeSwitch:
+		// For switch network instances, we always generate globally-scoped
+		// MAC addresses. There is no difference in behaviour between MAC address
+		// generators in this case.
 		return net.HardwareAddr{0x02, 0x16, 0x3e, hash[0], hash[1], hash[2]}
 	case types.NetworkInstanceTypeLocal:
-		if z.localLegacyMACAddr {
-			z.log.Noticef("generateAppMac: legacy MAC address for app %v", appUUID)
-			// Room to handle multiple underlays in 5th byte
-			return net.HardwareAddr{0x00, 0x16, 0x3e, 0x00, byte(adapterNum), byte(appNum)}
+		switch appStatus.MACGenerator {
+		case types.MACGeneratorNodeScoped:
+			return net.HardwareAddr{0x00, 0x16, 0x3e, 0x00,
+				byte(adapterNum), byte(appStatus.AppNum)}
+		case types.MACGeneratorGloballyScoped:
+			mac := net.HardwareAddr{hash[0], hash[1], hash[2], hash[3], hash[4], hash[5]}
+			// Mark this MAC address as unicast by setting the I/G bit to zero.
+			mac[0] &= ^byte(1)
+			// Mark this MAC address as locally administered by setting the U/L bit to 1.
+			mac[0] |= byte(1 << 1)
+			return mac
+		default:
+			z.log.Fatalf("undefined MAC generator")
 		}
-		z.log.Noticef("generateAppMac: random MAC address for app %v", appUUID)
-		mac := net.HardwareAddr{hash[0], hash[1], hash[2], hash[3], hash[4], hash[5]}
-		// Mark this MAC address as unicast by setting the I/G bit to zero.
-		mac[0] &= ^byte(1)
-		// Mark this MAC address as locally administered by setting the U/L bit to 1.
-		mac[0] |= byte(1 << 1)
-		return mac
 	default:
 		z.log.Fatalf("unsupported network instance type")
 	}

pkg/pillar/cmd/zedrouter/numallocators.go

@@ -110,6 +110,15 @@ func (z *zedrouter) initNumberAllocators() {
 			// Continue despite the error, this is best-effort.
 		}
 	}
+
+	// Persist ID of MAC generator used for each application.
+	macGeneratorPublisher, err := objtonum.NewObjNumPublisher(
+		z.log, z.pubSub, agentName, true, &types.AppMACGenerator{})
+	if err != nil {
+		z.log.Fatal(err)
+	}
+	z.appMACGeneratorMap = objtonum.NewPublishedMap(
+		z.log, macGeneratorPublisher, "appMACGenerator", objtonum.AllKeys)
 }
 
 // Either get existing or create a new allocator for app-interfaces connected

pkg/pillar/cmd/zedrouter/pubsubhandlers.go

@@ -469,6 +469,30 @@ func (z *zedrouter) handleAppNetworkCreate(ctxArg interface{}, key string,
 		return
 	}
 	status.AppNum = appNum
+
+	// For app already deployed (before node reboot), keep using the same MAC address
+	// generator. Changing MAC addresses could break network config inside the app.
+	macGenerator, _, err := z.appMACGeneratorMap.Get(appNumKey)
+	if err != nil || macGenerator == types.MACGeneratorUnspecified {
+		// New app or an existing app but without MAC generator ID persisted.
+		if z.localLegacyMACAddr {
+			// Use older node-scoped MAC address generator.
+			macGenerator = types.MACGeneratorNodeScoped
+		} else {
+			// Use newer (and preferred) globally-scoped MAC address generator.
+			macGenerator = types.MACGeneratorGloballyScoped
+		}
+		// Remember which MAC generator is being used for this app.
+		err = z.appMACGeneratorMap.Assign(appNumKey, macGenerator, false)
+		if err != nil {
+			err = fmt.Errorf("failed to persist MAC generator ID for app %s/%s: %v",
+				config.UUIDandVersion.UUID, config.DisplayName, err)
+			z.log.Errorf("handleAppNetworkCreate(%v): %v", config.UUIDandVersion.UUID, err)
+			z.addAppNetworkError(&status, "handleAppNetworkCreate", err)
+			return
+		}
+	}
+	status.MACGenerator = macGenerator
 	z.publishAppNetworkStatus(&status)
 
 	// Allocate application numbers on underlay network.
@@ -596,6 +620,12 @@ func (z *zedrouter) handleAppNetworkDelete(ctxArg interface{}, key string,
 			status.UUIDandVersion.UUID, status.DisplayName, err)
 		// Continue anyway...
 	}
+	err = z.appMACGeneratorMap.Delete(appNumKey, false)
+	if err != nil {
+		z.log.Errorf("failed to delete persisted MAC generator ID for app %s/%s: %v",
+			status.UUIDandVersion.UUID, status.DisplayName, err)
+		// Continue anyway...
+	}
 	z.freeAppIntfNums(status)
 
 	// Did this free up any last references against any deleted Network Instance?

pkg/pillar/cmd/zedrouter/zedrouter.go

@@ -105,6 +105,7 @@ type zedrouter struct {
 	bridgeNumAllocator  *objtonum.Allocator
 	appIntfNumPublisher *objtonum.ObjNumPublisher
 	appIntfNumAllocator map[string]*objtonum.Allocator // key: network instance UUID as string
+	appMACGeneratorMap  objtonum.Map
 
 	// Info published to application via metadata server
 	subLocationInfo pubsub.Subscription

pkg/pillar/types/zedroutertypes.go

@@ -17,6 +17,7 @@ import (
 	"github.com/eriknordmark/ipinfo"
 	"github.com/google/go-cmp/cmp"
 	"github.com/lf-edge/eve/pkg/pillar/base"
+	"github.com/lf-edge/eve/pkg/pillar/objtonum"
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -164,6 +165,8 @@ type AppNetworkStatus struct {
 	GetStatsIPAddr       net.IP
 	UnderlayNetworkList  []UnderlayNetworkStatus
 	AwaitNetworkInstance bool // If any Missing flag is set in the networks
+	// ID of the MAC generator variant that was used to generate MAC addresses for this app.
+	MACGenerator int
 	// Any errors from provisioning the network
 	// ErrorAndTime provides SetErrorNow() and ClearError()
 	ErrorAndTime
@@ -3543,3 +3546,66 @@ func (c CachedResolvedIPs) LogDelete(logBase *base.LogObject) {
 func (c CachedResolvedIPs) LogKey() string {
 	return string(base.CachedResolvedIPsLogType) + "-" + c.Key()
 }
+
+// AppMACGenerator persistently stores ID of the MAC generator that was used to generate
+// MAC addresses for interfaces of a given app.
+type AppMACGenerator struct {
+	*UuidToNum
+}
+
+// New is used by objtonum.ObjNumPublisher.
+func (g *AppMACGenerator) New(objKey objtonum.ObjKey) objtonum.ObjNumContainer {
+	uuidToNum, ok := g.UuidToNum.New(objKey).(*UuidToNum)
+	if !ok {
+		logrus.Fatalf("Wrong type returned by UuidToNum.New()")
+	}
+	return &AppMACGenerator{
+		UuidToNum: uuidToNum,
+	}
+}
+
+// LogCreate logs newly added AppMACGenerator entry.
+func (g AppMACGenerator) LogCreate(logBase *base.LogObject) {
+	logObject := base.NewLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	logObject.Noticef("AppMACGenerator item create")
+}
+
+// LogModify logs modified AppMACGenerator entry.
+func (g AppMACGenerator) LogModify(logBase *base.LogObject, old interface{}) {
+	logObject := base.EnsureLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	oldEntry, ok := old.(AppMACGenerator)
+	if !ok {
+		logObject.Clone().Fatalf("LogModify: old object is not of AppMACGenerator type")
+	}
+	logObject.CloneAndAddField("diff", cmp.Diff(oldEntry, g)).
+		Noticef("AppMACGenerator item modify")
+}
+
+// LogDelete logs deleted AppMACGenerator entry.
+func (g AppMACGenerator) LogDelete(logBase *base.LogObject) {
+	logObject := base.EnsureLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	logObject.Noticef("AppMACGenerator item delete")
+	base.DeleteLogObject(logBase, g.LogKey())
+}
+
+// LogKey identifies AppMACGenerator entry for logging purposes.
+func (g AppMACGenerator) LogKey() string {
+	return string(base.AppMACGeneratorLogType) + "-" + g.Key()
+}
+
+// IDs assigned to different variants of MAC generators.
+const (
+	// MACGeneratorUnspecified : MAC generator is not specified.
+	MACGeneratorUnspecified = 0
+	// MACGeneratorNodeScoped generates MAC addresses which are guaranteed to be unique
+	// only within the scope of the given single device.
+	// The exception are MAC addresses generated for switch network instances,
+	// which are always generated with global scope.
+	MACGeneratorNodeScoped = 1
+	// MACGeneratorGloballyScoped generates MAC addresses which are with high probability
+	// unique globally, i.e. across entire fleet of devices.
+	MACGeneratorGloballyScoped = 2
+)