Persist MAC generator ID used by a given app

Persisting MAC address generator ID avoids changing MAC addresses for
already deployed apps, even when the option network.local.legacy.mac.address
is changed by the user. Only newly deployed apps will apply the latest config
change and use the most recently selected MAC generator.

This is backported from master, commit 7c742f2,
but highly adjusted to 9.4, witch pre-refactoring zedrouter code.

Signed-off-by: Milan Lenco <milan@zededa.com>

pkg/pillar/base/logobjecttypes.go

@@ -164,6 +164,8 @@ const (
 	EncryptedVaultKeyFromDeviceLogType LogObjectType = "encrypted_vault_key_from_device"
 	// EncryptedVaultKeyFromControllerLogType:
 	EncryptedVaultKeyFromControllerLogType LogObjectType = "encrypted_vault_key_from_controller"
+	// AppMACGeneratorLogType : type for AppMACGenerator log entries
+	AppMACGeneratorLogType LogObjectType = "app_mac_generator"
 )
 
 // RelationObjectType :

pkg/pillar/cmd/zedrouter/appmacgenerator.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2023 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Back-ported code for persisting of MAC address generator ID per app.
+// This code is adjusted specifically for 9.4. In newer EVE versions,
+// this is implemented using objtonum package which is not yet in 9.4.
+// We have to be careful here and replicate the format of persisted data
+// to match newer EVE versions and enable seamless upgrades.
+
+package zedrouter
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/lf-edge/eve/pkg/pillar/types"
+	uuid "github.com/satori/go.uuid"
+)
+
+func getAppMacGeneratorID(ctx *zedrouterContext, appUUID uuid.UUID) (int, error) {
+	rawItem, err := ctx.pubAppMACGenerator.Get(appUUID.String())
+	if err != nil {
+		log.Errorf("failed to get published MAC generator ID for app %v: %v",
+			appUUID, err)
+		return 0, err
+	}
+	item, ok := rawItem.(types.AppMACGenerator)
+	if !ok {
+		return 0, fmt.Errorf("invalid item type: %T, expected AppMACGenerator", rawItem)
+	}
+	return item.Number, nil
+}
+
+func publishAppMacGeneratorID(ctx *zedrouterContext, appUUID uuid.UUID, macGenID int) error {
+	now := time.Now()
+	item := types.AppMACGenerator{
+		UuidToNum: types.UuidToNum{
+			UUID:        appUUID,
+			CreateTime:  now,
+			LastUseTime: now,
+			InUse:       true,
+			NumType:     "appMACGenerator",
+			Number:      macGenID,
+		},
+	}
+	err := ctx.pubAppMACGenerator.Publish(item.Key(), item)
+	if err != nil {
+		log.Errorf("failed to publish MAC generator ID for app %v: %v",
+			appUUID, err)
+	}
+	return err
+}
+
+func unpublishAppMacGeneratorID(ctx *zedrouterContext, appUUID uuid.UUID) error {
+	err := ctx.pubAppMACGenerator.Unpublish(appUUID.String())
+	if err != nil {
+		log.Errorf("failed to un-publish MAC generator ID for app %v: %v",
+			appUUID, err)
+	}
+	return err
+}

pkg/pillar/cmd/zedrouter/zedrouter.go

@@ -74,6 +74,7 @@ type zedrouterContext struct {
 	subGlobalConfig        pubsub.Subscription
 	GCInitialized          bool
 	pubUuidToNum           pubsub.Publication
+	pubAppMACGenerator     pubsub.Publication
 	dhcpLeases             []dnsmasqLease
 	subLocationInfo        pubsub.Subscription
 	subWwanStatus          pubsub.Subscription
@@ -173,6 +174,16 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 	}
 	pubUuidToNum.ClearRestarted()
 
+	pubAppMACGenerator, err := ps.NewPublication(pubsub.PublicationOptions{
+		AgentName:  agentName,
+		Persistent: true,
+		TopicType:  types.AppMACGenerator{},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	pubAppMACGenerator.ClearRestarted()
+
 	pubUUIDPairAndIfIdxToNum, err := ps.NewPublication(pubsub.PublicationOptions{
 		AgentName:  agentName,
 		Persistent: true,
@@ -262,6 +273,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 
 	zedrouterCtx.deviceNetworkStatus = &types.DeviceNetworkStatus{}
 	zedrouterCtx.pubUuidToNum = pubUuidToNum
+	zedrouterCtx.pubAppMACGenerator = pubAppMACGenerator
 	zedrouterCtx.pubUUIDPairAndIfIdxToNum = pubUUIDPairAndIfIdxToNum
 
 	// Create publish before subscribing and activating subscriptions
@@ -996,6 +1008,30 @@ func handleAppNetworkCreate(ctxArg interface{}, key string, configArg interface{
 		PendingAdd:     true,
 		DisplayName:    config.DisplayName,
 	}
+
+	// For app already deployed (before node reboot), keep using the same MAC address
+	// generator. Changing MAC addresses could break network config inside the app.
+	macGenerator, err := getAppMacGeneratorID(ctx, config.UUIDandVersion.UUID)
+	if err != nil || macGenerator == types.MACGeneratorUnspecified {
+		// New app or an existing app but without MAC generator ID persisted.
+		if ctx.localLegacyMACAddr {
+			// Use older node-scoped MAC address generator.
+			macGenerator = types.MACGeneratorNodeScoped
+		} else {
+			// Use newer (and preferred) globally-scoped MAC address generator.
+			macGenerator = types.MACGeneratorGloballyScoped
+		}
+		// Remember which MAC generator is being used for this app.
+		err = publishAppMacGeneratorID(ctx, config.UUIDandVersion.UUID, macGenerator)
+		if err != nil {
+			err = fmt.Errorf("failed to persist MAC generator ID for app %s/%s: %v",
+				config.UUIDandVersion.UUID, config.DisplayName, err)
+			log.Errorf("handleAppNetworkCreate(%v): %v", config.UUIDandVersion.UUID, err)
+			addError(ctx, &status, "handleAppNetworkCreate", err)
+			return
+		}
+	}
+	status.MACGenerator = macGenerator
 	publishAppNetworkStatus(ctx, &status)
 
 	// allocate application numbers on underlay network
@@ -1201,8 +1237,7 @@ func appNetworkDoActivateUnderlayNetwork(
 	if ulConfig.AppMacAddr != nil {
 		appMac = ulConfig.AppMacAddr.String()
 	} else {
-		appMac = generateAppMac(status.UUIDandVersion.UUID,
-			ulNum, status.AppNum, netInstStatus)
+		appMac = generateAppMac(ulNum, status, netInstStatus)
 	}
 	log.Functionf("appMac %s\n", appMac)
 
@@ -1337,27 +1372,39 @@ func appNetworkDoActivateUnderlayNetwork(
 // Since these MAC addresses will not appear on external Ethernet networks, we can also
 // use OUI octets for randomness. Only I/G and U/L bits need to stay constant and set
 // appropriately.
-func generateAppMac(appUUID uuid.UUID, ulNum int, appNum int,
+func generateAppMac(ulNum int, appStatus *types.AppNetworkStatus,
 	netInstStatus *types.NetworkInstanceStatus) string {
 	h := sha256.New()
-	h.Write(appUUID[:])
+	h.Write(appStatus.UUIDandVersion.UUID[:])
 	h.Write(netInstStatus.UUIDandVersion.UUID[:])
 	nums := make([]byte, 2)
 	nums[0] = byte(ulNum)
-	nums[1] = byte(appNum)
+	nums[1] = byte(appStatus.AppNum)
 	h.Write(nums)
 	hash := h.Sum(nil)
 	switch netInstStatus.Type {
 	case types.NetworkInstanceTypeSwitch:
+		// For switch network instances, we always generate globally-scoped
+		// MAC addresses. There is no difference in behaviour between MAC address
+		// generators in this case.
 		mac := net.HardwareAddr{0x02, 0x16, 0x3e, hash[0], hash[1], hash[2]}
 		return mac.String()
 	case types.NetworkInstanceTypeLocal, types.NetworkInstanceTypeCloud:
-		mac := net.HardwareAddr{hash[0], hash[1], hash[2], hash[3], hash[4], hash[5]}
-		// Mark this MAC address as unicast by setting the I/G bit to zero.
-		mac[0] &= ^byte(1)
-		// Mark this MAC address as locally administered by setting the U/L bit to 1.
-		mac[0] |= byte(1 << 1)
-		return mac.String()
+		switch appStatus.MACGenerator {
+		case types.MACGeneratorNodeScoped:
+			mac := net.HardwareAddr{0x00, 0x16, 0x3e, 0x00,
+				byte(ulNum), byte(appStatus.AppNum)}
+			return mac.String()
+		case types.MACGeneratorGloballyScoped:
+			mac := net.HardwareAddr{hash[0], hash[1], hash[2], hash[3], hash[4], hash[5]}
+			// Mark this MAC address as unicast by setting the I/G bit to zero.
+			mac[0] &= ^byte(1)
+			// Mark this MAC address as locally administered by setting the U/L bit to 1.
+			mac[0] |= byte(1 << 1)
+			return mac.String()
+		default:
+			log.Fatalf("undefined MAC generator")
+		}
 	default:
 		log.Fatalf("unsupported network instance type")
 	}
@@ -1903,6 +1950,7 @@ func handleDelete(ctx *zedrouterContext, key string,
 	unpublishAppNetworkStatus(ctx, status)
 
 	appNumFree(ctx, status.UUIDandVersion.UUID, true)
+	unpublishAppMacGeneratorID(ctx, status.UUIDandVersion.UUID)
 	appNumsOnUNetFree(ctx, status)
 	// Did this free up any last references against any Network Instance Status?
 	for ulNum := 0; ulNum < len(status.UnderlayNetworkList); ulNum++ {

pkg/pillar/types/zedroutertypes.go

@@ -148,6 +148,8 @@ type AppNetworkStatus struct {
 	GetStatsIPAddr       net.IP
 	UnderlayNetworkList  []UnderlayNetworkStatus
 	AwaitNetworkInstance bool // If any Missing flag is set in the networks
+	// ID of the MAC generator variant that was used to generate MAC addresses for this app.
+	MACGenerator int
 	// Any errors from provisioning the network
 	// ErrorAndTime provides SetErrorNow() and ClearError()
 	ErrorAndTime
@@ -3760,3 +3762,55 @@ type AppBlobsAvailable struct {
 type AppInfo struct {
 	AppBlobs []AppBlobsAvailable
 }
+
+// AppMACGenerator persistently stores ID of the MAC generator that was used to generate
+// MAC addresses for interfaces of a given app.
+type AppMACGenerator struct {
+	UuidToNum
+}
+
+// LogCreate logs newly added AppMACGenerator entry.
+func (g AppMACGenerator) LogCreate(logBase *base.LogObject) {
+	logObject := base.NewLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	logObject.Noticef("AppMACGenerator item create")
+}
+
+// LogModify logs modified AppMACGenerator entry.
+func (g AppMACGenerator) LogModify(logBase *base.LogObject, old interface{}) {
+	logObject := base.EnsureLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	oldEntry, ok := old.(AppMACGenerator)
+	if !ok {
+		logObject.Clone().Fatalf("LogModify: old object is not of AppMACGenerator type")
+	}
+	logObject.CloneAndAddField("diff", cmp.Diff(oldEntry, g)).
+		Noticef("AppMACGenerator item modify")
+}
+
+// LogDelete logs deleted AppMACGenerator entry.
+func (g AppMACGenerator) LogDelete(logBase *base.LogObject) {
+	logObject := base.EnsureLogObject(logBase, base.AppMACGeneratorLogType, "",
+		g.UUID, g.LogKey())
+	logObject.Noticef("AppMACGenerator item delete")
+	base.DeleteLogObject(logBase, g.LogKey())
+}
+
+// LogKey identifies AppMACGenerator entry for logging purposes.
+func (g AppMACGenerator) LogKey() string {
+	return string(base.AppMACGeneratorLogType) + "-" + g.Key()
+}
+
+// IDs assigned to different variants of MAC generators.
+const (
+	// MACGeneratorUnspecified : MAC generator is not specified.
+	MACGeneratorUnspecified = 0
+	// MACGeneratorNodeScoped generates MAC addresses which are guaranteed to be unique
+	// only within the scope of the given single device.
+	// The exception are MAC addresses generated for switch network instances,
+	// which are always generated with global scope.
+	MACGeneratorNodeScoped = 1
+	// MACGeneratorGloballyScoped generates MAC addresses which are with high probability
+	// unique globally, i.e. across entire fleet of devices.
+	MACGeneratorGloballyScoped = 2
+)