Implement features of the new cellular APIs

This is the second part of the commit series implementing the new
cellular APIs in EVE. The first part was mostly about extending zedagent
microservice to support parsing of the new protobuf messages and extending
structures used for delivering config and status between NIM and wwan
microservice. But wwan and for the most part also NIM still worked only
in the backward-compatible mode.
In this commit, we make substantial enhancements to the wwan microservice
and some additions to NIM to implement the new cellular APIs in their
full scope.

Part II changelog:
* implemented username/password authentication for cellular networks
* added logging with configurable verbose level to wwan microservice
* upgraded QMI and MBIM CLI tools to bring in some later added commands
  needed to implement the new cellular APIs. Note that some of the patches
  we were applying to the previously used versions are already included
  in the newer versions and we can therefore remove these patch files
* improved robustness of the wwan microservice - decreased reaction time
  and improved ability to recover from lost/broken connectivity
* finalized support for multiple cellular modems (just few changes were
  needed, most of the code was already there)
* more cellular status data are now published, e.g.: time when the
  connection was established, info about currently used provided,
  used Radio Access Technology (UMTS / LTE / ...), modem manufacturer,
  etc.
* added support for modems with multiple SIMs in the DSSS mode
  (single standby)

Signed-off-by: Milan Lenco <milan@zededa.com>

pkg/pillar/cipher/cipher.go

@@ -24,6 +24,8 @@ func getEncryptionBlock(
 	decBlock.DsPassword = zconfigDecBlockPtr.DsPassword
 	decBlock.WifiUserName = zconfigDecBlockPtr.WifiUserName
 	decBlock.WifiPassword = zconfigDecBlockPtr.WifiPassword
+	decBlock.CellNetUsername = zconfigDecBlockPtr.CellularNetUsername
+	decBlock.CellNetPassword = zconfigDecBlockPtr.CellularNetPassword
 	decBlock.ProtectedUserData = zconfigDecBlockPtr.ProtectedUserData
 	return decBlock
 }

pkg/pillar/dpcreconciler/linux.go

@@ -5,26 +5,27 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/Luzifer/go-openssl/v4"
 	dg "github.com/lf-edge/eve/libs/depgraph"
 	"github.com/lf-edge/eve/libs/reconciler"
 	"github.com/lf-edge/eve/pkg/pillar/base"
 	"github.com/lf-edge/eve/pkg/pillar/cipher"
 	"github.com/lf-edge/eve/pkg/pillar/devicenetwork"
-	"github.com/lf-edge/eve/pkg/pillar/iptables"
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
-	"github.com/lf-edge/eve/pkg/pillar/types"
-	"github.com/vishvananda/netlink"
-
 	generic "github.com/lf-edge/eve/pkg/pillar/dpcreconciler/genericitems"
 	linux "github.com/lf-edge/eve/pkg/pillar/dpcreconciler/linuxitems"
+	"github.com/lf-edge/eve/pkg/pillar/iptables"
 	"github.com/lf-edge/eve/pkg/pillar/netmonitor"
+	"github.com/lf-edge/eve/pkg/pillar/pubsub"
+	"github.com/lf-edge/eve/pkg/pillar/types"
 	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
+	"github.com/vishvananda/netlink"
 )
 
 // Device connectivity configuration is modeled using dependency graph (see libs/depgraph).
@@ -147,6 +148,13 @@ const (
 	kubeCNIBridge = "cni0"
 )
 
+const (
+	// File with random ID that is regenerated by the kernel on each boot.
+	// Used as a passphrase to encrypt cellular network password before publishing
+	// from NIM to the wwan microservice.
+	bootIDFile = "/proc/sys/kernel/random/boot_id"
+)
+
 // LinuxDpcReconciler is a DPC-reconciler for Linux network stack,
 // i.e. it configures and uses Linux networking to provide device connectivity.
 type LinuxDpcReconciler struct {
@@ -183,11 +191,11 @@ type LinuxDpcReconciler struct {
 	resumeReconcile  chan struct{}
 	resumeAsync      <-chan string // nil if no async ops
 
-	prevArgs     Args
-	prevStatus   ReconcileStatus
-	radioSilence types.RadioSilence
-
-	HVTypeKube bool
+	prevArgs             Args
+	prevStatus           ReconcileStatus
+	radioSilence         types.RadioSilence
+	HVTypeKube           bool
+	encCellularPasswords map[string]string
 }
 
 type pendingReconcile struct {
@@ -236,6 +244,7 @@ func (r *LinuxDpcReconciler) init() (startWatcher func()) {
 	configurator := registry.GetConfigurator(generic.Wwan{})
 	r.wwanConfigurator = configurator.(*generic.WwanConfigurator)
 	r.watcherControl = make(chan watcherCtrl, 10)
+	r.encCellularPasswords = make(map[string]string)
 	netEvents := r.NetworkMonitor.WatchEvents(
 		context.Background(), "linux-dpc-reconciler")
 	go r.watcher(netEvents)
@@ -371,6 +380,7 @@ func (r *LinuxDpcReconciler) Reconcile(ctx context.Context, args Args) Reconcile
 		}
 		if r.gcpChanged(args.GCP) {
 			r.addPendingReconcile(ACLsSG, "GCP change", false)
+			r.addPendingReconcile(WirelessSG, "GCP change", false)
 		}
 		if r.aaChanged(args.AA) {
 			changed := r.updateCurrentPhysicalIO(args.DPC, args.AA)
@@ -420,7 +430,7 @@ func (r *LinuxDpcReconciler) Reconcile(ctx context.Context, args Args) Reconcile
 		case L3SG:
 			intSG = r.getIntendedL3Cfg(args.DPC)
 		case WirelessSG:
-			intSG = r.getIntendedWirelessCfg(args.DPC, args.AA, args.RS)
+			intSG = r.getIntendedWirelessCfg(args.DPC, args.AA, args.RS, args.GCP)
 		case ACLsSG:
 			intSG = r.getIntendedACLs(args.DPC, args.GCP)
 		default:
@@ -615,6 +625,11 @@ func (r *LinuxDpcReconciler) gcpChanged(newGCP types.ConfigItemValueMap) bool {
 	if prevAllowVNC != newAllowVNC {
 		return true
 	}
+	prevWwanLogLevel := r.prevArgs.GCP.AgentSettingStringValue("wwan", types.LogLevel)
+	newWwanLogLevel := newGCP.AgentSettingStringValue("wwan", types.LogLevel)
+	if prevWwanLogLevel != newWwanLogLevel {
+		return true
+	}
 	return false
 }
 
@@ -815,7 +830,7 @@ func (r *LinuxDpcReconciler) updateIntendedState(args Args) {
 	r.intendedState.PutSubGraph(r.getIntendedPhysicalIO(args.DPC))
 	r.intendedState.PutSubGraph(r.getIntendedLogicalIO(args.DPC))
 	r.intendedState.PutSubGraph(r.getIntendedL3Cfg(args.DPC))
-	r.intendedState.PutSubGraph(r.getIntendedWirelessCfg(args.DPC, args.AA, args.RS))
+	r.intendedState.PutSubGraph(r.getIntendedWirelessCfg(args.DPC, args.AA, args.RS, args.GCP))
 	r.intendedState.PutSubGraph(r.getIntendedACLs(args.DPC, args.GCP))
 }
 
@@ -1226,7 +1241,8 @@ func (r *LinuxDpcReconciler) getIntendedArps(dpc types.DevicePortConfig) dg.Grap
 }
 
 func (r *LinuxDpcReconciler) getIntendedWirelessCfg(dpc types.DevicePortConfig,
-	aa types.AssignableAdapters, radioSilence types.RadioSilence) dg.Graph {
+	aa types.AssignableAdapters, radioSilence types.RadioSilence,
+	gcp types.ConfigItemValueMap) dg.Graph {
 	graphArgs := dg.InitArgs{
 		Name:        WirelessSG,
 		Description: "Configuration for wireless connectivity",
@@ -1244,7 +1260,7 @@ func (r *LinuxDpcReconciler) getIntendedWirelessCfg(dpc types.DevicePortConfig,
 		// because when the DPC arrives, wwan microservice won't be blocked on retrieving
 		// some state data but will be ready to apply the config immediately.
 		intendedWirelessCfg.PutItem(
-			r.getIntendedWwanConfig(dpc, aa, rsImposed), nil)
+			r.getIntendedWwanConfig(dpc, aa, rsImposed, gcp), nil)
 	}
 	return intendedWirelessCfg
 }
@@ -1333,11 +1349,14 @@ func (r *LinuxDpcReconciler) getWifiCredentials(wifi types.WifiConfig) (types.En
 }
 
 func (r *LinuxDpcReconciler) getIntendedWwanConfig(dpc types.DevicePortConfig,
-	aa types.AssignableAdapters, radioSilence bool) dg.Item {
+	aa types.AssignableAdapters, radioSilence bool, gcp types.ConfigItemValueMap) dg.Item {
+	logLevel := gcp.AgentSettingStringValue("wwan", types.LogLevel)
 	config := types.WwanConfig{
 		RadioSilence: radioSilence,
+		Verbose:      logLevel == "debug" || logLevel == "trace",
 		Networks:     []types.WwanNetworkConfig{},
 	}
+
 	for _, port := range dpc.Ports {
 		if port.WirelessCfg.WType != types.WirelessTypeCellular {
 			continue
@@ -1370,6 +1389,26 @@ func (r *LinuxDpcReconciler) getIntendedWwanConfig(dpc types.DevicePortConfig,
 				"skipping", port.Logicallabel)
 			continue
 		}
+		decBlock, err := r.getWwanCredentials(accessPoint)
+		if err != nil {
+			r.Log.Error(err)
+			r.Log.Warnf("getIntendedWwanConfig: failed to get credentials for port %s, "+
+				"skipping", port.Logicallabel)
+			continue
+		}
+
+		var encPassword string
+		if decBlock.CellNetUsername != "" {
+			encPassword, err = r.encryptCellularPassword(port.Logicallabel,
+				decBlock.CellNetPassword)
+			if err != nil {
+				r.Log.Error(err)
+				r.Log.Warnf(
+					"getIntendedWwanConfig: failed to encrypt user password for port %s, "+
+						"skipping", port.Logicallabel)
+				continue
+			}
+		}
 		// Prefer USB and PCI addresses over interface name.
 		// Plus we want to avoid changing /run/wwan/config.json just to put there
 		// discovered interface name (it is the wwan microservice that discovered it anyway
@@ -1383,23 +1422,92 @@ func (r *LinuxDpcReconciler) getIntendedWwanConfig(dpc types.DevicePortConfig,
 			physAddress.Interface = port.IfName
 		}
 		network := types.WwanNetworkConfig{
-			// TODO: Username + Password (will be done in the next PR)
-			LogicalLabel:     port.Logicallabel,
-			PhysAddrs:        physAddress,
-			SIMSlot:          accessPoint.SIMSlot,
-			APN:              accessPoint.APN,
-			PreferredPLMNs:   accessPoint.PreferredPLMNs,
-			PreferredRATs:    accessPoint.PreferredRATs,
-			ForbidRoaming:    accessPoint.ForbidRoaming,
-			Proxies:          port.Proxies,
-			Probe:            port.WirelessCfg.Cellular.Probe,
-			LocationTracking: port.WirelessCfg.Cellular.LocationTracking,
+			LogicalLabel:      port.Logicallabel,
+			PhysAddrs:         physAddress,
+			SIMSlot:           accessPoint.SIMSlot,
+			APN:               accessPoint.APN,
+			AuthProtocol:      accessPoint.AuthProtocol,
+			Username:          decBlock.CellNetUsername,
+			EncryptedPassword: encPassword,
+			PreferredPLMNs:    accessPoint.PreferredPLMNs,
+			PreferredRATs:     accessPoint.PreferredRATs,
+			ForbidRoaming:     accessPoint.ForbidRoaming,
+			Proxies:           port.Proxies,
+			Probe:             port.WirelessCfg.Cellular.Probe,
+			LocationTracking:  port.WirelessCfg.Cellular.LocationTracking,
 		}
 		config.Networks = append(config.Networks, network)
 	}
 	return generic.Wwan{Config: config}
 }
 
+func (r *LinuxDpcReconciler) getWwanCredentials(ap *types.CellularAccessPoint) (
+	decBlock types.EncryptionBlock, err error) {
+	if !ap.IsCipher {
+		return decBlock, nil
+	}
+	decryptAvailable := r.SubControllerCert != nil && r.SubEdgeNodeCert != nil
+	if !decryptAvailable {
+		r.CipherMetrics.RecordFailure(r.Log, types.NotReady)
+		return decBlock, fmt.Errorf(
+			"missing certificates for decryption of cellular network credentials")
+	}
+	status, decBlock, err := cipher.GetCipherCredentials(
+		&cipher.DecryptCipherContext{
+			Log:               r.Log,
+			AgentName:         r.AgentName,
+			AgentMetrics:      r.CipherMetrics,
+			SubControllerCert: r.SubControllerCert,
+			SubEdgeNodeCert:   r.SubEdgeNodeCert,
+		},
+		ap.CipherBlockStatus)
+	if r.PubCipherBlockStatus != nil {
+		r.PubCipherBlockStatus.Publish(status.Key(), status)
+	}
+	if err != nil {
+		r.CipherMetrics.RecordFailure(r.Log, types.DecryptFailed)
+		return decBlock, fmt.Errorf(
+			"failed to decrypt cellular network credentials: %v", err)
+	}
+	return decBlock, nil
+}
+
+// Encrypt user password using AES-256-CBC with the key derived by the PBKDF2
+// method, taking kernel-generated /proc/sys/kernel/random/boot_id as the input.
+// Note that even though the config with the password is passed from NIM to the wwan
+// microservice using the *in-memory only* /run filesystem, we still encrypt
+// the password to avoid accidental exposure when the content of /run/wwan
+// is dumped as part of a customer issue report.
+func (r *LinuxDpcReconciler) encryptCellularPassword(portLL, password string) (
+	ciphertext string, err error) {
+	// Key includes port logical label so that even if user uses the same password
+	// for multiple APNs, the encrypted passwords are different for more secrecy.
+	key := portLL + "/" + password
+	ciphertext, alreadyEncrypted := r.encCellularPasswords[key]
+	if alreadyEncrypted {
+		// EncryptBytes uses random salt meaning that it will return different
+		// ciphertext on each run even for the same password.
+		// However, we do not want to change generated /run/wwan/config.json unless
+		// there is an actual config change (triggers many operations in the wwan
+		// microservice), so we remember already encrypted passwords.
+		return ciphertext, nil
+	}
+	bootID, err := os.ReadFile(bootIDFile)
+	if err != nil {
+		return "", err
+	}
+	passphrase := strings.TrimSpace(string(bootID))
+	openSSL := openssl.New()
+	encBytes, err := openSSL.EncryptBytes(
+		passphrase, []byte(password), openssl.PBKDF2SHA256)
+	if err != nil {
+		return "", err
+	}
+	ciphertext = string(encBytes)
+	r.encCellularPasswords[key] = ciphertext
+	return ciphertext, nil
+}
+
 func (r *LinuxDpcReconciler) getIntendedACLs(
 	dpc types.DevicePortConfig, gcp types.ConfigItemValueMap) dg.Graph {
 	graphArgs := dg.InitArgs{

pkg/pillar/go.mod

@@ -3,6 +3,7 @@ module github.com/lf-edge/eve/pkg/pillar
 go 1.20
 
 require (
+	github.com/Luzifer/go-openssl/v4 v4.1.0
 	github.com/anatol/smart.go v0.0.0-20220615232124-371056cd18c3
 	github.com/bicomsystems/go-libzfs v0.4.0
 	github.com/containerd/cgroups v1.0.4

pkg/pillar/go.sum

@@ -157,6 +157,8 @@ github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5H
 github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo=
 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
+github.com/Luzifer/go-openssl/v4 v4.1.0 h1:8qi3Z6f8Aflwub/Cs4FVSmKUEg/lC8GlODbR2TyZ+nM=
+github.com/Luzifer/go-openssl/v4 v4.1.0/go.mod h1:3i1T3Pe6eQK19d86WhuQzjLyMwBaNmGmt3ZceWpWVa4=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -1770,6 +1772,7 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

pkg/pillar/types/cipherinfotypes.go

@@ -143,5 +143,7 @@ type EncryptionBlock struct {
 	DsPassword        string
 	WifiUserName      string // If the authentication type is EAP
 	WifiPassword      string
+	CellNetUsername   string
+	CellNetPassword   string
 	ProtectedUserData string
 }

pkg/pillar/types/zedroutertypes.go

@@ -3047,7 +3047,6 @@ func (wc WwanConfig) Equal(wc2 WwanConfig) bool {
 // WwanNetworkConfig contains configuration for a single cellular network.
 // In case there are multiple SIM cards/slots in the modem, WwanNetworkConfig
 // contains config only for the activated one.
-// TODO: Add username + password (will be done in the next PR)
 type WwanNetworkConfig struct {
 	// Logical label in PhysicalIO.
 	LogicalLabel string        `json:"logical-label"`
@@ -3061,6 +3060,17 @@ type WwanNetworkConfig struct {
 	// Access Point Network to connect into.
 	// By default, it is "internet".
 	APN string `json:"apn"`
+	// Some cellular networks require authentication.
+	AuthProtocol WwanAuthProtocol `json:"auth-protocol"`
+	Username     string           `json:"username,omitempty"`
+	// User password (if provided) is encrypted using AES-256-CBC with key derived
+	// by the PBKDF2 method, taking kernel-generated /proc/sys/kernel/random/boot_id
+	// as the input.
+	// Note that even though the config with the password is passed from NIM to the wwan
+	// microservice using the *in-memory only* /run filesystem, we still encrypt the password
+	// to avoid accidental exposure when the content of /run/wwan is dumped as part
+	// of a customer issue report.
+	EncryptedPassword string `json:"encrypted-password,omitempty"`
 	// The set of cellular network operators that modem should preferably try to register
 	// and connect into.
 	// Network operator should be referenced by PLMN (Public Land Mobile Network) code,
@@ -3137,6 +3147,11 @@ func (wnc WwanNetworkConfig) Equal(wnc2 WwanNetworkConfig) bool {
 		wnc.APN != wnc2.APN {
 		return false
 	}
+	if wnc.AuthProtocol != wnc2.AuthProtocol ||
+		wnc.Username != wnc2.Username ||
+		wnc.EncryptedPassword != wnc2.EncryptedPassword {
+		return false
+	}
 	if !generics.EqualLists(wnc.PreferredPLMNs, wnc2.PreferredPLMNs) ||
 		!generics.EqualLists(wnc.PreferredRATs, wnc2.PreferredRATs) ||
 		wnc.ForbidRoaming != wnc2.ForbidRoaming {