supply scanned controllers explicitly in format and write config call…

…s instead of using bdev cache

Features: control
Required-githooks: true

Signed-off-by: Tom Nabarro <tom.nabarro@intel.com>

src/control/server/ctl_storage_rpc.go

@@ -113,7 +113,7 @@ func bdevScanToProtoResp(scan scanBdevsFn, bdevCfgs storage.TierConfigs) (*ctlpb
 		return nil, err
 	}
 
-	if bdevCfgs.HaveBdevs() {
+	if bdevCfgs.HaveRealNVMe() {
 		// Update proto Ctrlrs with role info for offline display.
 		for _, c := range pbCtrlrs {
 			pciAddrStr, err := ctrlrToPciStr(c)
@@ -148,11 +148,11 @@ func bdevScanEngines(ctx context.Context, cs *ControlService, req *ctlpb.ScanNvm
 	instances := cs.harness.Instances()
 	resp := &ctlpb.ScanNvmeResp{}
 
-	for _, ei := range instances {
+	for _, engine := range instances {
 		eReq := new(ctlpb.ScanNvmeReq)
 		*eReq = *req
 		if req.Meta {
-			ms, rs, err := computeMetaRdbSz(cs, ei, nsps)
+			ms, rs, err := computeMetaRdbSz(cs, engine, nsps)
 			if err != nil {
 				return nil, errors.Wrap(err, "computing meta and rdb size")
 			}
@@ -161,9 +161,9 @@ func bdevScanEngines(ctx context.Context, cs *ControlService, req *ctlpb.ScanNvm
 
 		// If partial number of engines return results, indicate errors for non-ready
 		// engines whilst returning successful scanmresults.
-		respEng, err := scanEngineBdevs(ctx, ei, eReq)
+		respEng, err := scanEngineBdevs(ctx, engine, eReq)
 		if err != nil {
-			err = errors.Wrapf(err, "instance %d", ei.Index())
+			err = errors.Wrapf(err, "instance %d", engine.Index())
 			if errLast == nil && len(instances) > 1 {
 				errLast = err // Save err to preserve partial results.
 				cs.log.Error(err.Error())
@@ -859,28 +859,48 @@ type formatNvmeReq struct {
 
 func formatNvme(ctx context.Context, req formatNvmeReq, resp *ctlpb.StorageFormatResp) {
 	// Allow format to complete on one instance even if another fails
-	for idx, ei := range req.instances {
+	for idx, engine := range req.instances {
 		_, hasError := req.errored[idx]
 		_, skipped := req.skipped[idx]
 		if hasError || (skipped && !req.mdFormatted) {
-			// if scm failed to format or was already formatted, indicate skipping bdev format
-			ret := ei.newCret(storage.NilBdevAddress, nil)
-			ret.State.Info = fmt.Sprintf(msgNvmeFormatSkip, ei.Index())
+			// If scm failed to format or was already formatted, skip bdev format.
+			ret := engine.newCret(storage.NilBdevAddress, nil)
+			ret.State.Info = fmt.Sprintf(msgNvmeFormatSkip, engine.Index())
 			resp.Crets = append(resp.Crets, ret)
 			continue
 		}
 
+		respBdevs, err := scanEngineBdevs(ctx, engine, new(ctlpb.ScanNvmeReq))
+		if err != nil {
+			if errors.Is(err, errEngineBdevScanEmptyDevList) {
+				// No controllers assigned in config, continue.
+				continue
+			}
+			req.errored[idx] = err.Error()
+			resp.Crets = append(resp.Crets, engine.newCret("", err))
+			continue
+		}
+
+		// Convert proto ctrlr scan results to native when calling into storage provider.
+		pbCtrlrs := proto.NvmeControllers(respBdevs.Ctrlrs)
+		ctrlrs, err := pbCtrlrs.ToNative()
+		if err != nil {
+			req.errored[idx] = err.Error()
+			resp.Crets = append(resp.Crets, engine.newCret("", err))
+			continue
+		}
+
 		// SCM formatted correctly on this instance, format NVMe
-		cResults := ei.StorageFormatNVMe()
+		cResults := formatEngineBdevs(engine, ctrlrs)
 		if cResults.HasErrors() {
 			req.errored[idx] = cResults.Errors()
 			resp.Crets = append(resp.Crets, cResults...)
 			continue
 		}
 
-		if err := ei.GetStorage().WriteNvmeConfig(ctx, req.log); err != nil {
+		if err := engine.GetStorage().WriteNvmeConfig(ctx, req.log, ctrlrs); err != nil {
 			req.errored[idx] = err.Error()
-			cResults = append(cResults, ei.newCret("", err))
+			cResults = append(cResults, engine.newCret("", err))
 		}
 
 		resp.Crets = append(resp.Crets, cResults...)
@@ -1024,8 +1044,9 @@ func (cs *ControlService) StorageNvmeAddDevice(ctx context.Context, req *ctlpb.N
 	}
 	cs.log.Debugf("updated bdev list: %+v", tierCfg.Bdev.DeviceList)
 
+	// TODO: Supply scan results for VMD backing device address mapping.
 	resp = new(ctlpb.NvmeAddDeviceResp)
-	if err := engineStorage.WriteNvmeConfig(ctx, cs.log); err != nil {
+	if err := engineStorage.WriteNvmeConfig(ctx, cs.log, nil); err != nil {
 		err = errors.Wrapf(err, "write nvme config for engine %d", engineIndex)
 		cs.log.Error(err.Error())