diff --git a/pkg/dom0-ztools/efi/Makefile b/pkg/dom0-ztools/efi/Makefile index 13e6ce4c08..af3506ce07 100644 --- a/pkg/dom0-ztools/efi/Makefile +++ b/pkg/dom0-ztools/efi/Makefile @@ -29,4 +29,4 @@ clean: %.efi: %.so objcopy -j .text -j .sdata -j .data -j .dynamic \ -j .dynsym -j .rel -j .rela -j .reloc \ - --target=efi-app-$(ARCH) $^ $@ \ No newline at end of file + --target=efi-app-$(ARCH) $^ $@ diff --git a/pkg/dom0-ztools/efi/efi_run.sh b/pkg/dom0-ztools/efi/efi_run.sh index 67a35ab288..a5a62131a4 100644 --- a/pkg/dom0-ztools/efi/efi_run.sh +++ b/pkg/dom0-ztools/efi/efi_run.sh @@ -8,4 +8,4 @@ cp /hostfs/etc/ovmf/eve_efi.img /run/kvm/eve_efi.img -drive "file=/run/kvm/eve_efi.img,format=raw" \ -smbios "type=11,value=${2}" \ -net none \ - -nographic \ No newline at end of file + -nographic diff --git a/pkg/dom0-ztools/efi/main.c b/pkg/dom0-ztools/efi/main.c index 48cf023e58..672895410f 100644 --- a/pkg/dom0-ztools/efi/main.c +++ b/pkg/dom0-ztools/efi/main.c @@ -602,7 +602,7 @@ GetResolutionConfig( _Return: FreePool(ConfigValue); - return EFI_SUCCESS; + return Status; } EFI_STATUS @@ -694,4 +694,4 @@ efi_main( } return EFI_SUCCESS; -} \ No newline at end of file +} diff --git a/pkg/dom0-ztools/efi/startup.nsh b/pkg/dom0-ztools/efi/startup.nsh index 2c6331d110..b46a0bd962 100644 --- a/pkg/dom0-ztools/efi/startup.nsh +++ b/pkg/dom0-ztools/efi/startup.nsh @@ -1,3 +1,3 @@ fs0: efi_eveutil.efi -reset -s \ No newline at end of file +reset -s diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index fcfabd3e86..93fed3c28d 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -1119,14 +1119,21 @@ func maybeRetryBoot(ctx *domainContext, status *types.DomainStatus) { } defer file.Close() - if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil { + extra := types.ExtraArgs{ + AgentName: agentName, + Ps: ctx.ps, + WarnTime: warningTime, + ErrTime: errorTime, + } + + if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil { log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err) status.VirtualTPM = false } else { status.VirtualTPM = true } - if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file); err != nil { + if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file, &extra); err != nil { //it is retry, so omit error log.Errorf("Failed to create DomainStatus from %+v: %s", config, err) @@ -1671,7 +1678,14 @@ func doActivate(ctx *domainContext, config types.DomainConfig, } defer file.Close() - if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil { + extra := types.ExtraArgs{ + AgentName: agentName, + Ps: ctx.ps, + WarnTime: warningTime, + ErrTime: errorTime, + } + + if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil { log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err) status.VirtualTPM = false } else { @@ -1679,7 +1693,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, } globalConfig := agentlog.GetGlobalConfig(log, ctx.subGlobalConfig) - if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file); err != nil { + if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file, &extra); err != nil { log.Errorf("Failed to create DomainStatus from %+v: %s", config, err) status.SetErrorNow(err.Error()) diff --git a/pkg/pillar/hypervisor/containerd.go b/pkg/pillar/hypervisor/containerd.go index fc8fbcc7c2..e039e24323 100644 --- a/pkg/pillar/hypervisor/containerd.go +++ b/pkg/pillar/hypervisor/containerd.go @@ -11,7 +11,6 @@ import ( "time" "github.com/lf-edge/eve/pkg/pillar/containerd" - "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/opencontainers/runtime-spec/specs-go" @@ -101,7 +100,7 @@ func (ctx ctrdContext) setupSpec(status *types.DomainStatus, config *types.Domai } func (ctx ctrdContext) Setup(status types.DomainStatus, config types.DomainConfig, - aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error { + aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error { if status.OCIConfigDir == "" { return logError("failed to run domain %s: not based on an OCI image", status.DomainName) } @@ -327,7 +326,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) { return res, nil } -func (ctx ctrdContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error { +func (ctx ctrdContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error { return fmt.Errorf("not implemented") } diff --git a/pkg/pillar/hypervisor/kubevirt.go b/pkg/pillar/hypervisor/kubevirt.go index 4edb6bc5fb..f5de054f16 100644 --- a/pkg/pillar/hypervisor/kubevirt.go +++ b/pkg/pillar/hypervisor/kubevirt.go @@ -22,7 +22,6 @@ import ( "time" "github.com/lf-edge/eve/pkg/pillar/base" - "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -185,7 +184,7 @@ func (ctx kubevirtContext) Task(status *types.DomainStatus) types.Task { // Use eve DomainConfig and DomainStatus and generate k3s VMI config or a Pod config func (ctx kubevirtContext) Setup(status types.DomainStatus, config types.DomainConfig, - aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error { + aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error { diskStatusList := status.DiskStatusList domainName := status.DomainName @@ -1362,7 +1361,7 @@ func (ctx kubevirtContext) PCISameController(id1 string, id2 string) bool { return PCISameControllerGeneric(id1, id2) } -func (ctx kubevirtContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error { +func (ctx kubevirtContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error { return fmt.Errorf("not implemented") } diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index 8c4483fd15..c5286f3b08 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -4,9 +4,11 @@ package hypervisor import ( + "bytes" "fmt" "net" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -17,7 +19,6 @@ import ( zconfig "github.com/lf-edge/eve-api/go/config" "github.com/lf-edge/eve/pkg/pillar/agentlog" "github.com/lf-edge/eve/pkg/pillar/containerd" - "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/lf-edge/eve/pkg/pillar/utils" fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file" @@ -738,35 +739,18 @@ func getOVMFSettingsFilename(domainName string) (string, error) { return types.OVMFSettingsDir + "/" + domainUUID.String() + "_OVMF_VARS.fd", nil } -func prepareOVMFSettings(config types.DomainConfig, status types.DomainStatus, globalConfig *types.ConfigItemValueMap) error { +func prepareOVMFSettings(status types.DomainStatus) error { // Create the OVMF settings directory if it does not exist if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil { return logError("failed to create OVMF settings directory: %v", err) } - // Create a copy of the ovmf_vars.bin file in _ovmf_vars.bin + // Get the path for a copy of the ovmf_vars.bin file form of _OVMF_VARS.fd ovmfSettingsFile, err := getOVMFSettingsFilename(status.DomainName) if err != nil { return logError("failed to get OVMF settings file: %v", err) } - // Check if we need custom OVMF settings for the domain (the resolution) - fmlResolution := types.FmlResolutionUnset - if config.VirtualizationMode == types.FML { - // if we are not getting the resolution from the cloud-init, check the - // global config. - fmlResolution = status.FmlCustomResolution - if fmlResolution == types.FmlResolutionUnset { - if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil { - return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err) - } - } - } - // Find the necessary OVMF settings file - ovmfSettingsFileSrc := types.OVMFSettingsTemplate - if fmlResolution != types.FmlResolutionUnset { - ovmfSettingsFileSrc = types.CustomOVMFSettingsDir + "/OVMF_VARS_" + fmlResolution + ".fd" - } if _, err := os.Stat(ovmfSettingsFile); os.IsNotExist(err) { - if err := fileutils.CopyFile(ovmfSettingsFileSrc, ovmfSettingsFile); err != nil { + if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfSettingsFile); err != nil { return logError("failed to copy OVMF_VARS file: %v", err) } } @@ -790,7 +774,7 @@ func cleanupOVMFSettings(domainName string) error { // Setup sets up kvm func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig, - aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error { + aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error { diskStatusList := status.DiskStatusList domainName := status.DomainName @@ -814,9 +798,29 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig // for ARM produces a single QEMU_EFI.fd file that contains both OVMF_VARS.fd // and OVMF_CODE.fd. if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" { - if err := prepareOVMFSettings(config, status, globalConfig); err != nil { + // Copy the OVMF_VARS file and the eve_efi.img file to the domain directory + err := prepareOVMFSettings(status) + if err != nil { return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err) } + + fmlResolution := status.FmlCustomResolution + if fmlResolution == types.FmlResolutionUnset { + if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil { + return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err) + } + } + if fmlResolution != types.FmlResolutionUnset { + cfg := []string{ + "eve.fml.resolution:" + fmlResolution, + } + + logError("setOvmfVariables before call") + err := ctx.setOvmfVariables(cfg, status.DomainName, extra) + if err != nil { + return logError("failed to set OVMF variables for domain %s: %v", status.DomainName, err) + } + } } // first lets build the domain config @@ -846,6 +850,14 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig "-readconfig", file.Name(), "-pidfile", kvmStateDir+domainName+"/pid") + if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" { + args = append(args, "-drive", "file=/run/kvm/eve_efi.img,format=raw,if=none,id=drive-eve-efi") + args = append(args, "-device", "virtio-blk-pci,drive=drive-eve-efi") + args = append(args, "-smbios", "type=11,value=eve.try.all.boot.options:true") + } + + logError("guest args: %v", args) + spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir) if err != nil { @@ -1330,9 +1342,9 @@ func getQmpListenerSocket(domainName string) string { } // VirtualTPMSetup launches a vTPM instance for the domain -func (ctx KvmContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error { - if ps != nil { - wk := utils.NewWatchdogKick(ps, agentName, warnTime, errTime) +func (ctx KvmContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error { + if extra.Ps != nil { + wk := utils.NewWatchdogKick(extra.Ps, extra.AgentName, extra.WarnTime, extra.ErrTime) domainUUID, _, _, err := types.DomainnameToUUID(domainName) if err != nil { return fmt.Errorf("failed to extract UUID from domain name (vTPM setup): %v", err) @@ -1429,3 +1441,68 @@ func requestVtpmTermination(id uuid.UUID) error { return nil } + +// This is janky and hacky, but it is OK for testing +func (ctx KvmContext) setOvmfVariables(config []string, domainName string, extra *types.ExtraArgs) error { + cmd := exec.Command("/hostfs/usr/bin/ctr", "--namespace", "services.linuxkit", "t", "ls") + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return logError("setOvmfVariables error running ctr: %s\n", err) + } + output := out.String() + lines := strings.Split(output, "\n") + var xenToolsPID string + for _, line := range lines { + columns := strings.Fields(line) + if len(columns) > 2 && columns[0] == "xen-tools" { + xenToolsPID = columns[1] + break + } + } + if xenToolsPID != "" { + logError("setOvmfVariables The PID of xen-tools is: %s\n", xenToolsPID) + } else { + return logError("setOvmfVariables xen-tools not found.") + } + + ovmfSettingsFile, err := getOVMFSettingsFilename(domainName) + if err != nil { + return logError("setOvmfVariables failed to get OVMF settings file: %v", err) + } + + args := []string{"-F", "-a", "-t", xenToolsPID, + "/hostfs/etc/ovmf/efi_run.sh", + ovmfSettingsFile, + "eve.fml.resolution:1920x1080", + } + + logError("setOvmfVariables args: nsenter %v", args) + cmd = exec.Command("nsenter", args...) + if err := cmd.Start(); err != nil { + return logError("setOvmfVariables error starting nsenter: %s\n", err) + } + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + +waitLoop: + for { + select { + case err := <-done: + if err != nil { + logError("setOvmfVariables Process exited with error: %s\n", err) + } else { + logError("setOvmfVariables Process completed successfully.") + } + break waitLoop + default: + extra.Ps.StillRunning(extra.AgentName, extra.WarnTime, extra.ErrTime) + logError("setOvmfVariables still waiting.") + time.Sleep(1 * time.Second) + } + } + + return nil +} diff --git a/pkg/pillar/hypervisor/null.go b/pkg/pillar/hypervisor/null.go index 899c35ed27..c351986c35 100644 --- a/pkg/pillar/hypervisor/null.go +++ b/pkg/pillar/hypervisor/null.go @@ -6,9 +6,7 @@ package hypervisor import ( "fmt" "os" - "time" - "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" uuid "github.com/satori/go.uuid" @@ -69,7 +67,7 @@ func (ctx nullContext) Task(status *types.DomainStatus) types.Task { } func (ctx nullContext) Setup(types.DomainStatus, types.DomainConfig, - *types.AssignableAdapters, *types.ConfigItemValueMap, *os.File) error { + *types.AssignableAdapters, *types.ConfigItemValueMap, *os.File, *types.ExtraArgs) error { return nil } @@ -174,7 +172,7 @@ func (ctx nullContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) { return nil, nil } -func (ctx nullContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error { +func (ctx nullContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error { return fmt.Errorf("not implemented") } diff --git a/pkg/pillar/hypervisor/xen.go b/pkg/pillar/hypervisor/xen.go index 729f0d7432..394d180c26 100644 --- a/pkg/pillar/hypervisor/xen.go +++ b/pkg/pillar/hypervisor/xen.go @@ -11,9 +11,7 @@ import ( "runtime" "strconv" "strings" - "time" - "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/mem" @@ -121,7 +119,7 @@ func (ctx xenContext) Task(status *types.DomainStatus) types.Task { } func (ctx xenContext) Setup(status types.DomainStatus, config types.DomainConfig, - aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error { + aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error { // first lets build the domain config if err := ctx.CreateDomConfig(status.DomainName, config, status.DiskStatusList, aa, globalConfig, file); err != nil { @@ -877,7 +875,7 @@ func fallbackDomainMetric() map[string]types.DomainMetric { return dmList } -func (ctx xenContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error { +func (ctx xenContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error { return fmt.Errorf("not implemented") } diff --git a/pkg/pillar/types/domainmgrtypes.go b/pkg/pillar/types/domainmgrtypes.go index 3e7bd762cc..4937f0b3d5 100644 --- a/pkg/pillar/types/domainmgrtypes.go +++ b/pkg/pillar/types/domainmgrtypes.go @@ -261,6 +261,14 @@ type VmConfig struct { EnableVncShimVM bool } +// ExtraArgs contains the extra arguments for the domain +type ExtraArgs struct { + AgentName string + Ps *pubsub.PubSub + WarnTime time.Duration + ErrTime time.Duration +} + // VmMode is the type for the virtualization mode type VmMode uint8 @@ -276,10 +284,10 @@ const ( // Task represents any runnable entity on EVE type Task interface { Setup(DomainStatus, DomainConfig, *AssignableAdapters, - *ConfigItemValueMap, *os.File) error - VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error - VirtualTPMTerminate(domainName string) error - VirtualTPMTeardown(domainName string) error + *ConfigItemValueMap, *os.File, *ExtraArgs) error + VirtualTPMSetup(string, *ExtraArgs) error + VirtualTPMTerminate(string) error + VirtualTPMTeardown(string) error Create(string, string, *DomainConfig) (int, error) Start(string) error Stop(string, bool) error