Skip to content

Commit

Permalink
make it work
Browse files Browse the repository at this point in the history
Signed-off-by: Shahriyar Jalayeri <shahriyar@zededa.com>
  • Loading branch information
shjala committed Oct 4, 2024
1 parent d396edd commit b460fe6
Show file tree
Hide file tree
Showing 11 changed files with 146 additions and 53 deletions.
2 changes: 1 addition & 1 deletion pkg/dom0-ztools/efi/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ clean:
%.efi: %.so
objcopy -j .text -j .sdata -j .data -j .dynamic \
-j .dynsym -j .rel -j .rela -j .reloc \
--target=efi-app-$(ARCH) $^ $@
--target=efi-app-$(ARCH) $^ $@
2 changes: 1 addition & 1 deletion pkg/dom0-ztools/efi/efi_run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ cp /hostfs/etc/ovmf/eve_efi.img /run/kvm/eve_efi.img
-drive "file=/run/kvm/eve_efi.img,format=raw" \
-smbios "type=11,value=${2}" \
-net none \
-nographic
-nographic
4 changes: 2 additions & 2 deletions pkg/dom0-ztools/efi/main.c
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,7 @@ GetResolutionConfig(

_Return:
FreePool(ConfigValue);
return EFI_SUCCESS;
return Status;
}

EFI_STATUS
Expand Down Expand Up @@ -694,4 +694,4 @@ efi_main(
}

return EFI_SUCCESS;
}
}
2 changes: 1 addition & 1 deletion pkg/dom0-ztools/efi/startup.nsh
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
fs0:
efi_eveutil.efi
reset -s
reset -s
22 changes: 18 additions & 4 deletions pkg/pillar/cmd/domainmgr/domainmgr.go
Original file line number Diff line number Diff line change
Expand Up @@ -1119,14 +1119,21 @@ func maybeRetryBoot(ctx *domainContext, status *types.DomainStatus) {
}
defer file.Close()

if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil {
extra := types.ExtraArgs{
AgentName: agentName,
Ps: ctx.ps,
WarnTime: warningTime,
ErrTime: errorTime,
}

if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil {
log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err)
status.VirtualTPM = false
} else {
status.VirtualTPM = true
}

if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file); err != nil {
if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file, &extra); err != nil {
//it is retry, so omit error
log.Errorf("Failed to create DomainStatus from %+v: %s",
config, err)
Expand Down Expand Up @@ -1671,15 +1678,22 @@ func doActivate(ctx *domainContext, config types.DomainConfig,
}
defer file.Close()

if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil {
extra := types.ExtraArgs{
AgentName: agentName,
Ps: ctx.ps,
WarnTime: warningTime,
ErrTime: errorTime,
}

if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil {
log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err)
status.VirtualTPM = false
} else {
status.VirtualTPM = true
}

globalConfig := agentlog.GetGlobalConfig(log, ctx.subGlobalConfig)
if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file); err != nil {
if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file, &extra); err != nil {
log.Errorf("Failed to create DomainStatus from %+v: %s",
config, err)
status.SetErrorNow(err.Error())
Expand Down
5 changes: 2 additions & 3 deletions pkg/pillar/hypervisor/containerd.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"time"

"github.com/lf-edge/eve/pkg/pillar/containerd"
"github.com/lf-edge/eve/pkg/pillar/pubsub"
"github.com/lf-edge/eve/pkg/pillar/types"
"github.com/opencontainers/runtime-spec/specs-go"

Expand Down Expand Up @@ -101,7 +100,7 @@ func (ctx ctrdContext) setupSpec(status *types.DomainStatus, config *types.Domai
}

func (ctx ctrdContext) Setup(status types.DomainStatus, config types.DomainConfig,
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
if status.OCIConfigDir == "" {
return logError("failed to run domain %s: not based on an OCI image", status.DomainName)
}
Expand Down Expand Up @@ -327,7 +326,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
return res, nil
}

func (ctx ctrdContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
func (ctx ctrdContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
return fmt.Errorf("not implemented")
}

Expand Down
5 changes: 2 additions & 3 deletions pkg/pillar/hypervisor/kubevirt.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"time"

"github.com/lf-edge/eve/pkg/pillar/base"
"github.com/lf-edge/eve/pkg/pillar/pubsub"
"github.com/lf-edge/eve/pkg/pillar/types"

netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
Expand Down Expand Up @@ -185,7 +184,7 @@ func (ctx kubevirtContext) Task(status *types.DomainStatus) types.Task {

// Use eve DomainConfig and DomainStatus and generate k3s VMI config or a Pod config
func (ctx kubevirtContext) Setup(status types.DomainStatus, config types.DomainConfig,
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {

diskStatusList := status.DiskStatusList
domainName := status.DomainName
Expand Down Expand Up @@ -1362,7 +1361,7 @@ func (ctx kubevirtContext) PCISameController(id1 string, id2 string) bool {
return PCISameControllerGeneric(id1, id2)
}

func (ctx kubevirtContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
func (ctx kubevirtContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
return fmt.Errorf("not implemented")
}

Expand Down
129 changes: 103 additions & 26 deletions pkg/pillar/hypervisor/kvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
package hypervisor

import (
"bytes"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
Expand All @@ -17,7 +19,6 @@ import (
zconfig "github.com/lf-edge/eve-api/go/config"
"github.com/lf-edge/eve/pkg/pillar/agentlog"
"github.com/lf-edge/eve/pkg/pillar/containerd"
"github.com/lf-edge/eve/pkg/pillar/pubsub"
"github.com/lf-edge/eve/pkg/pillar/types"
"github.com/lf-edge/eve/pkg/pillar/utils"
fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
Expand Down Expand Up @@ -738,35 +739,18 @@ func getOVMFSettingsFilename(domainName string) (string, error) {
return types.OVMFSettingsDir + "/" + domainUUID.String() + "_OVMF_VARS.fd", nil
}

func prepareOVMFSettings(config types.DomainConfig, status types.DomainStatus, globalConfig *types.ConfigItemValueMap) error {
func prepareOVMFSettings(status types.DomainStatus) error {
// Create the OVMF settings directory if it does not exist
if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil {
return logError("failed to create OVMF settings directory: %v", err)
}
// Create a copy of the ovmf_vars.bin file in <domainName>_ovmf_vars.bin
// Get the path for a copy of the ovmf_vars.bin file form of <domainName>_OVMF_VARS.fd
ovmfSettingsFile, err := getOVMFSettingsFilename(status.DomainName)
if err != nil {
return logError("failed to get OVMF settings file: %v", err)
}
// Check if we need custom OVMF settings for the domain (the resolution)
fmlResolution := types.FmlResolutionUnset
if config.VirtualizationMode == types.FML {
// if we are not getting the resolution from the cloud-init, check the
// global config.
fmlResolution = status.FmlCustomResolution
if fmlResolution == types.FmlResolutionUnset {
if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil {
return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err)
}
}
}
// Find the necessary OVMF settings file
ovmfSettingsFileSrc := types.OVMFSettingsTemplate
if fmlResolution != types.FmlResolutionUnset {
ovmfSettingsFileSrc = types.CustomOVMFSettingsDir + "/OVMF_VARS_" + fmlResolution + ".fd"
}
if _, err := os.Stat(ovmfSettingsFile); os.IsNotExist(err) {
if err := fileutils.CopyFile(ovmfSettingsFileSrc, ovmfSettingsFile); err != nil {
if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfSettingsFile); err != nil {
return logError("failed to copy OVMF_VARS file: %v", err)
}
}
Expand All @@ -790,7 +774,7 @@ func cleanupOVMFSettings(domainName string) error {

// Setup sets up kvm
func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {

diskStatusList := status.DiskStatusList
domainName := status.DomainName
Expand All @@ -814,9 +798,29 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
// for ARM produces a single QEMU_EFI.fd file that contains both OVMF_VARS.fd
// and OVMF_CODE.fd.
if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" {
if err := prepareOVMFSettings(config, status, globalConfig); err != nil {
// Copy the OVMF_VARS file and the eve_efi.img file to the domain directory
err := prepareOVMFSettings(status)
if err != nil {
return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err)
}

fmlResolution := status.FmlCustomResolution
if fmlResolution == types.FmlResolutionUnset {
if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil {
return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err)
}
}
if fmlResolution != types.FmlResolutionUnset {
cfg := []string{
"eve.fml.resolution:" + fmlResolution,
}

logError("setOvmfVariables before call")
err := ctx.setOvmfVariables(cfg, status.DomainName, extra)
if err != nil {
return logError("failed to set OVMF variables for domain %s: %v", status.DomainName, err)
}
}
}

// first lets build the domain config
Expand Down Expand Up @@ -846,6 +850,14 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
"-readconfig", file.Name(),
"-pidfile", kvmStateDir+domainName+"/pid")

if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" {
args = append(args, "-drive", "file=/run/kvm/eve_efi.img,format=raw,if=none,id=drive-eve-efi")
args = append(args, "-device", "virtio-blk-pci,drive=drive-eve-efi")
args = append(args, "-smbios", "type=11,value=eve.try.all.boot.options:true")
}

logError("guest args: %v", args)

spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir)

if err != nil {
Expand Down Expand Up @@ -1330,9 +1342,9 @@ func getQmpListenerSocket(domainName string) string {
}

// VirtualTPMSetup launches a vTPM instance for the domain
func (ctx KvmContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
if ps != nil {
wk := utils.NewWatchdogKick(ps, agentName, warnTime, errTime)
func (ctx KvmContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
if extra.Ps != nil {
wk := utils.NewWatchdogKick(extra.Ps, extra.AgentName, extra.WarnTime, extra.ErrTime)
domainUUID, _, _, err := types.DomainnameToUUID(domainName)
if err != nil {
return fmt.Errorf("failed to extract UUID from domain name (vTPM setup): %v", err)
Expand Down Expand Up @@ -1429,3 +1441,68 @@ func requestVtpmTermination(id uuid.UUID) error {

return nil
}

// This is janky and hacky, but it is OK for testing
func (ctx KvmContext) setOvmfVariables(config []string, domainName string, extra *types.ExtraArgs) error {
cmd := exec.Command("/hostfs/usr/bin/ctr", "--namespace", "services.linuxkit", "t", "ls")
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return logError("setOvmfVariables error running ctr: %s\n", err)
}
output := out.String()
lines := strings.Split(output, "\n")
var xenToolsPID string
for _, line := range lines {
columns := strings.Fields(line)
if len(columns) > 2 && columns[0] == "xen-tools" {
xenToolsPID = columns[1]
break
}
}
if xenToolsPID != "" {
logError("setOvmfVariables The PID of xen-tools is: %s\n", xenToolsPID)
} else {
return logError("setOvmfVariables xen-tools not found.")
}

ovmfSettingsFile, err := getOVMFSettingsFilename(domainName)
if err != nil {
return logError("setOvmfVariables failed to get OVMF settings file: %v", err)
}

args := []string{"-F", "-a", "-t", xenToolsPID,
"/hostfs/etc/ovmf/efi_run.sh",
ovmfSettingsFile,
"eve.fml.resolution:1920x1080",
}

logError("setOvmfVariables args: nsenter %v", args)
cmd = exec.Command("nsenter", args...)
if err := cmd.Start(); err != nil {
return logError("setOvmfVariables error starting nsenter: %s\n", err)
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()

waitLoop:
for {
select {
case err := <-done:
if err != nil {
logError("setOvmfVariables Process exited with error: %s\n", err)
} else {
logError("setOvmfVariables Process completed successfully.")
}
break waitLoop
default:
extra.Ps.StillRunning(extra.AgentName, extra.WarnTime, extra.ErrTime)
logError("setOvmfVariables still waiting.")
time.Sleep(1 * time.Second)
}
}

return nil
}
6 changes: 2 additions & 4 deletions pkg/pillar/hypervisor/null.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@ package hypervisor
import (
"fmt"
"os"
"time"

"github.com/lf-edge/eve/pkg/pillar/pubsub"
"github.com/lf-edge/eve/pkg/pillar/types"

uuid "github.com/satori/go.uuid"
Expand Down Expand Up @@ -69,7 +67,7 @@ func (ctx nullContext) Task(status *types.DomainStatus) types.Task {
}

func (ctx nullContext) Setup(types.DomainStatus, types.DomainConfig,
*types.AssignableAdapters, *types.ConfigItemValueMap, *os.File) error {
*types.AssignableAdapters, *types.ConfigItemValueMap, *os.File, *types.ExtraArgs) error {
return nil
}

Expand Down Expand Up @@ -174,7 +172,7 @@ func (ctx nullContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
return nil, nil
}

func (ctx nullContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
func (ctx nullContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
return fmt.Errorf("not implemented")
}

Expand Down
6 changes: 2 additions & 4 deletions pkg/pillar/hypervisor/xen.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ import (
"runtime"
"strconv"
"strings"
"time"

"github.com/lf-edge/eve/pkg/pillar/pubsub"
"github.com/lf-edge/eve/pkg/pillar/types"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/mem"
Expand Down Expand Up @@ -121,7 +119,7 @@ func (ctx xenContext) Task(status *types.DomainStatus) types.Task {
}

func (ctx xenContext) Setup(status types.DomainStatus, config types.DomainConfig,
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
// first lets build the domain config
if err := ctx.CreateDomConfig(status.DomainName, config,
status.DiskStatusList, aa, globalConfig, file); err != nil {
Expand Down Expand Up @@ -877,7 +875,7 @@ func fallbackDomainMetric() map[string]types.DomainMetric {
return dmList
}

func (ctx xenContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
func (ctx xenContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
return fmt.Errorf("not implemented")
}

Expand Down
Loading

0 comments on commit b460fe6

Please sign in to comment.