Add bootable test for enki generated isos

Signed-off-by: Itxaka <itxaka@kairos.io>

.github/workflows/bootable.yaml

@@ -0,0 +1,27 @@
+---
+name: 'Enki Bootable tests'
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: enki-bootable-${{ github.ref || github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  bootable:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: earthly/actions-setup@v1.0.13
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build
+        run: earthly +test-bootable

Dockerfile

@@ -1,5 +1,5 @@
 ARG LUET_VERSION=0.35.2
-ARG GO_VERSION=1.22-alpine
+ARG GO_VERSION=1.22.5-alpine
 
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 FROM golang:$GO_VERSION AS builder

Earthfile

@@ -19,7 +19,7 @@ go-deps:
 
 version:
     FROM +go-deps
-    COPY . ./
+    COPY .git ./
     RUN --no-cache echo $(git describe --always --tags --dirty) > VERSION
     RUN --no-cache echo $(git describe --always --dirty) > COMMIT
     ARG VERSION=$(cat VERSION)
@@ -32,11 +32,10 @@ test:
     WORKDIR /build
     COPY . .
     ARG TEST_PATHS=./...
-    ARG LABEL_FILTER=
     ENV CGO_ENABLED=1
     # Some test require the docker sock exposed
     WITH DOCKER --load enki-image=(+enki-image)
-        RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "$LABEL_FILTER" -v --fail-fast --race --covermode=atomic --coverprofile=coverage.out --coverpkg=github.com/kairos-io/enki/... -p -r $TEST_PATHS
+        RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "build-uki && genkey" -v --fail-fast --race --covermode=atomic --coverprofile=coverage.out --coverpkg=github.com/kairos-io/enki/... -p -r $TEST_PATHS
     END
     SAVE ARTIFACT coverage.out AS LOCAL coverage.out
 
@@ -52,3 +51,30 @@ build:
     ENV CGO_ENABLED=0
     RUN go build -o enki -ldflags "${LDFLAGS}" main.go
     SAVE ARTIFACT enki enki AS LOCAL build/enki
+
+build-iso:
+    FROM +enki-image
+    ARG BASE_IMAGE=quay.io/kairos/ubuntu:24.04-core-amd64-generic-v3.1.1-uki
+    WORKDIR /build
+    RUN /enki genkey -e 7 --output /keys CIKEYS
+    # Extend the default cmdline to write everything to serial first :D
+    RUN /enki build-uki $BASE_IMAGE --output-dir /build/ -k /keys --output-type iso -x "console=ttyS0"
+    SAVE ARTIFACT /build/*.iso enki.iso AS LOCAL build/enki.iso
+
+
+test-bootable:
+    FROM +go-deps
+    WORKDIR /build
+    RUN . /etc/os-release && echo "deb http://deb.debian.org/debian $VERSION_CODENAME-backports main contrib non-free" > /etc/apt/sources.list.d/backports.list
+    RUN apt update
+    RUN apt install -y qemu-system-x86 qemu-utils git swtpm && apt clean
+    COPY . .
+    COPY +build-iso/enki.iso enki.iso
+    ARG ISO=/build/enki.iso
+    ARG FIRMWARE=/usr/share/OVMF/OVMF_CODE.fd
+    ARG USE_QEMU=true
+    ARG MEMORY=4000
+    ARG CPUS=2
+    ARG CREATE_VM=true
+    RUN date
+    RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "bootable" -v --fail-fast -r ./e2e

e2e/bootable_test.go

@@ -0,0 +1,306 @@
+package e2e_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+
+	"github.com/google/uuid"
+	process "github.com/mudler/go-processmanager"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/spectrocloud/peg/matcher"
+	"github.com/spectrocloud/peg/pkg/machine"
+	"github.com/spectrocloud/peg/pkg/machine/types"
+)
+
+var _ = Describe("bootable artifacts", Label("bootable"), func() {
+	var vm VM
+	var err error
+
+	BeforeEach(func() {
+		_, ok := os.Stat(os.Getenv("ISO"))
+		Expect(ok).To(BeNil(), "ISO should exist")
+		vm, err = startVM()
+		Expect(err).ToNot(HaveOccurred())
+		vm.EventuallyConnects(1200)
+	})
+	AfterEach(func() {
+		if CurrentSpecReport().Failed() {
+			gatherLogs(vm)
+			serial, _ := os.ReadFile(filepath.Join(vm.StateDir, "serial.log"))
+			_ = os.MkdirAll("logs", os.ModePerm|os.ModeDir)
+			_ = os.WriteFile(filepath.Join("logs", "serial.log"), serial, os.ModePerm)
+			fmt.Println(string(serial))
+		}
+
+		err := vm.Destroy(nil)
+		Expect(err).ToNot(HaveOccurred())
+	})
+	It("Should boot as expected", func() {
+		By("Have secureboot enabled", func() {
+			output, err := vm.Sudo("dmesg | grep -i secure")
+			Expect(err).ToNot(HaveOccurred(), output)
+			Expect(output).To(ContainSubstring("Secure boot enabled"))
+		})
+
+		By("Have our custom keys", func() {
+			output, err := vm.Sudo("kairos-agent state get \"kairos.eficerts|tojson\"")
+			Expect(err).ToNot(HaveOccurred(), output)
+			// Check the test keys we created for this
+			Expect(output).To(ContainSubstring("CIKEYS-db"))
+			Expect(output).To(ContainSubstring("CIKEYS-KEK"))
+			Expect(output).To(ContainSubstring("CIKEYS-PK"))
+		})
+	})
+})
+
+func emulateTPM(stateDir string) {
+	t := path.Join(stateDir, "tpm")
+	err := os.MkdirAll(t, os.ModePerm)
+	Expect(err).ToNot(HaveOccurred())
+
+	cmd := exec.Command("swtpm",
+		"socket",
+		"--tpmstate", fmt.Sprintf("dir=%s", t),
+		"--ctrl", fmt.Sprintf("type=unixio,path=%s/swtpm-sock", t),
+		"--tpm2", "--log", "level=20")
+	err = cmd.Start()
+	Expect(err).ToNot(HaveOccurred())
+
+	err = os.WriteFile(path.Join(t, "pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0744)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func startVM() (VM, error) {
+	stateDir, err := os.MkdirTemp("", "")
+	Expect(err).ToNot(HaveOccurred())
+	fmt.Printf("State dir: %s\n", stateDir)
+
+	opts := defaultVMOpts(stateDir)
+
+	m, err := machine.New(opts...)
+	Expect(err).ToNot(HaveOccurred())
+
+	vm := NewVM(m, stateDir)
+	_, err = vm.Start(context.Background())
+	return vm, err
+}
+
+func defaultVMOpts(stateDir string) []types.MachineOption {
+	opts := defaultVMOptsNoDrives(stateDir)
+
+	driveSize := os.Getenv("DRIVE_SIZE")
+	if driveSize == "" {
+		driveSize = "25000"
+	}
+
+	opts = append(opts, types.WithDriveSize(driveSize))
+
+	return opts
+}
+
+func defaultVMOptsNoDrives(stateDir string) []types.MachineOption {
+	var err error
+
+	if os.Getenv("ISO") == "" && os.Getenv("CREATE_VM") == "true" {
+		fmt.Println("ISO missing")
+		os.Exit(1)
+	}
+
+	var sshPort, spicePort int
+
+	vmName := uuid.New().String()
+
+	emulateTPM(stateDir)
+
+	sshPort, err = getFreePort()
+	Expect(err).ToNot(HaveOccurred())
+	fmt.Printf("Using ssh port: %d\n", sshPort)
+
+	memory := os.Getenv("MEMORY")
+	if memory == "" {
+		memory = "2096"
+	}
+	cpus := os.Getenv("CPUS")
+	if cpus == "" {
+		cpus = "2"
+	}
+
+	opts := []types.MachineOption{
+		types.QEMUEngine,
+		types.WithISO(os.Getenv("ISO")),
+		types.WithMemory(memory),
+		types.WithCPU(cpus),
+		types.WithSSHPort(strconv.Itoa(sshPort)),
+		types.WithID(vmName),
+		types.WithSSHUser("kairos"),
+		types.WithSSHPass("kairos"),
+		types.OnFailure(func(p *process.Process) {
+			var serial string
+
+			out, _ := os.ReadFile(p.StdoutPath())
+			err, _ := os.ReadFile(p.StderrPath())
+			status, _ := p.ExitCode()
+
+			if serialBytes, err := os.ReadFile(path.Join(p.StateDir(), "serial.log")); err != nil {
+				serial = fmt.Sprintf("Error reading serial log file: %s\n", err)
+			} else {
+				serial = string(serialBytes)
+			}
+
+			// We are explicitly killing the qemu process. We don't treat that as an error,
+			// but we just print the output just in case.
+			fmt.Printf("\nVM Aborted.\nstdout: %s\nstderr: %s\nserial: %s\nExit status: %s\n", out, err, serial, status)
+			Fail(fmt.Sprintf("\nVM Aborted.\nstdout: %s\nstderr: %s\nserial: %s\nExit status: %s\n",
+				out, err, serial, status))
+		}),
+		types.WithStateDir(stateDir),
+		// Serial output to file: https://superuser.com/a/1412150
+		func(m *types.MachineConfig) error {
+			m.Args = append(m.Args,
+				"-chardev", fmt.Sprintf("stdio,mux=on,id=char0,logfile=%s,signal=off", path.Join(stateDir, "serial.log")),
+				"-serial", "chardev:char0",
+				"-mon", "chardev=char0",
+			)
+			m.Args = append(m.Args,
+				"-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s/swtpm-sock", path.Join(stateDir, "tpm")),
+				"-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0",
+			)
+			return nil
+		},
+		// Firmware
+		func(m *types.MachineConfig) error {
+			FW := os.Getenv("FIRMWARE")
+			if FW != "" {
+				getwd, err := os.Getwd()
+				if err != nil {
+					return err
+				}
+				m.Args = append(m.Args, "-drive",
+					fmt.Sprintf("file=%s,if=pflash,format=raw,readonly=on", FW),
+				)
+
+				// Copy the empty efivars to not modify it
+				f, err := os.ReadFile(filepath.Join(getwd, "assets/efivars.empty.fd"))
+				if err != nil {
+					return err
+				}
+				err = os.WriteFile(filepath.Join(stateDir, "efivars.empty.fd"), f, os.ModePerm)
+				if err != nil {
+					return err
+				}
+
+				m.Args = append(m.Args, "-drive",
+					fmt.Sprintf("file=%s,if=pflash,format=raw", filepath.Join(stateDir, "efivars.empty.fd")),
+				)
+
+				// Needed to be set for secureboot!
+				m.Args = append(m.Args, "-machine", "q35,smm=on")
+			}
+
+			return nil
+		},
+		types.WithDataSource(os.Getenv("DATASOURCE")),
+	}
+	if os.Getenv("KVM") != "" {
+		opts = append(opts, func(m *types.MachineConfig) error {
+			m.Args = append(m.Args,
+				"-enable-kvm",
+			)
+			return nil
+		})
+	}
+
+	if os.Getenv("USE_QEMU") == "true" {
+		opts = append(opts, types.QEMUEngine)
+
+		// You can connect to it with "spicy" or other tool.
+		// DISPLAY is already taken on Linux X sessions
+		if os.Getenv("MACHINE_SPICY") != "" {
+			spicePort, _ = getFreePort()
+			for spicePort == sshPort { // avoid collision
+				spicePort, _ = getFreePort()
+			}
+			display := fmt.Sprintf("-spice port=%d,addr=127.0.0.1,disable-ticketing=yes", spicePort)
+			opts = append(opts, types.WithDisplay(display))
+
+			cmd := exec.Command("spicy",
+				"-h", "127.0.0.1",
+				"-p", strconv.Itoa(spicePort))
+			err = cmd.Start()
+			Expect(err).ToNot(HaveOccurred())
+		}
+	} else {
+		opts = append(opts, types.VBoxEngine)
+	}
+
+	return opts
+}
+
+func getFreePort() (port int, err error) {
+	var a *net.TCPAddr
+	if a, err = net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
+		var l *net.TCPListener
+		if l, err = net.ListenTCP("tcp", a); err == nil {
+			defer l.Close()
+			return l.Addr().(*net.TCPAddr).Port, nil
+		}
+	}
+	return
+}
+
+func gatherLogs(vm VM) {
+	vm.Scp("assets/kubernetes_logs.sh", "/tmp/logs.sh", "0770")
+	vm.Sudo("sh /tmp/logs.sh > /run/kube_logs")
+	vm.Sudo("cat /oem/* > /run/oem.yaml")
+	vm.Sudo("cat /etc/resolv.conf > /run/resolv.conf")
+	vm.Sudo("k3s kubectl get pods -A -o json > /run/pods.json")
+	vm.Sudo("k3s kubectl get events -A -o json > /run/events.json")
+	vm.Sudo("cat /proc/cmdline > /run/cmdline")
+	vm.Sudo("chmod 777 /run/events.json")
+
+	vm.Sudo("df -h > /run/disk")
+	vm.Sudo("mount > /run/mounts")
+	vm.Sudo("blkid > /run/blkid")
+	vm.Sudo("dmesg > /run/dmesg.log")
+
+	// zip all files under /var/log/kairos
+	vm.Sudo("tar -czf /run/kairos-agent-logs.tar.gz /var/log/kairos")
+
+	vm.GatherAllLogs(
+		[]string{
+			"edgevpn@kairos",
+			"kairos-agent",
+			"cos-setup-boot",
+			"cos-setup-network",
+			"cos-setup-reconcile",
+			"kairos",
+			"k3s",
+			"k3s-agent",
+		},
+		[]string{
+			"/var/log/edgevpn.log",
+			"/var/log/kairos/agent.log",
+			"/run/pods.json",
+			"/run/disk",
+			"/run/mounts",
+			"/run/blkid",
+			"/run/events.json",
+			"/run/kube_logs",
+			"/run/cmdline",
+			"/run/oem.yaml",
+			"/run/resolv.conf",
+			"/run/dmesg.log",
+			"/run/immucore/immucore.log",
+			"/run/immucore/initramfs_stage.log",
+			"/run/immucore/rootfs_stage.log",
+			"/tmp/ovmf_debug.log",
+			"/run/kairos-agent-logs.tar.gz",
+		})
+}