Improved upgrade test

.github/workflows/build.yaml

@@ -50,17 +50,12 @@ jobs:
     env:
       PLATFORM: ${{ needs.detect.outputs.platform }}
       TOOLKIT_REPO: ghcr.io/${{github.repository}}/elemental-cli 
-    outputs:
-      version: ${{ steps.set-version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: "${{ github.event.pull_request.merge_commit_sha }}"
       - run: |
           git fetch --prune --unshallow
-      - id: set-version
-        run: |
-          echo "version="$(git describe --tags --candidates=50) >> $GITHUB_OUTPUT
       - name: Log in to ghcr.io
         uses: docker/login-action@v3
         with:
@@ -82,20 +77,3 @@ jobs:
     uses: ./.github/workflows/build_and_test_x86.yaml
     with:
       flavor: ${{ matrix.flavor }}
-
-  # TODO: use integer package-version-id instead of tag.
-  # cleanup:
-  #   runs-on: ubuntu-latest
-  #   permissions:
-  #     packages: write
-  #   needs:
-  #     - build-toolkit
-  #     - build-matrix
-  #   steps:
-  #     - uses: actions/delete-package-versions@v4
-  #       if: ${{ !startsWith(github.event.ref, 'refs/tags/v') }}
-  #       with:
-  #         package-version-ids: ${{ needs.build-toolkit.outputs.version }}
-  #         owner: ${{ github.owner }}
-  #         package-name: elemental-toolkit/elemental-cli
-  #         package-type: container

.github/workflows/build_and_test_x86.yaml

@@ -136,16 +136,31 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       tests: ${{ steps.detect.outputs.tests }}
+      upgrade: ${{ steps.image.outputs.upgrade }}
+      toolkit: ${{ steps.image.outputs.toolkit }}
     steps:
       - id: detect
         env:
           FLAVOR: ${{ inputs.flavor }}
         run: |
           if [ "${FLAVOR}" == green ]; then
-            echo "tests=['test-upgrade', 'test-recovery', 'test-fallback', 'test-fsck', 'test-grubfallback']" >> $GITHUB_OUTPUT
+            echo "tests=['test-upgrade', 'test-downgrade', 'test-recovery', 'test-fallback', 'test-fsck', 'test-grubfallback']" >> $GITHUB_OUTPUT
           else
             echo "tests=['test-active']" >> $GITHUB_OUTPUT
           fi
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ github.event.pull_request.merge_commit_sha }}"
+      - id: image
+        env:
+          FLAVOR: ${{ inputs.flavor }}
+        run: |
+          git fetch --prune --unshallow
+          export TAG=$(git describe --tags --candidates=50 --abbrev=0)
+          export COMMIT=$(git rev-parse --short HEAD)
+          export VERSION=${TAG}-g${COMMIT}
+          echo "upgrade=ghcr.io/rancher/elemental-toolkit/elemental-${FLAVOR}:${VERSION}" >> $GITHUB_OUTPUT
+          echo "toolkit=ghcr.io/rancher/elemental-toolkit/elemental-cli:${VERSION}" >> $GITHUB_OUTPUT
 
   tests-matrix:
     needs:
@@ -156,6 +171,7 @@ jobs:
       FLAVOR: ${{ inputs.flavor }}
       ARCH: x86_64
       COS_TIMEOUT: 1600
+      UPGRADE_ARGS: --toolkit-image=${{ needs.detect.outputs.toolkit }} --upgrade-image=${{ needs.detect.outputs.upgrade }}
     strategy:
       matrix:
         test: ${{ fromJson(needs.detect.outputs.tests) }}
@@ -173,7 +189,12 @@ jobs:
           make test-deps
       - run: |
           git fetch --prune --unshallow
-      - name: Cached Disk
+      - if: ${{ matrix.test == 'test-upgrade' }}
+        name: Download specific disk
+        run: |
+          wget -q --tries=3 https://github.com/rancher/elemental-toolkit/releases/download/v1.1.4/elemental-${{ inputs.flavor }}-v1.1.4.${{ env.ARCH }}.qcow2 -O /tmp/elemental-${{ inputs.flavor }}.${{ env.ARCH }}.qcow2
+      - if: ${{ matrix.test != 'test-upgrade' }}
+        name: Cached Disk
         id: cache-disk
         uses: actions/cache/restore@v4
         env:

make/Makefile.test

@@ -1,6 +1,6 @@
-GINKGO?="github.com/onsi/ginkgo/v2/ginkgo"
-
-GINKGO_ARGS?=-v --fail-fast -r --timeout=3h
+GINKGO       ?= "github.com/onsi/ginkgo/v2/ginkgo"
+GINKGO_ARGS  ?= -v --fail-fast -r --timeout=3h
+UPGRADE_ARGS ?= 
 
 .PHONY: prepare-test
 prepare-test:
@@ -57,9 +57,13 @@ test-fallback: test-active
 test-fsck: test-active
 	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/fsck
 
+.PHONY: test-downgrade
+test-downgrade: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/downgrade
+
 .PHONY: test-upgrade
 test-upgrade: test-active
-	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/upgrade
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/upgrade -- $(UPGRADE_ARGS)
 
 .PHONY: test-deps
 test-deps:

tests/assets/chroot_hooks.yaml

@@ -1,8 +1,8 @@
-name: "Setup cos config"
+name: "Debug hooks"
 stages:
    after-upgrade-chroot:
      - commands:
         - echo > /after-upgrade-chroot
    after-reset-chroot:
      - commands:
-        - echo > /after-reset-chroot
+        - echo > /after-reset-chroot

tests/common/common.go

@@ -18,14 +18,21 @@ package common
 
 import "flag"
 
-const upImg = "ghcr.io/rancher/elemental-toolkit/elemental-green:v0.10.7-g3e4a3c56"
+const defaultUpgradeImage = "ghcr.io/rancher/elemental-toolkit/elemental-green:v1.1.4"
+const defaultToolkitImage = "ghcr.io/rancher/elemental-toolkit/elemental-cli:v1.1.4"
 
-var upgradeImg string
+var upgradeImage string
+var toolkitImage string
 
 func init() {
-	flag.StringVar(&upgradeImg, "upgradeImg", upImg, "Default image to use in `upgrade` calls")
+	flag.StringVar(&upgradeImage, "upgrade-image", defaultUpgradeImage, "Default image to use in `upgrade` calls")
+	flag.StringVar(&toolkitImage, "toolkit-image", defaultToolkitImage, "Default image to use when calling `upgrade`")
 }
 
 func UpgradeImage() string {
-	return upgradeImg
+	return upgradeImage
+}
+
+func ToolkitImage() string {
+	return toolkitImage
 }

tests/downgrade/downgrade_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright © 2022 - 2024 SUSE LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package elemental_test
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	sut "github.com/rancher/elemental-toolkit/tests/vm"
+
+	comm "github.com/rancher/elemental-toolkit/tests/common"
+)
+
+func TestTests(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Elemental upgrade test Suite")
+}
+
+var _ = Describe("Elemental Feature tests", func() {
+	var s *sut.SUT
+	BeforeEach(func() {
+		s = sut.NewSUT()
+		s.EventuallyConnects()
+		s.EventuallyBootedFrom(sut.Active)
+	})
+
+	Context("After install", func() {
+		It("downgrades to a signed image including upgrade and reset hooks", func() {
+			By("setting /oem/chroot_hooks.yaml")
+			err := s.SendFile("../assets/chroot_hooks.yaml", "/oem/chroot_hooks.yaml", "0770")
+			Expect(err).ToNot(HaveOccurred())
+			originalVersion := s.GetOSRelease("TIMESTAMP")
+
+			By(fmt.Sprintf("and upgrading the to %s", comm.UpgradeImage()))
+
+			out, err := s.Command(s.ElementalCmd("upgrade", "--system", comm.UpgradeImage()))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).Should(ContainSubstring("Upgrade completed"))
+
+			s.Reboot()
+			s.EventuallyBootedFrom(sut.Active)
+			currentVersion := s.GetOSRelease("TIMESTAMP")
+			Expect(currentVersion).NotTo(Equal(originalVersion))
+
+			_, err = s.Command("cat /after-upgrade-chroot")
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = s.Command("cat /after-reset-chroot")
+			Expect(err).To(HaveOccurred())
+
+			s.Reset()
+			currentVersion = s.GetOSRelease("TIMESTAMP")
+			Expect(currentVersion).To(Equal(originalVersion))
+			_, err = s.Command("cat /after-reset-chroot")
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

tests/upgrade/upgrade_test.go

@@ -42,9 +42,14 @@ var _ = Describe("Elemental Feature tests", func() {
 			Expect(err).ToNot(HaveOccurred())
 			originalVersion := s.GetOSRelease("TIMESTAMP")
 
-			By(fmt.Sprintf("and upgrading the to %s", comm.UpgradeImage()))
+			By(fmt.Sprintf("and upgrading to %s", comm.UpgradeImage()))
+
+			upgradeCmd := s.ElementalCmd("upgrade", "--bootloader", "--system", comm.UpgradeImage())
+			out, err := s.NewPodmanRunCommand(comm.ToolkitImage(), fmt.Sprintf("-c \"mount --rbind /host/run /run && %s\"", upgradeCmd)).
+				Privileged().
+				WithMount("/", "/host").
+				Run()
 
-			out, err := s.Command(s.ElementalCmd("upgrade", "--system", comm.UpgradeImage()))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(out).Should(ContainSubstring("Upgrade completed"))
 
@@ -58,12 +63,6 @@ var _ = Describe("Elemental Feature tests", func() {
 
 			_, err = s.Command("cat /after-reset-chroot")
 			Expect(err).To(HaveOccurred())
-
-			s.Reset()
-			currentVersion = s.GetOSRelease("TIMESTAMP")
-			Expect(currentVersion).To(Equal(originalVersion))
-			_, err = s.Command("cat /after-reset-chroot")
-			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 })

tests/vm/podman.go

@@ -0,0 +1,67 @@
+/*
+Copyright © 2022 - 2024 SUSE LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vm
+
+import (
+	"fmt"
+	"strings"
+)
+
+type PodmanRunCommand struct {
+	sut        *SUT
+	privileged bool
+	image      string
+	mounts     []VolumeMount
+	entrypoint string
+	command    string
+}
+
+type VolumeMount struct {
+	from string
+	to   string
+}
+
+func (p *PodmanRunCommand) Privileged() *PodmanRunCommand {
+	p.privileged = true
+	return p
+}
+
+func (p *PodmanRunCommand) WithMount(from, to string) *PodmanRunCommand {
+	p.mounts = append(p.mounts, VolumeMount{from: from, to: to})
+	return p
+}
+
+func (p *PodmanRunCommand) Run() (string, error) {
+	priv := ""
+	if p.privileged {
+		priv = "--privileged"
+	}
+
+	mounts := []string{}
+	for _, m := range p.mounts {
+		mounts = append(mounts, fmt.Sprintf("-v %s:%s", m.from, m.to))
+	}
+
+	entrypoint := ""
+	if p.entrypoint != "" {
+		entrypoint = fmt.Sprintf("--entrypoint=%s", p.entrypoint)
+	}
+
+	cmd := fmt.Sprintf("podman run %s %s %s %s %s", priv, strings.Join(mounts, " "), entrypoint, p.image, p.command)
+	fmt.Printf("Running podman command: '%s'", cmd)
+	return p.sut.Command(cmd)
+}

tests/vm/sut.go

@@ -279,6 +279,21 @@ func (s *SUT) IsVMRunning() bool {
 	return proc.Signal(syscall.Signal(0)) == nil
 }
 
+func (s *SUT) NewPodmanRunCommand(image, command string) *PodmanRunCommand {
+	return &PodmanRunCommand{
+		sut:        s,
+		image:      image,
+		entrypoint: "/bin/bash",
+		command:    command,
+		mounts: []VolumeMount{
+			{
+				from: "/tmp",
+				to:   "/tmp",
+			},
+		},
+	}
+}
+
 // Command sends a command to the SUIT and waits for reply
 func (s *SUT) Command(cmd string) (string, error) {
 	if !s.IsVMRunning() {