diff --git a/.github/run_test.sh b/.github/run_test.sh new file mode 100755 index 000000000..9664a89ba --- /dev/null +++ b/.github/run_test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +go get github.com/onsi/ginkgo/v2 +go get github.com/onsi/gomega/... +go get github.com/onsi/ginkgo/v2/ginkgo/internal@v2.1.3 +go install github.com/onsi/ginkgo/v2/ginkgo +ginkgo --label-filter "$1" --fail-fast -r ./tests/ \ No newline at end of file diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 7d16658a7..c0d4f2975 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -46,10 +46,14 @@ jobs: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} run: | docker push quay.io/c3os/c3os:${{ matrix.flavor }}-latest - + - name: Push to testing + run: | + docker tag quay.io/c3os/c3os:${{ matrix.flavor }}-latest ttl.sh/c3os-${{ matrix.flavor }}-${{ github.sha }}:8h + docker push ttl.sh/c3os-${{ matrix.flavor }}-${{ github.sha }}:8h # Test start build-vm-images: - needs: build + needs: + - build runs-on: macos-10.15 strategy: fail-fast: false @@ -115,8 +119,11 @@ jobs: config.yaml if-no-files-found: error - single-test: - needs: prepare-test + upgrade-single-node-k8s-test: + needs: + - prepare-test + - upgrade-test + - upgrade-latest-with-cli-test runs-on: macos-10.15 strategy: fail-fast: true @@ -149,12 +156,11 @@ jobs: ls -liah export ISO=$PWD/$(ls *.iso) export GOPATH="/Users/runner/go" - PATH=$PATH:$GOPATH/bin - go get github.com/onsi/ginkgo/v2 - go get github.com/onsi/gomega/... - go get github.com/onsi/ginkgo/v2/ginkgo/internal@v2.1.3 - go install github.com/onsi/ginkgo/v2/ginkgo - CLOUD_INIT=$PWD/config.yaml CREATE_VM=true FLAVOR=${{ matrix.flavor }} ginkgo --label-filter upgrade --fail-fast -r ./tests/ + export PATH=$PATH:$GOPATH/bin + export CLOUD_INIT=$PWD/config.yaml + export CREATE_VM=true + export FLAVOR=${{ matrix.flavor }} + ./.github/run_test.sh "upgrade-k8s" - uses: actions/upload-artifact@v2 if: failure() with: @@ -162,8 +168,12 @@ jobs: path: tests/**/logs/* if-no-files-found: warn - test: - needs: prepare-test + decentralized-k8s-test-opensuse: + needs: + - prepare-test + - upgrade-single-node-k8s-test + - upgrade-test + - upgrade-latest-with-cli-test runs-on: macos-10.15 strategy: fail-fast: true @@ -196,12 +206,11 @@ jobs: ls -liah export ISO=$PWD/$(ls *.iso) export GOPATH="/Users/runner/go" - PATH=$PATH:$GOPATH/bin - go get github.com/onsi/ginkgo/v2 - go get github.com/onsi/gomega/... - go get github.com/onsi/ginkgo/v2/ginkgo/internal@v2.1.3 - go install github.com/onsi/ginkgo/v2/ginkgo - CLOUD_INIT=$PWD/config.yaml CREATE_VM=true FLAVOR=${{ matrix.flavor }} ginkgo --label-filter smoke --fail-fast -r ./tests/ + export PATH=$PATH:$GOPATH/bin + export CLOUD_INIT=$PWD/config.yaml + export CREATE_VM=true + export FLAVOR=${{ matrix.flavor }} + ./.github/run_test.sh "decentralized-k8s" - uses: actions/upload-artifact@v2 if: failure() with: @@ -239,10 +248,12 @@ jobs: config.yaml if-no-files-found: error - test-alpine: + decentralized-k8s-test-alpine: needs: - prepare-alpine-test - - test + - decentralized-k8s-test-opensuse + - upgrade-test + - upgrade-latest-with-cli-test runs-on: macos-10.15 strategy: fail-fast: true @@ -275,12 +286,11 @@ jobs: ls -liah export ISO=$PWD/$(ls *.iso) export GOPATH="/Users/runner/go" - PATH=$PATH:$GOPATH/bin - go get github.com/onsi/ginkgo/v2 - go get github.com/onsi/gomega/... - go get github.com/onsi/ginkgo/v2/ginkgo/internal@v2.1.3 - go install github.com/onsi/ginkgo/v2/ginkgo - CLOUD_INIT=$PWD/config.yaml CREATE_VM=true FLAVOR=${{ matrix.flavor }} ginkgo --label-filter smoke --fail-fast -r ./tests/ + export PATH=$PATH:$GOPATH/bin + export CLOUD_INIT=$PWD/config.yaml + export CREATE_VM=true + export FLAVOR=${{ matrix.flavor }} + ./.github/run_test.sh "decentralized-k8s" - uses: actions/upload-artifact@v2 if: failure() with: @@ -288,7 +298,114 @@ jobs: path: tests/**/logs/* if-no-files-found: warn + latest-release: + runs-on: ubuntu-latest + steps: + - uses: robinraju/release-downloader@v1.3 + with: + # A flag to set the download target as latest release + # The default value is 'false' + latest: true + repository: "c3os-io/c3os" + fileName: "*" + out-file-path: "last-release" + - uses: actions/upload-artifact@v2 + with: + name: latest-release.zip + path: last-release + if-no-files-found: error + + upgrade-latest-with-cli-test: + needs: + - build + - latest-release + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + include: + - flavor: "alpine" + node: "A" # Arbitrary field + - flavor: "opensuse" + node: "B" +# - flavor: "alpine" +# node: "C" + steps: + - uses: actions/checkout@v2 + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: latest-release.zip + - name: Install deps + run: | + brew install cdrtools jq + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + - run: | + ls -liah + export ISO=$PWD/$(ls c3os-${{ matrix.flavor }}-*.iso) + export GOPATH="/Users/runner/go" + export CONTAINER_IMAGE=ttl.sh/c3os-${{ matrix.flavor }}-${{ github.sha }}:8h + export PATH=$PATH:$GOPATH/bin + export CLOUD_INIT=$PWD/tests/assets/config.yaml + export CREATE_VM=true + export FLAVOR=${{ matrix.flavor }} + ./.github/run_test.sh "upgrade-latest-with-cli" + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: ${{ matrix.flavor }}-upgrade-test.logs.zip + path: tests/**/logs/* + if-no-files-found: warn + + + upgrade-test: + needs: + - build + - latest-release + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + include: + - flavor: "alpine" + node: "A" # Arbitrary field + - flavor: "opensuse" + node: "B" +# - flavor: "alpine" +# node: "C" + steps: + - uses: actions/checkout@v2 + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: latest-release.zip + - name: Install deps + run: | + brew install cdrtools jq + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: '^1.16' + - run: | + ls -liah + export ISO=$PWD/$(ls c3os-${{ matrix.flavor }}-*.iso) + export GOPATH="/Users/runner/go" + export CONTAINER_IMAGE=ttl.sh/c3os-${{ matrix.flavor }}-${{ github.sha }}:8h + export PATH=$PATH:$GOPATH/bin + export CLOUD_INIT=$PWD/tests/assets/config.yaml + export CREATE_VM=true + export FLAVOR=${{ matrix.flavor }} + ./.github/run_test.sh "upgrade-latest-with-kubernetes" + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: ${{ matrix.flavor }}-upgrade-test.logs.zip + path: tests/**/logs/* + if-no-files-found: warn # test-tumbleweed: # needs: # - prepare-test diff --git a/tests/smoke_test.go b/tests/decentralized_test.go similarity index 87% rename from tests/smoke_test.go rename to tests/decentralized_test.go index cc3f741e9..dd23da80b 100644 --- a/tests/smoke_test.go +++ b/tests/decentralized_test.go @@ -12,37 +12,14 @@ import ( "github.com/onsi/gomega/types" ) -var _ = Describe("c3os smoke", Label("smoke"), func() { +var _ = Describe("c3os decentralized k8s test", Label("decentralized-k8s"), func() { BeforeEach(func() { machine.EventuallyConnects() }) AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - machine.SSHCommand("sudo k3s kubectl get pods -A -o json > /run/pods.json") - machine.SSHCommand("sudo k3s kubectl get events -A -o json > /run/events.json") - machine.SSHCommand("sudo df -h > /run/disk") - machine.SSHCommand("sudo mount > /run/mounts") - machine.SSHCommand("sudo blkid > /run/blkid") - - machine.GatherAllLogs( - []string{ - "edgevpn@c3os", - "c3os-agent", - "cos-setup-boot", - "cos-setup-network", - "c3os", - "k3s", - }, - []string{ - "/var/log/edgevpn.log", - "/var/log/c3os-agent.log", - "/run/pods.json", - "/run/disk", - "/run/mounts", - "/run/blkid", - "/run/events.json", - }) + gatherLogs() } }) @@ -125,7 +102,7 @@ var _ = Describe("c3os smoke", Label("smoke"), func() { It("has default image sizes", func() { for _, p := range []string{"active.img", "passive.img"} { - out, _ := machine.SSHCommand(`sudo stat -c "%s" /run/initramfs/cos-state/cOS/` + p ) + out, _ := machine.SSHCommand(`sudo stat -c "%s" /run/initramfs/cos-state/cOS/` + p) Expect(out).Should(ContainSubstring("2097152000")) } }) @@ -166,9 +143,6 @@ var _ = Describe("c3os smoke", Label("smoke"), func() { }) It("propagate kubeconfig", func() { - if os.Getenv("FLAVOR") == "alpine" { - Skip("Skip on alpine") - } Eventually(func() string { out, _ := machine.SSHCommand("c3os get-kubeconfig") return out @@ -182,9 +156,6 @@ var _ = Describe("c3os smoke", Label("smoke"), func() { }) It("has roles", func() { - if os.Getenv("FLAVOR") == "alpine" { - Skip("Skip on alpine") - } uuid, _ := machine.SSHCommand("c3os uuid") Expect(uuid).ToNot(Equal("")) Eventually(func() string { @@ -200,9 +171,6 @@ var _ = Describe("c3os smoke", Label("smoke"), func() { }) It("has machines with different IPs", func() { - if os.Getenv("FLAVOR") == "alpine" { - Skip("Skip on alpine") - } Eventually(func() string { out, _ := machine.SSHCommand(`curl http://localhost:8080/api/machines`) return out @@ -232,9 +200,6 @@ var _ = Describe("c3os smoke", Label("smoke"), func() { }) It("upgrades to a specific version", func() { - if os.Getenv("FLAVOR") == "alpine" { - Skip("not working on alpine yet") - } version, _ := machine.SSHCommand("source /etc/os-release; echo $VERSION") out, _ := machine.SSHCommand("sudo c3os upgrade v1.21.4-32") diff --git a/tests/machine/machine.go b/tests/machine/machine.go index dc6d78705..de1f7eeeb 100644 --- a/tests/machine/machine.go +++ b/tests/machine/machine.go @@ -3,6 +3,7 @@ package machine import ( "context" "fmt" + "net" "os" "path/filepath" "time" @@ -164,7 +165,7 @@ func connectToHost() (*ssh.Client, *ssh.Session, error) { sshConfig.HostKeyCallback = ssh.InsecureIgnoreHostKey() - client, err := ssh.Dial("tcp", host(), sshConfig) + client, err := SSHDialTimeout("tcp", host(), sshConfig, 30*time.Second) if err != nil { return nil, nil, err } @@ -283,3 +284,53 @@ func GatherLog(logPath string) { fmt.Printf("File %s copied!\n", baseName) } + +type Conn struct { + net.Conn + ReadTimeout time.Duration + WriteTimeout time.Duration +} + +func (c *Conn) Read(b []byte) (int, error) { + err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout)) + if err != nil { + return 0, err + } + return c.Conn.Read(b) +} + +func (c *Conn) Write(b []byte) (int, error) { + err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout)) + if err != nil { + return 0, err + } + return c.Conn.Write(b) +} + +func SSHDialTimeout(network, addr string, config *ssh.ClientConfig, timeout time.Duration) (*ssh.Client, error) { + conn, err := net.DialTimeout(network, addr, timeout) + if err != nil { + return nil, err + } + + timeoutConn := &Conn{conn, timeout, timeout} + c, chans, reqs, err := ssh.NewClientConn(timeoutConn, addr, config) + if err != nil { + return nil, err + } + client := ssh.NewClient(c, chans, reqs) + + // this sends keepalive packets every 2 seconds + // there's no useful response from these, so we can just abort if there's an error + go func() { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + for range t.C { + _, _, err := client.Conn.SendRequest("keepalive@golang.org", true, nil) + if err != nil { + return + } + } + }() + return client, nil +} diff --git a/tests/tests_suite_test.go b/tests/tests_suite_test.go index a7905e9d0..26a857b28 100644 --- a/tests/tests_suite_test.go +++ b/tests/tests_suite_test.go @@ -66,3 +66,34 @@ func prepareVM() { machine.Create(sshPort) } } + +func gatherLogs() { + machine.Sudo("k3s kubectl get pods -A -o json > /run/pods.json") + machine.Sudo("k3s kubectl get events -A -o json > /run/events.json") + machine.Sudo("cat /proc/cmdline > /run/cmdline") + machine.Sudo("chmod 777 /run/events.json") + + machine.Sudo("df -h > /run/disk") + machine.Sudo("mount > /run/mounts") + machine.Sudo("blkid > /run/blkid") + + machine.GatherAllLogs( + []string{ + "edgevpn@c3os", + "c3os-agent", + "cos-setup-boot", + "cos-setup-network", + "c3os", + "k3s", + }, + []string{ + "/var/log/edgevpn.log", + "/var/log/c3os-agent.log", + "/run/pods.json", + "/run/disk", + "/run/mounts", + "/run/blkid", + "/run/events.json", + "/run/cmdline", + }) +} diff --git a/tests/upgrade_test.go b/tests/upgrade_k8s_test.go similarity index 85% rename from tests/upgrade_test.go rename to tests/upgrade_k8s_test.go index 150b4cb22..dc5ba3e76 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_k8s_test.go @@ -14,41 +14,14 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("k3s upgrade test", Label("upgrade"), func() { +var _ = Describe("k3s upgrade test", Label("upgrade-k8s"), func() { BeforeEach(func() { machine.EventuallyConnects() }) AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - machine.Sudo("k3s kubectl get pods -A -o json > /run/pods.json") - machine.Sudo("k3s kubectl get events -A -o json > /run/events.json") - machine.Sudo("cat /proc/cmdline > /run/cmdline") - machine.Sudo("chmod 777 /run/events.json") - - machine.Sudo("df -h > /run/disk") - machine.Sudo("mount > /run/mounts") - machine.Sudo("blkid > /run/blkid") - - machine.GatherAllLogs( - []string{ - "edgevpn@c3os", - "c3os-agent", - "cos-setup-boot", - "cos-setup-network", - "c3os", - "k3s", - }, - []string{ - "/var/log/edgevpn.log", - "/var/log/c3os-agent.log", - "/run/pods.json", - "/run/disk", - "/run/mounts", - "/run/blkid", - "/run/events.json", - "/run/cmdline", - }) + gatherLogs() } }) diff --git a/tests/upgrade_latest_cli_test.go b/tests/upgrade_latest_cli_test.go new file mode 100644 index 000000000..9bc3ec7c4 --- /dev/null +++ b/tests/upgrade_latest_cli_test.go @@ -0,0 +1,91 @@ +package mos_test + +import ( + "fmt" + "os" + "time" + + "github.com/c3os-io/c3os/tests/machine" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("k3s upgrade manual test", Label("upgrade-latest-with-cli"), func() { + + containerImage := os.Getenv("CONTAINER_IMAGE") + + BeforeEach(func() { + machine.EventuallyConnects() + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + gatherLogs() + } + }) + + Context("live cd", func() { + + It("has default service active", func() { + if containerImage == "" { + Fail("CONTAINER_IMAGE needs to be set") + } + + if os.Getenv("FLAVOR") == "alpine" { + out, _ := machine.Sudo("rc-status") + Expect(out).Should(ContainSubstring("c3os")) + Expect(out).Should(ContainSubstring("c3os-agent")) + } else { + // Eventually(func() string { + // out, _ := machine.SSHCommand("sudo systemctl status c3os-agent") + // return out + // }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token")) + + out, _ := machine.Sudo("systemctl status c3os") + Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os.service; enabled; vendor preset: disabled)")) + } + }) + }) + + Context("install", func() { + It("to disk with custom config", func() { + err := machine.SendFile("assets/config.yaml", "/tmp/config.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + + out, _ := machine.Sudo("elemental install --cloud-init /tmp/config.yaml /dev/sda") + Expect(out).Should(ContainSubstring("Running after-install hook")) + fmt.Println(out) + machine.Sudo("sync") + machine.DetachCD() + machine.Restart() + }) + }) + + Context("upgrades", func() { + It("can upgrade to current image", func() { + + currentVersion, err := machine.SSHCommand("source /etc/os-release; echo $VERSION") + Expect(err).ToNot(HaveOccurred()) + Expect(currentVersion).To(ContainSubstring("c3OS")) + out, err := machine.Sudo("c3os upgrade --force --image " + containerImage) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("Upgrade completed")) + Expect(out).To(ContainSubstring("Unpacking docker image: " + containerImage)) + + machine.Sudo("reboot") + machine.EventuallyConnects(750) + + Eventually(func() error { + _, err := machine.SSHCommand("source /etc/os-release; echo $VERSION") + return err + }, 10*time.Minute, 10*time.Second).ShouldNot(HaveOccurred()) + + var v string + Eventually(func() string { + v, _ = machine.SSHCommand("source /etc/os-release; echo $VERSION") + return v + }, 10*time.Minute, 10*time.Second).Should(ContainSubstring("c3OS")) + Expect(v).ToNot(Equal(currentVersion)) + }) + }) +}) diff --git a/tests/upgrade_latest_k8s_test.go b/tests/upgrade_latest_k8s_test.go new file mode 100644 index 000000000..33964f2e3 --- /dev/null +++ b/tests/upgrade_latest_k8s_test.go @@ -0,0 +1,197 @@ +package mos_test + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strings" + "time" + + "github.com/c3os-io/c3os/tests/machine" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func sucYAML(image, version string) string { + return ` +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: os-upgrade + namespace: system-upgrade + labels: + k3s-upgrade: server +spec: + concurrency: 1 + version: "` + version + `" + nodeSelector: + matchExpressions: + - {key: kubernetes.io/hostname, operator: Exists} + serviceAccountName: system-upgrade + cordon: false + upgrade: + image: "` + image + `" + command: + - "/usr/sbin/suc-upgrade" +` + +} + +var _ = Describe("k3s upgrade test from k8s", Label("upgrade-latest-with-kubernetes"), func() { + containerImage := os.Getenv("CONTAINER_IMAGE") + + BeforeEach(func() { + machine.EventuallyConnects() + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + gatherLogs() + } + }) + + Context("live cd", func() { + It("has default service active", func() { + if containerImage == "" { + Fail("CONTAINER_IMAGE needs to be set") + } + if os.Getenv("FLAVOR") == "alpine" { + out, _ := machine.Sudo("rc-status") + Expect(out).Should(ContainSubstring("c3os")) + Expect(out).Should(ContainSubstring("c3os-agent")) + } else { + // Eventually(func() string { + // out, _ := machine.SSHCommand("sudo systemctl status c3os-agent") + // return out + // }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token")) + + out, _ := machine.Sudo("systemctl status c3os") + Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os.service; enabled; vendor preset: disabled)")) + } + }) + }) + + Context("install", func() { + It("to disk with custom config", func() { + err := machine.SendFile("assets/single.yaml", "/tmp/config.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + + out, _ := machine.Sudo("elemental install --cloud-init /tmp/config.yaml /dev/sda") + Expect(out).Should(ContainSubstring("Running after-install hook")) + fmt.Println(out) + machine.Sudo("sync") + machine.DetachCD() + machine.Restart() + }) + }) + + Context("first-boot", func() { + + It("has default services on", func() { + if os.Getenv("FLAVOR") == "alpine" { + out, _ := machine.Sudo("rc-status") + Expect(out).Should(ContainSubstring("c3os")) + Expect(out).Should(ContainSubstring("c3os-agent")) + } else { + // Eventually(func() string { + // out, _ := machine.SSHCommand("sudo systemctl status c3os-agent") + // return out + // }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token")) + + out, _ := machine.Sudo("systemctl status c3os-agent") + Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os-agent.service; enabled; vendor preset: disabled)")) + + out, _ = machine.Sudo("systemctl status systemd-timesyncd") + Expect(out).Should(ContainSubstring("loaded (/usr/lib/systemd/system/systemd-timesyncd.service; enabled; vendor preset: disabled)")) + } + }) + + It("upgrades from kubernetes", func() { + Eventually(func() string { + var out string + if os.Getenv("FLAVOR") == "alpine" { + out, _ = machine.Sudo("cat /var/log/c3os-agent.log") + } else { + out, _ = machine.Sudo("systemctl status c3os-agent") + } + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("One time bootstrap starting")) + + Eventually(func() string { + out, _ := machine.Sudo("cat /etc/rancher/k3s/k3s.yaml") + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("https:")) + + kubectl := func(s string) (string, error) { + return machine.Sudo("k3s kubectl " + s) + } + + currentVersion, err := machine.SSHCommand("source /etc/os-release; echo $VERSION") + Expect(err).ToNot(HaveOccurred()) + Expect(currentVersion).To(ContainSubstring("c3OS")) + + By("installing system-upgrade-controller", func() { + resp, err := http.Get("https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml") + Expect(err).ToNot(HaveOccurred()) + defer resp.Body.Close() + data := bytes.NewBuffer([]byte{}) + + _, err = io.Copy(data, resp.Body) + Expect(err).ToNot(HaveOccurred()) + + temp, err := ioutil.TempFile("", "temp") + Expect(err).ToNot(HaveOccurred()) + + defer os.RemoveAll(temp.Name()) + err = ioutil.WriteFile(temp.Name(), data.Bytes(), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = machine.SendFile(temp.Name(), "/tmp/kubectl.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() string { + out, _ := kubectl("apply -f /tmp/kubectl.yaml") + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("unchanged")) + }) + + By("triggering an upgrade", func() { + suc := sucYAML(strings.ReplaceAll(containerImage, ":8h", ""), "8h") + + err := ioutil.WriteFile("assets/generated.yaml", []byte(suc), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = machine.SendFile("assets/generated.yaml", "./suc.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + fmt.Println(suc) + + Eventually(func() string { + out, _ := kubectl("apply -f suc.yaml") + fmt.Println(out) + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("created")) + + Eventually(func() string { + out, _ := kubectl("get pods -A") + fmt.Println(out) + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("apply-os-upgrade-on-")) + + Eventually(func() string { + out, _ := kubectl("get pods -A") + fmt.Println(out) + version, err := machine.SSHCommand("source /etc/os-release; echo $VERSION") + if err != nil || !strings.Contains(version, "c3OS") { + // If we met error, keep going with the Eventually + return currentVersion + } + return version + }, 20*time.Minute, 10*time.Second).ShouldNot(Equal(currentVersion)) + }) + }) + }) +})