test/e2e: multik8s: add tests when changing mantle-controller roles t…

…o standalone

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>

test/e2e/multik8s/suite_test.go

@@ -2,6 +2,7 @@ package multik8s
 
 import (
 	_ "embed"
+	"encoding/json"
 	"errors"
 	"os"
 	"reflect"
@@ -35,6 +36,7 @@ func TestMtest(t *testing.T) {
 var _ = Describe("Mantle", func() {
 	Context("wait controller to be ready", waitControllerToBeReady)
 	Context("replication test", replicationTestSuite)
+	Context("change to standalone", changeToStandalone)
 })
 
 func waitControllerToBeReady() {
@@ -181,3 +183,102 @@ func replicationTestSuite() {
 		})
 	})
 }
+
+func changeToStandalone() {
+	Describe("change to standalone", func() {
+		var namespace, pvcName, backupName string
+
+		It("should replicate a MantleBackup resource", func() {
+			namespace = util.GetUniqueName("ns-")
+			pvcName = util.GetUniqueName("pvc-")
+			backupName = util.GetUniqueName("mb-")
+
+			By("setting up the environment")
+			Eventually(func() error {
+				return createNamespace(primaryK8sCluster, namespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return createNamespace(secondaryK8sCluster, namespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace)
+			}).Should(Succeed())
+			Eventually(func() error {
+				return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
+			}).Should(Succeed())
+
+			By("creating a MantleBackup resource")
+			Eventually(func() error {
+				return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
+			}).Should(Succeed())
+
+			By("checking MantleBackup's SyncedToRemote status")
+			Eventually(func() error {
+				mb, err := getMB(primaryK8sCluster, namespace, backupName)
+				if err != nil {
+					return err
+				}
+				if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
+					return errors.New("status of SyncedToRemote condition is not True")
+				}
+				return nil
+			}, "10m", "1s").Should(Succeed())
+		})
+
+		It("should change the roles to standalone", func() {
+			By("changing the primary K8s cluster to standalone")
+			err := changeClusterRole(primaryK8sCluster, roleStandalone)
+			Expect(err).NotTo(HaveOccurred())
+			By("changing the secondary K8s cluster to standalone")
+			err = changeClusterRole(secondaryK8sCluster, roleStandalone)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should delete MantleBackup in the primary cluster", func(ctx SpecContext) {
+			By("deleting the MantleBackup in the primary cluster")
+			_, _, err := kubectl(primaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("checking that the MantleBackup is actually deleted")
+			Eventually(ctx, func(g Gomega) {
+				stdout, _, err := kubectl(primaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json")
+				g.Expect(err).NotTo(HaveOccurred())
+				var mbs mantlev1.MantleBackupList
+				err = json.Unmarshal(stdout, &mbs)
+				g.Expect(err).NotTo(HaveOccurred())
+				found := false
+				for _, mb := range mbs.Items {
+					if mb.GetName() == backupName {
+						found = true
+					}
+				}
+				g.Expect(found).To(BeFalse())
+			}).Should(Succeed())
+		})
+
+		It("should NOT delete MantleBackup in the secondary cluster", func(ctx SpecContext) {
+			By("deleting the MantleBackup in the secondary cluster")
+			_, _, err := kubectl(secondaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("checking that the MantleBackup is NOT deleted")
+			Consistently(ctx, func(g Gomega) {
+				stdout, _, err := kubectl(secondaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json")
+				g.Expect(err).NotTo(HaveOccurred())
+				var mbs mantlev1.MantleBackupList
+				err = json.Unmarshal(stdout, &mbs)
+				g.Expect(err).NotTo(HaveOccurred())
+				found := false
+				for _, mb := range mbs.Items {
+					if mb.GetName() == backupName {
+						found = true
+					}
+				}
+				g.Expect(found).To(BeTrue())
+			}, "10s", "1s").Should(Succeed())
+		})
+	})
+}

test/e2e/multik8s/util.go

@@ -4,19 +4,24 @@ import (
 	"bytes"
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
+	"time"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
 const (
 	cephClusterNamespace = "rook-ceph"
 	primaryK8sCluster    = 1
 	secondaryK8sCluster  = 2
+	roleStandalone       = "standalone"
 )
 
 var (
@@ -112,7 +117,7 @@ func createNamespace(clusterNo int, name string) error {
 	return nil
 }
 
-func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error {
+func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error { //nolint:unparam
 	manifest := fmt.Sprintf(
 		testRBDPoolSCTemplate, namespace,
 		namespace, namespace, namespace, namespace)
@@ -148,3 +153,68 @@ func getPVC(clusterNo int, namespace, name string) (*corev1.PersistentVolumeClai
 func getMR(clusterNo int, namespace, name string) (*mantlev1.MantleRestore, error) {
 	return getObject[mantlev1.MantleRestore](clusterNo, "mantlerestore", namespace, name)
 }
+
+func getDeploy(clusterNo int, namespace, name string) (*appsv1.Deployment, error) {
+	return getObject[appsv1.Deployment](clusterNo, "deploy", namespace, name)
+}
+
+func changeClusterRole(clusterNo int, newRole string) error {
+	deployName := "mantle-controller"
+	deploy, err := getDeploy(clusterNo, cephClusterNamespace, deployName)
+	if err != nil {
+		return fmt.Errorf("failed to get mantle-controller deploy: %w", err)
+	}
+
+	args := deploy.Spec.Template.Spec.Containers[0].Args
+	roleIndex := -1
+	for i, arg := range args {
+		if strings.HasPrefix(arg, "--role=") {
+			roleIndex = i
+			break
+		}
+	}
+	if roleIndex == -1 {
+		return errors.New("failed to find --role= argument")
+	}
+
+	_, _, err = kubectl(
+		clusterNo, nil, "patch", "deploy", "-n", cephClusterNamespace, deployName, "--type=json",
+		fmt.Sprintf(
+			`-p=[{"op": "replace", "path": "/spec/template/spec/containers/0/args/%d", "value":"--role=%s"}]`,
+			roleIndex,
+			newRole,
+		),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to patch mantle-controller deploy: %w", err)
+	}
+
+	// Wait for the new controller to start
+	for i := 0; i < 10; i++ {
+		stdout, _, err := kubectl(clusterNo, nil, "get", "pod", "-n", cephClusterNamespace, "-o", "json")
+		if err != nil {
+			return fmt.Errorf("failed to get pod: %w", err)
+		}
+		var pods corev1.PodList
+		err = json.Unmarshal(stdout, &pods)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal pod list: %w", err)
+		}
+		ready := true
+		for _, pod := range pods.Items {
+			if strings.HasPrefix(pod.GetName(), deployName) {
+				for _, container := range pod.Spec.Containers {
+					if !slices.Contains(container.Args, fmt.Sprintf("--role=%s", newRole)) {
+						ready = false
+					}
+				}
+			}
+		}
+		if ready {
+			break
+		}
+		time.Sleep(10 * time.Second)
+	}
+
+	return nil
+}