From 8ee441d853c416d239b9ea7e8c116d70733dcd2d Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Mon, 23 Dec 2024 04:55:16 +0000 Subject: [PATCH] test/e2e: multik8s: add tests when changing mantle-controller roles to standalone Signed-off-by: Ryotaro Banno --- test/e2e/multik8s/suite_test.go | 101 ++++++++++++++++++++++++++++++++ test/e2e/multik8s/util.go | 72 ++++++++++++++++++++++- 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go index 4796eea4..a19ffb6a 100644 --- a/test/e2e/multik8s/suite_test.go +++ b/test/e2e/multik8s/suite_test.go @@ -2,6 +2,7 @@ package multik8s import ( _ "embed" + "encoding/json" "errors" "os" "reflect" @@ -35,6 +36,7 @@ func TestMtest(t *testing.T) { var _ = Describe("Mantle", func() { Context("wait controller to be ready", waitControllerToBeReady) Context("replication test", replicationTestSuite) + Context("change to standalone", changeToStandalone) }) func waitControllerToBeReady() { @@ -181,3 +183,102 @@ func replicationTestSuite() { }) }) } + +func changeToStandalone() { + Describe("change to standalone", func() { + var namespace, pvcName, backupName string + + It("should replicate a MantleBackup resource", func() { + namespace = util.GetUniqueName("ns-") + pvcName = util.GetUniqueName("pvc-") + backupName = util.GetUniqueName("mb-") + + By("setting up the environment") + Eventually(func() error { + return createNamespace(primaryK8sCluster, namespace) + }).Should(Succeed()) + Eventually(func() error { + return createNamespace(secondaryK8sCluster, namespace) + }).Should(Succeed()) + Eventually(func() error { + return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace) + }).Should(Succeed()) + Eventually(func() error { + return applyRBDPoolAndSCTemplate(secondaryK8sCluster, cephClusterNamespace) + }).Should(Succeed()) + Eventually(func() error { + return applyPVCTemplate(primaryK8sCluster, namespace, pvcName) + }).Should(Succeed()) + + By("creating a MantleBackup resource") + Eventually(func() error { + return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName) + }).Should(Succeed()) + + By("checking MantleBackup's SyncedToRemote status") + Eventually(func() error { + mb, err := getMB(primaryK8sCluster, namespace, backupName) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) { + return errors.New("status of SyncedToRemote condition is not True") + } + return nil + }, "10m", "1s").Should(Succeed()) + }) + + It("should change the roles to standalone", func() { + By("changing the primary K8s cluster to standalone") + err := changeClusterRole(primaryK8sCluster, roleStandalone) + Expect(err).NotTo(HaveOccurred()) + By("changing the secondary K8s cluster to standalone") + err = changeClusterRole(secondaryK8sCluster, roleStandalone) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should delete MantleBackup in the primary cluster", func(ctx SpecContext) { + By("deleting the MantleBackup in the primary cluster") + _, _, err := kubectl(primaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false") + Expect(err).NotTo(HaveOccurred()) + + By("checking that the MantleBackup is actually deleted") + Eventually(ctx, func(g Gomega) { + stdout, _, err := kubectl(primaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + var mbs mantlev1.MantleBackupList + err = json.Unmarshal(stdout, &mbs) + g.Expect(err).NotTo(HaveOccurred()) + found := false + for _, mb := range mbs.Items { + if mb.GetName() == backupName { + found = true + } + } + g.Expect(found).To(BeFalse()) + }).Should(Succeed()) + }) + + It("should NOT delete MantleBackup in the secondary cluster", func(ctx SpecContext) { + By("deleting the MantleBackup in the secondary cluster") + _, _, err := kubectl(secondaryK8sCluster, nil, "delete", "mb", "-n", namespace, backupName, "--wait=false") + Expect(err).NotTo(HaveOccurred()) + + By("checking that the MantleBackup is NOT deleted") + Consistently(ctx, func(g Gomega) { + stdout, _, err := kubectl(secondaryK8sCluster, nil, "get", "mb", "-n", namespace, "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + var mbs mantlev1.MantleBackupList + err = json.Unmarshal(stdout, &mbs) + g.Expect(err).NotTo(HaveOccurred()) + found := false + for _, mb := range mbs.Items { + if mb.GetName() == backupName { + found = true + } + } + g.Expect(found).To(BeTrue()) + }, "10s", "1s").Should(Succeed()) + }) + }) +} diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go index ca08b530..c491100e 100644 --- a/test/e2e/multik8s/util.go +++ b/test/e2e/multik8s/util.go @@ -4,12 +4,16 @@ import ( "bytes" _ "embed" "encoding/json" + "errors" "fmt" "os" "os/exec" + "slices" "strings" + "time" mantlev1 "github.com/cybozu-go/mantle/api/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -17,6 +21,7 @@ const ( cephClusterNamespace = "rook-ceph" primaryK8sCluster = 1 secondaryK8sCluster = 2 + roleStandalone = "standalone" ) var ( @@ -112,7 +117,7 @@ func createNamespace(clusterNo int, name string) error { return nil } -func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error { +func applyRBDPoolAndSCTemplate(clusterNo int, namespace string) error { //nolint:unparam manifest := fmt.Sprintf( testRBDPoolSCTemplate, namespace, namespace, namespace, namespace, namespace) @@ -148,3 +153,68 @@ func getPVC(clusterNo int, namespace, name string) (*corev1.PersistentVolumeClai func getMR(clusterNo int, namespace, name string) (*mantlev1.MantleRestore, error) { return getObject[mantlev1.MantleRestore](clusterNo, "mantlerestore", namespace, name) } + +func getDeploy(clusterNo int, namespace, name string) (*appsv1.Deployment, error) { + return getObject[appsv1.Deployment](clusterNo, "deploy", namespace, name) +} + +func changeClusterRole(clusterNo int, newRole string) error { + deployName := "mantle-controller" + deploy, err := getDeploy(clusterNo, cephClusterNamespace, deployName) + if err != nil { + return fmt.Errorf("failed to get mantle-controller deploy: %w", err) + } + + args := deploy.Spec.Template.Spec.Containers[0].Args + roleIndex := -1 + for i, arg := range args { + if strings.HasPrefix(arg, "--role=") { + roleIndex = i + break + } + } + if roleIndex == -1 { + return errors.New("failed to find --role= argument") + } + + _, _, err = kubectl( + clusterNo, nil, "patch", "deploy", "-n", cephClusterNamespace, deployName, "--type=json", + fmt.Sprintf( + `-p=[{"op": "replace", "path": "/spec/template/spec/containers/0/args/%d", "value":"--role=%s"}]`, + roleIndex, + newRole, + ), + ) + if err != nil { + return fmt.Errorf("failed to patch mantle-controller deploy: %w", err) + } + + // Wait for the new controller to start + for i := 0; i < 10; i++ { + stdout, _, err := kubectl(clusterNo, nil, "get", "pod", "-n", cephClusterNamespace, "-o", "json") + if err != nil { + return fmt.Errorf("failed to get pod: %w", err) + } + var pods corev1.PodList + err = json.Unmarshal(stdout, &pods) + if err != nil { + return fmt.Errorf("failed to unmarshal pod list: %w", err) + } + ready := true + for _, pod := range pods.Items { + if strings.HasPrefix(pod.GetName(), deployName) { + for _, container := range pod.Spec.Containers { + if !slices.Contains(container.Args, fmt.Sprintf("--role=%s", newRole)) { + ready = false + } + } + } + } + if ready { + break + } + time.Sleep(10 * time.Second) + } + + return nil +}