test/e2e: check MantleRestore correctly restores replicated backups i…

…n both clusters Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
cybozu-go · Dec 24, 2024 · 631465a · 631465a
1 parent 8ee441d
commit 631465a
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 16 deletions.
diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
@@ -53,11 +53,13 @@ func waitControllerToBeReady() {
 
 func replicationTestSuite() {
 	Describe("replication test", func() {
-		It("should correctly replicate PVC and MantleBackup resources", func() {
+		It("should correctly replicate PVC and MantleBackup resources", func(ctx SpecContext) {
 			namespace := util.GetUniqueName("ns-")
 			pvcName := util.GetUniqueName("pvc-")
 			backupName := util.GetUniqueName("mb-")
 			restoreName := util.GetUniqueName("mr-")
+			mountDeployName := util.GetUniqueName("deploy-")
+			writeJobName := util.GetUniqueName("job-")
 
 			By("setting up the environment")
 			Eventually(func() error {
@@ -76,6 +78,17 @@ func replicationTestSuite() {
 				return applyPVCTemplate(primaryK8sCluster, namespace, pvcName)
 			}).Should(Succeed())
 
+			By("writing some random data to PV(C)")
+			writtenDataHash := ""
+			Eventually(ctx, func(g Gomega) {
+				err := applyWriteJobTemplate(primaryK8sCluster, namespace, writeJobName, pvcName)
+				g.Expect(err).NotTo(HaveOccurred())
+				stdout, _, err := kubectl(primaryK8sCluster, nil, "logs", "-n", namespace, "job/"+writeJobName)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(len(stdout)).NotTo(Equal(0))
+				writtenDataHash = string(stdout)
+			}).Should(Succeed())
+
 			By("creating a MantleBackup object")
 			Eventually(func() error {
 				return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName)
@@ -164,22 +177,32 @@ func replicationTestSuite() {
 				return nil
 			}).Should(Succeed())
 
-			By("creating MantleRestore on the secondary k8s cluster by using the MantleBackup replicated above")
-			Eventually(func() error {
-				return applyMantleRestoreTemplate(secondaryK8sCluster, namespace, restoreName, backupName)
-			}).Should(Succeed())
-
-			By("checking MantleRestore can be ready to use")
-			Eventually(func() error {
-				mr, err := getMR(secondaryK8sCluster, namespace, restoreName)
-				if err != nil {
-					return err
+			By("checking MantleRestore correctly restores the replicated backups in both clusters")
+			for _, clusterNo := range []int{primaryK8sCluster, secondaryK8sCluster} {
+				clusterName := "primary"
+				if clusterNo == secondaryK8sCluster {
+					clusterName = "secondary"
 				}
-				if !meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse") {
-					return errors.New("ReadyToUse of .Status.Conditions is not True")
-				}
-				return nil
-			}).Should(Succeed())
+				By(clusterName + ": creating MantleRestore by using the MantleBackup replicated above")
+				Eventually(ctx, func() error {
+					return applyMantleRestoreTemplate(clusterNo, namespace, restoreName, backupName)
+				}).Should(Succeed())
+				By(clusterName + ": checking the MantleRestore can be ready to use")
+				Eventually(ctx, func(g Gomega) {
+					mr, err := getMR(clusterNo, namespace, restoreName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse")).To(BeTrue())
+				}).Should(Succeed())
+				By(clusterName + ": checking the MantleRestore has the correct contents")
+				Eventually(ctx, func(g Gomega) {
+					err := applyMountDeployTemplate(clusterNo, namespace, mountDeployName, restoreName)
+					g.Expect(err).NotTo(HaveOccurred())
+					stdout, _, err := kubectl(clusterNo, nil, "exec", "-n", namespace, "deploy/"+mountDeployName, "--",
+						"bash", "-c", "sha256sum /volume/data | awk '{print $1}'")
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(string(stdout)).To(Equal(writtenDataHash))
+				}).Should(Succeed())
+			}
 		})
 	})
 }

diff --git a/test/e2e/multik8s/testdata/mount-deploy-template.yaml b/test/e2e/multik8s/testdata/mount-deploy-template.yaml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  selector:
+    matchLabels:
+      app: %s
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: %s
+    spec:
+      securityContext:
+        runAsUser: 10000
+        runAsGroup: 10000
+      containers:
+      - name: ubuntu
+        image: ubuntu:22.04
+        volumeMounts:
+          - name: volume
+            mountPath: /volume
+        command:
+          - bash
+          - -c
+          - |
+            sleep infinity
+      volumes:
+        - name: volume
+          persistentVolumeClaim:
+            claimName: %s
diff --git a/test/e2e/multik8s/testdata/write-job-template.yaml b/test/e2e/multik8s/testdata/write-job-template.yaml
@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  template:
+    spec:
+      containers:
+      - name: ubuntu
+        image: ubuntu:22.04
+        command:
+        - bash
+        - -c
+        - |
+          dd if=/dev/urandom of=/volume/data bs=1K count=1 >&/dev/null
+          sha256sum /volume/data | awk '{print $1}'
+        volumeMounts:
+        - name: volume
+          mountPath: /volume
+      restartPolicy: Never
+      volumes:
+      - name: volume
+        persistentVolumeClaim:
+          claimName: %s
diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go
@@ -33,6 +33,10 @@ var (
 	testMantleBackupTemplate string
 	//go:embed testdata/mantlerestore-template.yaml
 	testMantleRestoreTemplate string
+	//go:embed testdata/mount-deploy-template.yaml
+	mountDeployTemplate string
+	//go:embed testdata/write-job-template.yaml
+	writeJobTemplate string
 
 	kubectlPrefixPrimary   = os.Getenv("KUBECTL_PRIMARY")
 	kubectlPrefixSecondary = os.Getenv("KUBECTL_SECONDARY")
@@ -109,6 +113,24 @@ func applyPVCTemplate(clusterNo int, namespace, name string) error {
 	return nil
 }
 
+func applyMountDeployTemplate(clusterNo int, namespace, name, pvcName string) error {
+	manifest := fmt.Sprintf(mountDeployTemplate, name, namespace, name, name, pvcName)
+	_, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-")
+	if err != nil {
+		return fmt.Errorf("kubectl apply mount deploy failed. err: %w", err)
+	}
+	return nil
+}
+
+func applyWriteJobTemplate(clusterNo int, namespace, name, pvcName string) error {
+	manifest := fmt.Sprintf(writeJobTemplate, name, namespace, pvcName)
+	_, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-")
+	if err != nil {
+		return fmt.Errorf("kubectl apply write job failed. err: %w", err)
+	}
+	return nil
+}
+
 func createNamespace(clusterNo int, name string) error {
 	_, _, err := kubectl(clusterNo, nil, "create", "ns", name)
 	if err != nil {