diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go index a19ffb6a..24ca514b 100644 --- a/test/e2e/multik8s/suite_test.go +++ b/test/e2e/multik8s/suite_test.go @@ -53,11 +53,13 @@ func waitControllerToBeReady() { func replicationTestSuite() { Describe("replication test", func() { - It("should correctly replicate PVC and MantleBackup resources", func() { + It("should correctly replicate PVC and MantleBackup resources", func(ctx SpecContext) { namespace := util.GetUniqueName("ns-") pvcName := util.GetUniqueName("pvc-") backupName := util.GetUniqueName("mb-") restoreName := util.GetUniqueName("mr-") + mountDeployName := util.GetUniqueName("deploy-") + writeJobName := util.GetUniqueName("job-") By("setting up the environment") Eventually(func() error { @@ -76,6 +78,17 @@ func replicationTestSuite() { return applyPVCTemplate(primaryK8sCluster, namespace, pvcName) }).Should(Succeed()) + By("writing some random data to PV(C)") + writtenDataHash := "" + Eventually(ctx, func(g Gomega) { + err := applyWriteJobTemplate(primaryK8sCluster, namespace, writeJobName, pvcName) + g.Expect(err).NotTo(HaveOccurred()) + stdout, _, err := kubectl(primaryK8sCluster, nil, "logs", "-n", namespace, "job/"+writeJobName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(stdout)).NotTo(Equal(0)) + writtenDataHash = string(stdout) + }).Should(Succeed()) + By("creating a MantleBackup object") Eventually(func() error { return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName) @@ -164,22 +177,32 @@ func replicationTestSuite() { return nil }).Should(Succeed()) - By("creating MantleRestore on the secondary k8s cluster by using the MantleBackup replicated above") - Eventually(func() error { - return applyMantleRestoreTemplate(secondaryK8sCluster, namespace, restoreName, backupName) - }).Should(Succeed()) - - By("checking MantleRestore can be ready to use") - Eventually(func() error { - mr, err := getMR(secondaryK8sCluster, namespace, restoreName) - if err != nil { - return err + By("checking MantleRestore correctly restores the replicated backups in both clusters") + for _, clusterNo := range []int{primaryK8sCluster, secondaryK8sCluster} { + clusterName := "primary" + if clusterNo == secondaryK8sCluster { + clusterName = "secondary" } - if !meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse") { - return errors.New("ReadyToUse of .Status.Conditions is not True") - } - return nil - }).Should(Succeed()) + By(clusterName + ": creating MantleRestore by using the MantleBackup replicated above") + Eventually(ctx, func() error { + return applyMantleRestoreTemplate(clusterNo, namespace, restoreName, backupName) + }).Should(Succeed()) + By(clusterName + ": checking the MantleRestore can be ready to use") + Eventually(ctx, func(g Gomega) { + mr, err := getMR(clusterNo, namespace, restoreName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(meta.IsStatusConditionTrue(mr.Status.Conditions, "ReadyToUse")).To(BeTrue()) + }).Should(Succeed()) + By(clusterName + ": checking the MantleRestore has the correct contents") + Eventually(ctx, func(g Gomega) { + err := applyMountDeployTemplate(clusterNo, namespace, mountDeployName, restoreName) + g.Expect(err).NotTo(HaveOccurred()) + stdout, _, err := kubectl(clusterNo, nil, "exec", "-n", namespace, "deploy/"+mountDeployName, "--", + "bash", "-c", "sha256sum /volume/data | awk '{print $1}'") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(stdout)).To(Equal(writtenDataHash)) + }).Should(Succeed()) + } }) }) } diff --git a/test/e2e/multik8s/testdata/mount-deploy-template.yaml b/test/e2e/multik8s/testdata/mount-deploy-template.yaml new file mode 100644 index 00000000..a2d2fb7e --- /dev/null +++ b/test/e2e/multik8s/testdata/mount-deploy-template.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %s + namespace: %s +spec: + selector: + matchLabels: + app: %s + replicas: 1 + template: + metadata: + labels: + app: %s + spec: + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + containers: + - name: ubuntu + image: ubuntu:22.04 + volumeMounts: + - name: volume + mountPath: /volume + command: + - bash + - -c + - | + sleep infinity + volumes: + - name: volume + persistentVolumeClaim: + claimName: %s diff --git a/test/e2e/multik8s/testdata/write-job-template.yaml b/test/e2e/multik8s/testdata/write-job-template.yaml new file mode 100644 index 00000000..88d0bcde --- /dev/null +++ b/test/e2e/multik8s/testdata/write-job-template.yaml @@ -0,0 +1,25 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: %s + namespace: %s +spec: + template: + spec: + containers: + - name: ubuntu + image: ubuntu:22.04 + command: + - bash + - -c + - | + dd if=/dev/urandom of=/volume/data bs=1K count=1 >&/dev/null + sha256sum /volume/data | awk '{print $1}' + volumeMounts: + - name: volume + mountPath: /volume + restartPolicy: Never + volumes: + - name: volume + persistentVolumeClaim: + claimName: %s diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go index c491100e..509028be 100644 --- a/test/e2e/multik8s/util.go +++ b/test/e2e/multik8s/util.go @@ -33,6 +33,10 @@ var ( testMantleBackupTemplate string //go:embed testdata/mantlerestore-template.yaml testMantleRestoreTemplate string + //go:embed testdata/mount-deploy-template.yaml + mountDeployTemplate string + //go:embed testdata/write-job-template.yaml + writeJobTemplate string kubectlPrefixPrimary = os.Getenv("KUBECTL_PRIMARY") kubectlPrefixSecondary = os.Getenv("KUBECTL_SECONDARY") @@ -109,6 +113,24 @@ func applyPVCTemplate(clusterNo int, namespace, name string) error { return nil } +func applyMountDeployTemplate(clusterNo int, namespace, name, pvcName string) error { + manifest := fmt.Sprintf(mountDeployTemplate, name, namespace, name, name, pvcName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply mount deploy failed. err: %w", err) + } + return nil +} + +func applyWriteJobTemplate(clusterNo int, namespace, name, pvcName string) error { + manifest := fmt.Sprintf(writeJobTemplate, name, namespace, pvcName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply write job failed. err: %w", err) + } + return nil +} + func createNamespace(clusterNo int, name string) error { _, _, err := kubectl(clusterNo, nil, "create", "ns", name) if err != nil {