From 0745dd7f6dacf301cdbf0b83a736e08649a6b060 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Wed, 30 Oct 2024 07:22:03 +0000 Subject: [PATCH 1/6] make sure FROM_SNAP_NAME is correctly filled Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index d41c74b5..585cbfec 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -489,6 +489,16 @@ var _ = Describe("MantleBackup controller", func() { Expect(*jobExport.Spec.Template.Spec.SecurityContext.RunAsGroup).To(Equal(int64(10000))) Expect(*jobExport.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(Equal(true)) + // Make sure FROM_SNAP_NAME is empty because we're performing a full backup. + isFromSnapNameFound := false + for _, evar := range jobExport.Spec.Template.Spec.Containers[0].Env { + if evar.Name == "FROM_SNAP_NAME" { + Expect(evar.Value).To(Equal("")) + isFromSnapNameFound = true + } + } + Expect(isFromSnapNameFound).To(BeTrue()) + // Make the export Job completed to proceed the reconciliation for backup. err = resMgr.ChangeJobCondition(ctx, &jobExport, batchv1.JobComplete, corev1.ConditionTrue) Expect(err).NotTo(HaveOccurred()) @@ -541,6 +551,28 @@ var _ = Describe("MantleBackup controller", func() { Expect(ok).To(BeTrue()) Expect(diffTo).To(Equal(backup2.GetName())) + // Make sure export() creates a Job to export data for backup2. + var jobExport2 batchv1.Job + err = k8sClient.Get( + ctx, + types.NamespacedName{ + Name: fmt.Sprintf("mantle-export-%s", backup2.GetUID()), + Namespace: resMgr.ClusterID, + }, + &jobExport2, + ) + Expect(err).NotTo(HaveOccurred()) + + // Make sure FROM_SNAP_NAME is filled correctly because we're performing an incremental backup. + isFromSnapNameFound = false + for _, evar := range jobExport2.Spec.Template.Spec.Containers[0].Env { + if evar.Name == "FROM_SNAP_NAME" { + Expect(evar.Value).To(Equal(backup.GetName())) + isFromSnapNameFound = true + } + } + Expect(isFromSnapNameFound).To(BeTrue()) + // remove diffTo annotation of backup here to allow it to be deleted. // FIXME: this process is for testing purposes only and should be removed in the near future. _, err = ctrl.CreateOrUpdate(ctx, k8sClient, backup, func() error { From 582faab424767d4e45dffdbdd7d8314f1a9077d0 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Wed, 30 Oct 2024 07:30:55 +0000 Subject: [PATCH 2/6] make sure export data upload job is not created before export job is completed Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 585cbfec..573dfa2d 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -17,6 +17,7 @@ import ( "google.golang.org/grpc" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + aerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -499,6 +500,21 @@ var _ = Describe("MantleBackup controller", func() { } Expect(isFromSnapNameFound).To(BeTrue()) + // Make sure an upload Jobs has not yet been created. + Consistently(ctx, func(g Gomega) error { + var jobUpload batchv1.Job + err = k8sClient.Get( + ctx, + types.NamespacedName{ + Name: fmt.Sprintf("mantle-upload-%s", backup.GetUID()), + Namespace: resMgr.ClusterID, + }, + &jobUpload, + ) + g.Expect(aerrors.IsNotFound(err)).To(BeTrue()) + return nil + }, "1s").Should(Succeed()) + // Make the export Job completed to proceed the reconciliation for backup. err = resMgr.ChangeJobCondition(ctx, &jobExport, batchv1.JobComplete, corev1.ConditionTrue) Expect(err).NotTo(HaveOccurred()) From e7ee96718a0a100636e862bfca7e0820a5da6f03 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Thu, 31 Oct 2024 02:38:45 +0000 Subject: [PATCH 3/6] add unit tests for SetSynchronizing Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 573dfa2d..b4980a2e 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "encoding/json" + "errors" "fmt" "slices" "sync" @@ -942,3 +943,192 @@ var _ = Describe("prepareForDataSynchronization", func() { ), ) }) + +var _ = Describe("SetSynchronizing", func() { + doTestCallOnce := func( + target, source *mantlev1.MantleBackup, + shouldBeError bool, + check func(target, source *mantlev1.MantleBackup) error, + ) { + var err error + targetName := "target-name" + target.SetName(targetName) + backupNamespace := "target-ns" + target.SetNamespace(backupNamespace) + sourceName := "source" + + ctrlClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + + err = ctrlClient.Create(context.Background(), target) + Expect(err).NotTo(HaveOccurred()) + + var diffFrom *string + if source != nil { + source.SetName(sourceName) + source.SetNamespace(backupNamespace) + err = ctrlClient.Create(context.Background(), source) + Expect(err).NotTo(HaveOccurred()) + diffFrom = &sourceName + } + + secondaryServer := NewSecondaryServer(ctrlClient, ctrlClient) + _, err = secondaryServer.SetSynchronizing(context.Background(), &proto.SetSynchronizingRequest{ + Name: targetName, + Namespace: backupNamespace, + DiffFrom: diffFrom, + }) + if shouldBeError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).NotTo(HaveOccurred()) + } + + err = ctrlClient.Get( + context.Background(), + types.NamespacedName{Name: targetName, Namespace: backupNamespace}, + target, + ) + Expect(err).NotTo(HaveOccurred()) + if source != nil { + err = ctrlClient.Get( + context.Background(), + types.NamespacedName{Name: sourceName, Namespace: backupNamespace}, + source, + ) + Expect(err).NotTo(HaveOccurred()) + } + err = check(target, source) + Expect(err).NotTo(HaveOccurred()) + } + DescribeTable("call SetSynchronizing once", doTestCallOnce, + Entry( + "a full backup should succeed", + &mantlev1.MantleBackup{ + Status: mantlev1.MantleBackupStatus{ + Conditions: []metav1.Condition{ + { + Type: mantlev1.BackupConditionReadyToUse, + Status: metav1.ConditionFalse, + }, + }, + }, + }, + nil, + false, + func(target, source *mantlev1.MantleBackup) error { + syncMode, ok := target.GetAnnotations()[annotSyncMode] + if !ok || syncMode != syncModeFull { + return errors.New("syncMode is invalid") + } + if _, ok := target.GetAnnotations()[annotDiffFrom]; ok { + return errors.New("diffFrom should not exist") + } + return nil + }, + ), + Entry( + "an incremental backup should succeed", + &mantlev1.MantleBackup{ + Status: mantlev1.MantleBackupStatus{ + Conditions: []metav1.Condition{ + { + Type: mantlev1.BackupConditionReadyToUse, + Status: metav1.ConditionFalse, + }, + }, + }, + }, + &mantlev1.MantleBackup{}, + false, + func(target, source *mantlev1.MantleBackup) error { + syncMode, ok := target.GetAnnotations()[annotSyncMode] + if !ok || syncMode != syncModeIncremental { + return errors.New("syncMode is invalid") + } + diffFrom, ok := target.GetAnnotations()[annotDiffFrom] + if !ok || diffFrom != source.GetName() { + return errors.New("diffFrom is invalid") + } + diffTo, ok := source.GetAnnotations()[annotDiffTo] + if !ok || diffTo != target.GetName() { + return errors.New("diffTo is invalid") + } + return nil + }, + ), + Entry( + "a backup should fail if target's ReadyToUse is True", + &mantlev1.MantleBackup{ + Status: mantlev1.MantleBackupStatus{ + Conditions: []metav1.Condition{ + { + Type: mantlev1.BackupConditionReadyToUse, + Status: metav1.ConditionTrue, + }, + }, + }, + }, + nil, + true, + func(_, _ *mantlev1.MantleBackup) error { return nil }, + ), + ) + + doTestCallTwice := func( + name1 string, + diffFrom1 *string, + name2 string, + diffFrom2 *string, + shouldBeError bool, + ) { + var err error + + ctrlClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + + for _, name := range []string{"M0", "M1", "M2"} { + err = ctrlClient.Create(context.Background(), &mantlev1.MantleBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: mantlev1.MantleBackupStatus{ + Conditions: []metav1.Condition{ + { + Type: mantlev1.BackupConditionReadyToUse, + Status: metav1.ConditionFalse, + }, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + } + + secondaryServer := NewSecondaryServer(ctrlClient, ctrlClient) + _, err = secondaryServer.SetSynchronizing(context.Background(), &proto.SetSynchronizingRequest{ + Name: name1, + Namespace: "", + DiffFrom: diffFrom1, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = secondaryServer.SetSynchronizing(context.Background(), &proto.SetSynchronizingRequest{ + Name: name2, + Namespace: "", + DiffFrom: diffFrom2, + }) + if shouldBeError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).NotTo(HaveOccurred()) + } + } + m0 := "M0" + m1 := "M1" + m2 := "M2" + DescribeTable("call SetSynchronizing twice", doTestCallTwice, + Entry("case 1", m0, nil, m0, nil, false), + Entry("case 2", m1, &m0, m1, &m0, false), + Entry("case 3", m1, nil, m1, &m0, true), + Entry("case 4", m1, &m0, m1, nil, true), + Entry("case 5", m2, &m0, m2, &m1, true), + Entry("case 6", m1, &m0, m2, &m0, true), + ) +}) From e80a9441e88b4d98a7252408fabc8ae1fd116c57 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Thu, 31 Oct 2024 06:31:42 +0000 Subject: [PATCH 4/6] add test for export() to make sure diff-from annot is not set in full backup Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index b4980a2e..63572d31 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" aerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" @@ -1132,3 +1133,103 @@ var _ = Describe("SetSynchronizing", func() { Entry("case 6", m1, &m0, m2, &m0, true), ) }) + +var _ = Describe("export", func() { + var mockCtrl *gomock.Controller + var grpcClient *proto.MockMantleServiceClient + var mbr *MantleBackupReconciler + var ns string + + BeforeEach(func() { + var t reporter + mockCtrl = gomock.NewController(t) + grpcClient = proto.NewMockMantleServiceClient(mockCtrl) + + mbr = NewMantleBackupReconciler( + k8sClient, + scheme.Scheme, + resMgr.ClusterID, + RolePrimary, + &PrimarySettings{ + Client: grpcClient, + ExportDataStorageClass: resMgr.StorageClassName, + }, + "dummy image", + "", + nil, + ) + + ns = resMgr.CreateNamespace() + }) + + AfterEach(func() { + if mockCtrl != nil { + mockCtrl.Finish() + } + }) + + It("should set correct annotations after export() is called", func(ctx SpecContext) { + pvc := corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + pvcManifest, err := json.Marshal(pvc) + Expect(err).NotTo(HaveOccurred()) + + pv := corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + VolumeAttributes: map[string]string{ + "pool": "dummy", + "imageName": "dummy", + }, + }, + }, + }, + } + pvManifest, err := json.Marshal(pv) + Expect(err).NotTo(HaveOccurred()) + + target := &mantlev1.MantleBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target", + Namespace: ns, + }, + Spec: mantlev1.MantleBackupSpec{ + PVC: "dummy", + Expire: "1d", + }, + } + err = k8sClient.Create(ctx, target) + Expect(err).NotTo(HaveOccurred()) + err = updateStatus(ctx, k8sClient, target, func() error { + target.Status.PVManifest = string(pvManifest) + target.Status.PVCManifest = string(pvcManifest) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + grpcClient.EXPECT().SetSynchronizing(gomock.Any(), gomock.Any()). + Times(1).Return(&proto.SetSynchronizingResponse{}, nil) + + ret, err := mbr.export(ctx, target, &dataSyncPrepareResult{ + isIncremental: false, + isSecondaryMantleBackupReadyToUse: false, + diffFrom: nil, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(ret.Requeue).To(BeTrue()) + + err = k8sClient.Get(ctx, + types.NamespacedName{Name: target.GetName(), Namespace: target.GetNamespace()}, target) + Expect(err).NotTo(HaveOccurred()) + _, ok := target.GetAnnotations()[annotDiffFrom] + Expect(ok).To(BeFalse()) + }) +}) From 0241f1552ac6e7394933841d0ad8d7f35584dca9 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Wed, 6 Nov 2024 04:33:53 +0000 Subject: [PATCH 5/6] add test for export() to make sure annots are set in incremental backup Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 63572d31..e50b5fef 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -1196,6 +1196,7 @@ var _ = Describe("export", func() { pvManifest, err := json.Marshal(pv) Expect(err).NotTo(HaveOccurred()) + // test a full backup target := &mantlev1.MantleBackup{ ObjectMeta: metav1.ObjectMeta{ Name: "target", @@ -1231,5 +1232,50 @@ var _ = Describe("export", func() { Expect(err).NotTo(HaveOccurred()) _, ok := target.GetAnnotations()[annotDiffFrom] Expect(ok).To(BeFalse()) + + // test an incremental backup + target2 := &mantlev1.MantleBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target2", + Namespace: ns, + }, + Spec: mantlev1.MantleBackupSpec{ + PVC: "dummy", + Expire: "1d", + }, + } + err = k8sClient.Create(ctx, target2) + Expect(err).NotTo(HaveOccurred()) + err = updateStatus(ctx, k8sClient, target2, func() error { + target2.Status.PVManifest = string(pvManifest) + target2.Status.PVCManifest = string(pvcManifest) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + grpcClient.EXPECT().SetSynchronizing(gomock.Any(), gomock.Any()). + Times(1).Return(&proto.SetSynchronizingResponse{}, nil) + + ret, err = mbr.export(ctx, target2, &dataSyncPrepareResult{ + isIncremental: true, + isSecondaryMantleBackupReadyToUse: false, + diffFrom: target, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(ret.Requeue).To(BeTrue()) + + err = k8sClient.Get(ctx, + types.NamespacedName{Name: target.GetName(), Namespace: target.GetNamespace()}, target) + Expect(err).NotTo(HaveOccurred()) + diffTo, ok := target.GetAnnotations()[annotDiffTo] + Expect(ok).To(BeTrue()) + Expect(diffTo).To(Equal(target2.GetName())) + + err = k8sClient.Get(ctx, + types.NamespacedName{Name: target2.GetName(), Namespace: target2.GetNamespace()}, target2) + Expect(err).NotTo(HaveOccurred()) + diffFrom, ok := target2.GetAnnotations()[annotDiffFrom] + Expect(ok).To(BeTrue()) + Expect(diffFrom).To(Equal(target.GetName())) }) }) From b395af88a15e5d7ca9deb4dd5345c28d3abb7142 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Wed, 6 Nov 2024 07:48:35 +0000 Subject: [PATCH 6/6] add tests for export to make sure throttling works correctly Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 128 +++++++++++++++--- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index e50b5fef..51ec9a26 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -22,10 +22,12 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/kube-openapi/pkg/validation/strfmt" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" @@ -1138,21 +1140,25 @@ var _ = Describe("export", func() { var mockCtrl *gomock.Controller var grpcClient *proto.MockMantleServiceClient var mbr *MantleBackupReconciler - var ns string + var nsController, ns string + var dummyPVCManifest, dummyPVManifest []byte BeforeEach(func() { var t reporter mockCtrl = gomock.NewController(t) grpcClient = proto.NewMockMantleServiceClient(mockCtrl) + nsController = resMgr.CreateNamespace() + mbr = NewMantleBackupReconciler( k8sClient, scheme.Scheme, - resMgr.ClusterID, + nsController, RolePrimary, &PrimarySettings{ Client: grpcClient, ExportDataStorageClass: resMgr.StorageClassName, + MaxExportJobs: 1, }, "dummy image", "", @@ -1160,15 +1166,9 @@ var _ = Describe("export", func() { ) ns = resMgr.CreateNamespace() - }) - AfterEach(func() { - if mockCtrl != nil { - mockCtrl.Finish() - } - }) + var err error - It("should set correct annotations after export() is called", func(ctx SpecContext) { pvc := corev1.PersistentVolumeClaim{ Spec: corev1.PersistentVolumeClaimSpec{ Resources: corev1.VolumeResourceRequirements{ @@ -1178,7 +1178,7 @@ var _ = Describe("export", func() { }, }, } - pvcManifest, err := json.Marshal(pvc) + dummyPVCManifest, err = json.Marshal(pvc) Expect(err).NotTo(HaveOccurred()) pv := corev1.PersistentVolume{ @@ -1193,8 +1193,18 @@ var _ = Describe("export", func() { }, }, } - pvManifest, err := json.Marshal(pv) + dummyPVManifest, err = json.Marshal(pv) Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + if mockCtrl != nil { + mockCtrl.Finish() + } + }) + + It("should set correct annotations after export() is called", func(ctx SpecContext) { + var err error // test a full backup target := &mantlev1.MantleBackup{ @@ -1210,8 +1220,8 @@ var _ = Describe("export", func() { err = k8sClient.Create(ctx, target) Expect(err).NotTo(HaveOccurred()) err = updateStatus(ctx, k8sClient, target, func() error { - target.Status.PVManifest = string(pvManifest) - target.Status.PVCManifest = string(pvcManifest) + target.Status.PVManifest = string(dummyPVManifest) + target.Status.PVCManifest = string(dummyPVCManifest) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -1246,9 +1256,9 @@ var _ = Describe("export", func() { } err = k8sClient.Create(ctx, target2) Expect(err).NotTo(HaveOccurred()) - err = updateStatus(ctx, k8sClient, target2, func() error { - target2.Status.PVManifest = string(pvManifest) - target2.Status.PVCManifest = string(pvcManifest) + err = updateStatus(context.Background(), k8sClient, target2, func() error { + target2.Status.PVManifest = string(dummyPVManifest) + target2.Status.PVCManifest = string(dummyPVCManifest) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -1278,4 +1288,90 @@ var _ = Describe("export", func() { Expect(ok).To(BeTrue()) Expect(diffFrom).To(Equal(target.GetName())) }) + + It("should throttle export jobs correctly", func(ctx SpecContext) { + var err error + + getNumOfExportJobs := func(ns string) (int, error) { + var jobs batchv1.JobList + err := k8sClient.List(ctx, &jobs, &client.ListOptions{ + Namespace: ns, + LabelSelector: labels.SelectorFromSet(map[string]string{ + "app.kubernetes.io/name": labelAppNameValue, + "app.kubernetes.io/component": labelComponentExportJob, + }), + }) + return len(jobs.Items), err + } + + createAndExportMantleBackup := func(mbr *MantleBackupReconciler, name, ns string) { + target := &mantlev1.MantleBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: mantlev1.MantleBackupSpec{ + PVC: "dummy", + Expire: "1d", + }, + } + err = k8sClient.Create(ctx, target) + Expect(err).NotTo(HaveOccurred()) + + err = updateStatus(ctx, k8sClient, target, func() error { + target.Status.PVManifest = string(dummyPVManifest) + target.Status.PVCManifest = string(dummyPVCManifest) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + grpcClient.EXPECT().SetSynchronizing(gomock.Any(), gomock.Any()). + Times(1).Return(&proto.SetSynchronizingResponse{}, nil) + + _, err = mbr.export(ctx, target, &dataSyncPrepareResult{ + isIncremental: false, + isSecondaryMantleBackupReadyToUse: false, + diffFrom: nil, + }) + Expect(err).NotTo(HaveOccurred()) + } + + // create 5 different MantleBackup resources and call export() for each of them + for i := 0; i < 5; i++ { + createAndExportMantleBackup(mbr, fmt.Sprintf("target1-%d", i), ns) + } + + // make sure that only 1 Job is created + Consistently(ctx, func(g Gomega) error { + numJobs, err := getNumOfExportJobs(nsController) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(numJobs).To(Equal(1)) + return nil + }, "1s").Should(Succeed()) + + // make sure that another mantle-controller existing in a different namespace can create an export Job. + nsController2 := resMgr.CreateNamespace() + mbr2 := NewMantleBackupReconciler( + k8sClient, + scheme.Scheme, + nsController2, + RolePrimary, + &PrimarySettings{ + Client: grpcClient, + ExportDataStorageClass: resMgr.StorageClassName, + MaxExportJobs: 1, + }, + "dummy image", + "", + nil, + ) + ns2 := resMgr.CreateNamespace() + createAndExportMantleBackup(mbr2, "target2", ns2) + Eventually(ctx, func(g Gomega) error { + numJobs, err := getNumOfExportJobs(nsController2) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(numJobs).To(Equal(1)) + return nil + }).Should(Succeed()) + }) })