Skip to content

Commit

Permalink
Merge pull request #82 from cybozu-go/enhance-tests-part2
Browse files Browse the repository at this point in the history
enhance tests (part2)
  • Loading branch information
satoru-takeuchi authored Jan 14, 2025
2 parents 1e81d44 + e138da3 commit 083ea99
Show file tree
Hide file tree
Showing 23 changed files with 1,333 additions and 936 deletions.
10 changes: 9 additions & 1 deletion .github/workflows/e2e-multiple-k8s-clusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ concurrency:

jobs:
build:
strategy:
fail-fast: false
matrix:
package:
- replication
- changetostandalone
- changetoprimary
- changetosecondary
#runs-on: "ubuntu-22.04"
runs-on: mantle_large_runner_16core
steps:
Expand All @@ -29,4 +37,4 @@ jobs:
- run: sudo apt-get update
- uses: ./.github/actions/set-up-kvm-for-e2e-tests
- run: make -C test/e2e setup
- run: make -C test/e2e test-multiple-k8s-clusters
- run: make -C test/e2e test-multiple-k8s-clusters TEST_MULTIK8S_PACKAGES=${{ matrix.package }}
3 changes: 3 additions & 0 deletions charts/mantle/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ spec:
fieldPath: metadata.namespace
- name: POD_IMAGE
value: {{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}
{{- range .Values.controller.env }}
- {{ toYaml . | nindent 14 }}
{{- end }}
ports:
{{- toYaml .Values.controller.ports | nindent 12 }}
- command:
Expand Down
1 change: 1 addition & 0 deletions charts/mantle/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ affinity: {}
controller:
role: standalone
ports: []
env: []

secondaryService:
# type:
Expand Down
32 changes: 16 additions & 16 deletions internal/controller/mantlebackup_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ func (r *MantleBackupReconciler) getSnapshotTarget(ctx context.Context, backup *
}
if !ok {
logger.Info("waiting for PVC bound.")
return nil, ctrl.Result{Requeue: true}, nil
return nil, requeueReconciliation(), nil
}

pvName := pvc.Spec.VolumeName
Expand Down Expand Up @@ -392,7 +392,7 @@ func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, back
default:
return ctrl.Result{}, getSnapshotTargetErr
}
if result.Requeue {
if !result.IsZero() {
return result, nil
}

Expand All @@ -415,7 +415,7 @@ func (r *MantleBackupReconciler) reconcileAsStandalone(ctx context.Context, back
if err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if err := r.expire(ctx, backup); err != nil {
Expand Down Expand Up @@ -461,7 +461,7 @@ func (r *MantleBackupReconciler) reconcileAsSecondary(ctx context.Context, backu
default:
return ctrl.Result{}, getSnapshotTargetErr
}
if result.Requeue {
if !result.IsZero() {
return result, nil
}

Expand Down Expand Up @@ -564,7 +564,7 @@ func (r *MantleBackupReconciler) replicateManifests(
*backup1.Status.SnapID < *backup.Status.SnapID &&
backup1.ObjectMeta.DeletionTimestamp.IsZero() &&
!meta.IsStatusConditionTrue(backup1.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}
}

Expand Down Expand Up @@ -703,7 +703,7 @@ func (r *MantleBackupReconciler) finalizeStandalone(
) (ctrl.Result, error) {
logger := log.FromContext(ctx)
if _, ok := backup.GetAnnotations()[annotDiffTo]; ok {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
Expand Down Expand Up @@ -741,7 +741,7 @@ func (r *MantleBackupReconciler) finalizeSecondary(
) (ctrl.Result, error) {
logger := log.FromContext(ctx)
if _, ok := backup.GetAnnotations()[annotDiffTo]; ok {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if !controllerutil.ContainsFinalizer(backup, MantleBackupFinalizerName) {
Expand Down Expand Up @@ -965,7 +965,7 @@ func (r *MantleBackupReconciler) export(
return ctrl.Result{}, err
}

return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

func (r *MantleBackupReconciler) annotateExportTargetMantleBackup(
Expand Down Expand Up @@ -1025,7 +1025,7 @@ func (r *MantleBackupReconciler) checkIfNewJobCanBeCreated(ctx context.Context)
}

if len(jobs.Items) >= r.primarySettings.MaxExportJobs {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

return ctrl.Result{}, nil
Expand Down Expand Up @@ -1271,7 +1271,7 @@ func (r *MantleBackupReconciler) checkIfExportJobIsCompleted(
return ctrl.Result{}, nil
}

return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.Context, target *mantlev1.MantleBackup) error {
Expand Down Expand Up @@ -1422,7 +1422,7 @@ func (r *MantleBackupReconciler) startImport(
) (ctrl.Result, error) {
if !r.doesMantleBackupHaveSyncModeAnnot(backup) {
// SetSynchronizingg is not called yet or the cache is stale.
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if result, err := r.isExportDataAlreadyUploaded(ctx, backup); err != nil || !result.IsZero() {
Expand All @@ -1431,7 +1431,7 @@ func (r *MantleBackupReconciler) startImport(

// Requeue if the PV is smaller than the PVC. (This may be the case if pvc-autoresizer is used.)
if isPVSmallerThanPVC(target.pv, target.pvc) {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if err := r.updateStatusManifests(ctx, backup, target.pv, target.pvc); err != nil {
Expand Down Expand Up @@ -1528,7 +1528,7 @@ func (r *MantleBackupReconciler) isExportDataAlreadyUploaded(
if uploaded {
return ctrl.Result{}, nil
}
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

func isPVSmallerThanPVC(
Expand Down Expand Up @@ -1592,7 +1592,7 @@ func (r *MantleBackupReconciler) reconcileDiscardJob(
if completed {
return ctrl.Result{}, nil
}
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

func (r *MantleBackupReconciler) createOrUpdateDiscardPV(
Expand Down Expand Up @@ -1777,11 +1777,11 @@ func (r *MantleBackupReconciler) reconcileImportJob(
if err := r.createOrImportJob(ctx, backup, snapshotTarget); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

if !IsJobConditionTrue(job.Status.Conditions, batchv1.JobComplete) {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

snapshot, err := ceph.FindRBDSnapshot(
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/mantlerestore_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (r *MantleRestoreReconciler) restore(ctx context.Context, restore *mantlev1
// check if the backup is ReadyToUse
if !meta.IsStatusConditionTrue(backup.Status.Conditions, mantlev1.BackupConditionReadyToUse) {
logger.Info("backup is not ready to use", "backup", backup.Name, "namespace", backup.Namespace)
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

// store the pool name in the status
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/persistentvolume_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (r *PersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Req

// Wait until the PV's status becomes Released.
if pv.Status.Phase != corev1.VolumeReleased {
return ctrl.Result{Requeue: true}, nil
return requeueReconciliation(), nil
}

// Delete the RBD clone image.
Expand Down
15 changes: 15 additions & 0 deletions internal/controller/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@ import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"

batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
Expand Down Expand Up @@ -81,3 +84,15 @@ func IsJobConditionTrue(conditions []batchv1.JobCondition, conditionType batchv1
}
return false
}

func requeueReconciliation() ctrl.Result {
requeueAfter := os.Getenv("REQUEUE_RECONCILIATION_AFTER")
if requeueAfter != "" {
duration, err := time.ParseDuration(requeueAfter)
if err != nil {
panic(fmt.Sprintf("set REQUEUE_RECONCILIATION_AFTER properly: %v", err))
}
return ctrl.Result{RequeueAfter: duration}
}
return ctrl.Result{Requeue: true}
}
3 changes: 2 additions & 1 deletion test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ LOOP_DEV2 := /dev/loop1
MINIKUBE_PROFILE_PRIMARY=profile1
MINIKUBE_PROFILE_SECONDARY=profile2
TMPDIR := tmp
TEST_MULTIK8S_PACKAGES := replication changetostandalone changetoprimary changetosecondary

export MINIKUBE_HOME

Expand Down Expand Up @@ -240,4 +241,4 @@ do-test-multik8s: $(GINKGO)
E2ETEST=1 \
KUBECTL_PRIMARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_PRIMARY) kubectl -- " \
KUBECTL_SECONDARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_SECONDARY) kubectl -- " \
$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) multik8s
$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) $(addprefix multik8s/, $(TEST_MULTIK8S_PACKAGES))
122 changes: 122 additions & 0 deletions test/e2e/multik8s/changetoprimary/suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package changetoprimary

import (
"context"
"os"
"testing"
"time"

"github.com/cybozu-go/mantle/internal/controller"
. "github.com/cybozu-go/mantle/test/e2e/multik8s/testutil"
"github.com/cybozu-go/mantle/test/util"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestMtest(t *testing.T) {
if os.Getenv("E2ETEST") == "" {
t.Skip("Run under e2e/")
}

RegisterFailHandler(Fail)

SetDefaultEventuallyPollingInterval(time.Second)
SetDefaultEventuallyTimeout(3 * time.Minute)
EnforceDefaultTimeoutsWhenUsingContexts()

RunSpecs(t, "replication test with multiple k8s clusters")
}

var _ = Describe("Mantle", func() {
Context("wait controller to be ready", WaitControllerToBeReady)
Context("change to primary", changePrimaryToStandaloneTemporarily)
})

func changePrimaryToStandaloneTemporarily() {
Describe("change to primary", func() {
var (
namespace string
pvcName0, backupName00, backupName01, writtenDataHash00, writtenDataHash01 string
pvcName1, backupName10, writtenDataHash10 string
)

/*
Overview of the test:
primary k8s cluster | secondary k8s cluster
============================|==========================
role=primary | role=secondary
PVC0, MB00 (created) |
| PVC0, MB00 (synced)
role=standalone (changed) |
MB01, PVC1, MB10 (created)|
role=primary (changed) |
| MB01, PVC1, MB10 (synced)
*/

It("should replicate a MantleBackup resource", func(ctx context.Context) {
namespace = util.GetUniqueName("ns-")
pvcName0 = util.GetUniqueName("pvc-")
backupName00 = util.GetUniqueName("mb-")

SetupEnvironment(namespace)
CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName0)
writtenDataHash00 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName00)
WaitMantleBackupSynced(namespace, backupName00)
})

It("should change the role from primary to standalone", func() {
By("changing the primary mantle to standalone")
err := ChangeClusterRole(PrimaryK8sCluster, controller.RoleStandalone)
Expect(err).NotTo(HaveOccurred())
})

It("should restore the synced MantleBackup in the both clusters", func(ctx context.Context) {
restoreName00 := util.GetUniqueName("mr-")
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
})

It("should create a MantleBackup resource", func(ctx SpecContext) {
backupName01 = util.GetUniqueName("mb-")
writtenDataHash01 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)

CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName01)

pvcName1 = util.GetUniqueName("pvc-")
backupName10 = util.GetUniqueName("mb-")

Eventually(func() error {
return ApplyPVCTemplate(PrimaryK8sCluster, namespace, pvcName1)
}).Should(Succeed())
writtenDataHash10 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName1)
CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName1, backupName10)
})

It("should change the role from standalone to primary", func() {
By("changing the standalone mantle to primary")
err := ChangeClusterRole(PrimaryK8sCluster, controller.RolePrimary)
Expect(err).NotTo(HaveOccurred())
})

It("should synchronize MantleBackups correctly", func() {
WaitMantleBackupSynced(namespace, backupName01)
WaitMantleBackupSynced(namespace, backupName10)
})

It("should restore MantleBackups correctly", func(ctx SpecContext) {
restoreName00 := util.GetUniqueName("mr-")
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)

restoreName01 := util.GetUniqueName("mr-")
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)

restoreName10 := util.GetUniqueName("mr-")
EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
})
})
}
Loading

0 comments on commit 083ea99

Please sign in to comment.