diff --git a/api/v1beta2/mysqlcluster_types.go b/api/v1beta2/mysqlcluster_types.go index 3232f44b8..e5ca500de 100644 --- a/api/v1beta2/mysqlcluster_types.go +++ b/api/v1beta2/mysqlcluster_types.go @@ -624,6 +624,10 @@ type BackupStatus struct { // SourceUUID is the `server_uuid` of the backup source instance. SourceUUID string `json:"sourceUUID"` + // UUIDSet is the `server_uuid` set of all candidate instances for the backup source. + // +optional + UUIDSet map[string]string `json:"uuidSet"` + // BinlogFilename is the binlog filename that the backup source instance was writing to // at the backup. BinlogFilename string `json:"binlogFilename"` diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 6843b116e..66f1afd8b 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -120,6 +120,13 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in in.Time.DeepCopyInto(&out.Time) out.Elapsed = in.Elapsed + if in.UUIDSet != nil { + in, out := &in.UUIDSet, &out.UUIDSet + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Warnings != nil { in, out := &in.Warnings, &out.Warnings *out = make([]string, len(*in)) diff --git a/backup/backup.go b/backup/backup.go index f518b14dd..df3fb047b 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -2,12 +2,14 @@ package backup import ( "context" + "errors" "fmt" "io" "io/fs" "os" "os/exec" "path/filepath" + "sort" "strconv" "strings" "syscall" @@ -45,6 +47,7 @@ type BackupManager struct { startTime time.Time sourceIndex int status bkop.ServerStatus + uuidSet map[string]string gtidSet string dumpSize int64 binlogSize int64 @@ -117,7 +120,13 @@ func (bm *BackupManager) Backup(ctx context.Context) error { orderedPods[index] = &pods.Items[i] } - sourceIndex, err := bm.ChoosePod(ctx, orderedPods) + uuidSet, err := bm.GetUUIDSet(ctx, orderedPods) + if err != nil { + return fmt.Errorf("failed to get server_uuid set: %w", err) + } + bm.uuidSet = uuidSet + + sourceIndex, doBackupBinlog, err := bm.ChoosePod(ctx, orderedPods) if err != nil { return fmt.Errorf("failed to choose source instance: %w", err) } @@ -146,8 +155,7 @@ func (bm *BackupManager) Backup(ctx context.Context) error { } // dump and upload binlog for the second or later backups - lastBackup := &bm.cluster.Status.Backup - if !lastBackup.Time.IsZero() { + if doBackupBinlog { if err := bm.backupBinlog(ctx, op); err != nil { // since the full backup has succeeded, we should continue ev := event.BackupNoBinlog.ToEvent(bm.clusterRef) @@ -172,6 +180,7 @@ func (bm *BackupManager) Backup(ctx context.Context) error { sb.Elapsed = metav1.Duration{Duration: elapsed} sb.SourceIndex = sourceIndex sb.SourceUUID = bm.status.UUID + sb.UUIDSet = bm.uuidSet sb.BinlogFilename = bm.status.CurrentBinlog sb.GTIDSet = bm.gtidSet sb.DumpSize = bm.dumpSize @@ -194,89 +203,87 @@ func (bm *BackupManager) Backup(ctx context.Context) error { return nil } -func (bm *BackupManager) ChoosePod(ctx context.Context, pods []*corev1.Pod) (int, error) { +func (bm *BackupManager) GetUUIDSet(ctx context.Context, pods []*corev1.Pod) (map[string]string, error) { cluster := bm.cluster - // if this is the first time - if cluster.Status.Backup.Time.IsZero() { - if len(pods) == 1 { - return 0, nil - } + uuids := make(map[string]string, len(pods)) + for i := range pods { + if podIsReady(pods[i]) { + op, err := newOperator(cluster.PodHostname(i), + constants.MySQLPort, + constants.BackupUser, + bm.mysqlPassword, + bm.threads) + if err != nil { + return nil, fmt.Errorf("failed to create operator: %w", err) + } + defer op.Close() - for i := range pods { - if i == int(cluster.Status.CurrentPrimaryIndex) { + if err := op.GetServerStatus(ctx, &bm.status); err != nil { continue } - if podIsReady(pods[i]) { - return i, nil - } + uuids[strconv.Itoa(i)] = bm.status.UUID } - return int(cluster.Status.CurrentPrimaryIndex), nil - } - - lastIndex := cluster.Status.Backup.SourceIndex - op, err := newOperator(cluster.PodHostname(lastIndex), - constants.MySQLPort, - constants.BackupUser, - bm.mysqlPassword, - bm.threads) - if err != nil { - return -1, err - } - defer op.Close() - - st := &bkop.ServerStatus{} - if err := op.GetServerStatus(ctx, st); err != nil { - return -1, err } + return uuids, nil +} - if st.UUID != cluster.Status.Backup.SourceUUID { - bm.log.Info("server_uuid of the last backup source has changed", "index", lastIndex) - +// ChoosePod chooses a pod to take a backup from. +// It returns the index of the chosen pod and whether backupBinlog should be called. +func (bm *BackupManager) ChoosePod(ctx context.Context, pods []*corev1.Pod) (int, bool, error) { + currentPrimaryIndex := int(bm.cluster.Status.CurrentPrimaryIndex) + lastBackup := &bm.cluster.Status.Backup + // if this is the first time + if lastBackup.Time.IsZero() { for i := range pods { - if i == lastIndex { - continue - } - if i == int(cluster.Status.CurrentPrimaryIndex) { + if i == currentPrimaryIndex { continue } if podIsReady(pods[i]) { - return i, nil + return i, false, nil } } - return cluster.Status.CurrentPrimaryIndex, nil + if podIsReady(pods[currentPrimaryIndex]) { + return currentPrimaryIndex, false, nil + } else { + return 0, false, errors.New("no ready pod exists") + } } - if !podIsReady(pods[lastIndex]) { - bm.log.Info("the last backup source is not ready", "index", lastIndex) + lastIndex := lastBackup.SourceIndex + choosableIndexes := getIdxsWithUnchangedUUID(bm.uuidSet, lastBackup.UUIDSet) + if len(choosableIndexes) == 0 { + bm.log.Info("the server_uuid of all pods has changed or some pods are not ready") + bm.warnings = append(bm.warnings, "skip binlog backups because some binlog files may be missing") for i := range pods { - if i == lastIndex { - continue - } - if i == int(cluster.Status.CurrentPrimaryIndex) { + if i == currentPrimaryIndex { continue } if podIsReady(pods[i]) { - return i, nil + return i, false, nil } } - return cluster.Status.CurrentPrimaryIndex, nil + if podIsReady(pods[currentPrimaryIndex]) { + return currentPrimaryIndex, false, nil + } else { + return 0, false, errors.New("no ready pod exists") + } } - if lastIndex == int(cluster.Status.CurrentPrimaryIndex) { - bm.log.Info("the last backup source is not a replica", "index", lastIndex) - for i := range pods { - if i == lastIndex { - continue - } - if podIsReady(pods[i]) { - return i, nil - } + replicas := []int{} + for _, i := range choosableIndexes { + if i == currentPrimaryIndex { + continue + } + if i == lastIndex { + return i, true, nil } - return cluster.Status.CurrentPrimaryIndex, nil + replicas = append(replicas, i) } - - return lastIndex, nil + if len(replicas) != 0 { + return replicas[0], true, nil + } + return currentPrimaryIndex, true, nil } func (bm *BackupManager) backupFull(ctx context.Context, op bkop.Operator) error { @@ -361,7 +368,7 @@ func (bm *BackupManager) backupBinlog(ctx context.Context, op bkop.Operator) err } if err := op.DumpBinlog(ctx, binlogDir, binlogName, lastBackup.GTIDSet); err != nil { - return fmt.Errorf("failed to take a binlog backup: %w", err) + return fmt.Errorf("failed to exec mysqlbinlog command: %w", err) } usage, err := dirUsage(binlogDir) @@ -470,3 +477,18 @@ func dirUsage(dir string) (int64, error) { return usage, nil } + +func getIdxsWithUnchangedUUID(current, last map[string]string) []int { + idxs := []int{} + for key, currentUUID := range current { + if lastUUID, ok := last[key]; ok && currentUUID == lastUUID { + i, err := strconv.Atoi(key) + if err != nil { + continue + } + idxs = append(idxs, i) + } + } + sort.Ints(idxs) + return idxs +} diff --git a/backup/backup_test.go b/backup/backup_test.go index 4d6ef2ab6..5dff432d0 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -2,6 +2,9 @@ package backup import ( "context" + "errors" + "reflect" + "strconv" "testing" "time" @@ -12,94 +15,247 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type choosePodMockOp struct { +type getUUIDSetMockOp struct { closed bool uuid string } -func (o *choosePodMockOp) Ping() error { +func (o *getUUIDSetMockOp) Ping() error { panic("not implemented") } -func (o *choosePodMockOp) Close() { +func (o *getUUIDSetMockOp) Close() { o.closed = true } -func (o *choosePodMockOp) GetServerStatus(_ context.Context, st *bkop.ServerStatus) error { +func (o *getUUIDSetMockOp) GetServerStatus(_ context.Context, st *bkop.ServerStatus) error { st.UUID = o.uuid return nil } -func (o *choosePodMockOp) DumpFull(ctx context.Context, dir string) error { +func (o *getUUIDSetMockOp) DumpFull(ctx context.Context, dir string) error { panic("not implemented") } -func (o *choosePodMockOp) GetBinlogs(_ context.Context) ([]string, error) { +func (o *getUUIDSetMockOp) GetBinlogs(_ context.Context) ([]string, error) { panic("not implemented") } -func (o *choosePodMockOp) DumpBinlog(ctx context.Context, dir string, binlogName string, filterGTID string) error { +func (o *getUUIDSetMockOp) DumpBinlog(ctx context.Context, dir string, binlogName string, filterGTID string) error { panic("not implemented") } -func (o *choosePodMockOp) PrepareRestore(_ context.Context) error { +func (o *getUUIDSetMockOp) PrepareRestore(_ context.Context) error { panic("not implemented") } -func (o *choosePodMockOp) LoadDump(ctx context.Context, dir string) error { +func (o *getUUIDSetMockOp) LoadDump(ctx context.Context, dir string) error { panic("not implemented") } -func (o *choosePodMockOp) LoadBinlog(ctx context.Context, binlogDir, tmpDir string, restorePoint time.Time) error { +func (o *getUUIDSetMockOp) LoadBinlog(ctx context.Context, binlogDir, tmpDir string, restorePoint time.Time) error { panic("not implemented") } -func (o *choosePodMockOp) FinishRestore(_ context.Context) error { +func (o *getUUIDSetMockOp) FinishRestore(_ context.Context) error { panic("not implemented") } -func TestChoosePod(t *testing.T) { - makePod := func(ready bool) *corev1.Pod { - pod := &corev1.Pod{} - if !ready { - return pod - } - pod.Status.Conditions = []corev1.PodCondition{{ - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }} +func makePod(ready bool) *corev1.Pod { + pod := &corev1.Pod{} + if !ready { return pod } + pod.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }} + return pod +} - makePod1 := func(ready bool) []*corev1.Pod { - return []*corev1.Pod{makePod(ready)} - } - makePod3 := func(ready0, ready1, ready2 bool) []*corev1.Pod { - return []*corev1.Pod{makePod(ready0), makePod(ready1), makePod(ready2)} - } +func makePod1(ready bool) []*corev1.Pod { + return []*corev1.Pod{makePod(ready)} +} + +func makePod3(ready0, ready1, ready2 bool) []*corev1.Pod { + return []*corev1.Pod{makePod(ready0), makePod(ready1), makePod(ready2)} +} - var lastOp *choosePodMockOp +func TestGetUUIDSet(t *testing.T) { + var lastOp *getUUIDSetMockOp newOperator = func(host string, port int, user, password string, threads int) (bkop.Operator, error) { - lastOp = &choosePodMockOp{uuid: "123"} + lastOp = &getUUIDSetMockOp{uuid: "123"} return lastOp, nil } - makeBM := func(replicas, current int, bkup mocov1beta2.BackupStatus) *BackupManager { + testCases := []struct { + name string + pods []*corev1.Pod + + expected map[string]string + }{ + { + name: "empty", + pods: []*corev1.Pod{}, + expected: map[string]string{}, + }, + { + name: "single-not-ready", + pods: makePod1(false), + expected: map[string]string{}, + }, + { + name: "single-ready", + pods: makePod1(true), + expected: map[string]string{"0": "123"}, + }, + { + name: "triple-not-ready", + pods: makePod3(false, false, false), + expected: map[string]string{}, + }, + { + name: "triple-1st-ready", + pods: makePod3(true, false, false), + expected: map[string]string{"0": "123"}, + }, + { + name: "triple-2nd-ready", + pods: makePod3(false, true, false), + expected: map[string]string{"1": "123"}, + }, + { + name: "triple-3rd-ready", + pods: makePod3(false, false, true), + expected: map[string]string{"2": "123"}, + }, + { + name: "triple-all-ready", + pods: makePod3(true, true, true), + expected: map[string]string{"0": "123", "1": "123", "2": "123"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + lastOp = nil + bm := &BackupManager{ + cluster: &mocov1beta2.MySQLCluster{}, + status: bkop.ServerStatus{ + UUID: "123", + }, + } + uuids, err := bm.GetUUIDSet(context.Background(), tc.pods) + if lastOp != nil && !lastOp.closed { + t.Error("op was not closed") + } + if err != nil { + t.Error("unexpected error", err) + } + if !reflect.DeepEqual(uuids, tc.expected) { + t.Errorf("unexpected uuids %v, expected %v", uuids, tc.expected) + } + }) + + } +} + +func TestGetIdxsWithUnchangedUUID(t *testing.T) { + testCases := []struct { + name string + current map[string]string + last map[string]string + + expectIdxs []int + }{ + { + name: "single-uuid-not-changed", + current: map[string]string{"0": "uuid-0"}, + last: map[string]string{"0": "uuid-0"}, + expectIdxs: []int{0}, + }, + { + name: "single-uuid-changed", + current: map[string]string{"0": "uuid-0"}, + last: map[string]string{"0": "uuid-a"}, + expectIdxs: []int{}, + }, + { + name: "triple-uuid-not-changed", + current: map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}, + last: map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}, + expectIdxs: []int{0, 1, 2}, + }, + { + name: "triple-some-uuid-changed", + current: map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}, + last: map[string]string{"0": "uuid-0", "1": "uuid-a", "2": "uuid-2"}, + expectIdxs: []int{0, 2}, + }, + { + name: "triple-all-uuid-changed", + current: map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}, + last: map[string]string{"0": "uuid-a", "1": "uuid-b", "2": "uuid-c"}, + expectIdxs: []int{}, + }, + { + name: "triple-some-uuid-changed-or-not-exist-1", + current: map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}, + last: map[string]string{"0": "uuid-a", "2": "uuid-2"}, + expectIdxs: []int{2}, + }, + { + name: "triple-some-uuid-changed-or-not-exist-2", + current: map[string]string{"0": "uuid-0", "2": "uuid-2"}, + last: map[string]string{"0": "uuid-a", "1": "uuid-1", "2": "uuid-2"}, + expectIdxs: []int{2}, + }, + { + name: "triple-with-invalid-index", + current: map[string]string{"0": "uuid-0", "1": "uuid-1", "hoge": "uuid-2"}, + last: map[string]string{"0": "uuid-0", "fuga": "uuid-1", "2": "uuid-2"}, + expectIdxs: []int{0}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + idxs := getIdxsWithUnchangedUUID(tc.current, tc.last) + if !reflect.DeepEqual(idxs, tc.expectIdxs) { + t.Errorf("unexpected indexes %v, expected %v", idxs, tc.expectIdxs) + } + }) + } +} + +func TestChoosePod(t *testing.T) { + makeBM := func(replicas, current int, bkup mocov1beta2.BackupStatus, pods []*corev1.Pod) *BackupManager { cluster := &mocov1beta2.MySQLCluster{} cluster.Spec.Replicas = int32(replicas) cluster.Status.CurrentPrimaryIndex = current cluster.Status.Backup = bkup + uuidSet := make(map[string]string) + for i, pod := range pods { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + uuidSet[strconv.Itoa(i)] = "uuid-" + strconv.Itoa(i) + } + } + } + return &BackupManager{ log: logr.Discard(), cluster: cluster, + uuidSet: uuidSet, } } - makeBS := func(idx int, uuid string) mocov1beta2.BackupStatus { + makeBS := func(sourceIdx int, uuidSet map[string]string) mocov1beta2.BackupStatus { return mocov1beta2.BackupStatus{ Time: metav1.Now(), - SourceIndex: idx, - SourceUUID: uuid, + SourceIndex: sourceIdx, + SourceUUID: uuidSet[strconv.Itoa(sourceIdx)], + UUIDSet: uuidSet, } } @@ -110,38 +266,208 @@ func TestChoosePod(t *testing.T) { bkup mocov1beta2.BackupStatus pods []*corev1.Pod - expectIdx int + err error + expectIdx int + doBackupBinlog bool + warnings int }{ - {"single", 1, 0, mocov1beta2.BackupStatus{}, makePod1(true), 0}, - {"single-not-ready", 1, 0, mocov1beta2.BackupStatus{}, makePod1(false), 0}, - {"triple-ready", 3, 0, mocov1beta2.BackupStatus{}, makePod3(true, false, true), 2}, - {"triple-not-ready", 3, 1, mocov1beta2.BackupStatus{}, makePod3(false, true, false), 1}, - {"single-2nd", 1, 0, makeBS(0, "123"), makePod1(true), 0}, - {"single-2nd-uuid-changed", 1, 0, makeBS(0, "abc"), makePod1(true), 0}, - {"single-2nd-not-ready", 1, 0, makeBS(0, "123"), makePod1(false), 0}, - {"triple-2nd", 3, 0, makeBS(1, "123"), makePod3(true, true, true), 1}, - {"triple-2nd-uuid-changed", 3, 0, makeBS(1, "abc"), makePod3(true, true, true), 2}, - {"triple-2nd-not-ready", 3, 0, makeBS(1, "123"), makePod3(true, false, true), 2}, - {"triple-2nd-primary", 3, 1, makeBS(1, "123"), makePod3(true, true, true), 0}, - {"triple-2nd-all-not-ready", 3, 0, makeBS(1, "123"), makePod3(true, false, false), 0}, + { + name: "single", + replicas: 1, + current: 0, + bkup: mocov1beta2.BackupStatus{}, + pods: makePod1(true), + err: nil, + expectIdx: 0, + doBackupBinlog: false, + warnings: 0, + }, + { + name: "single-not-ready", + replicas: 1, + current: 0, + bkup: mocov1beta2.BackupStatus{}, + pods: makePod1(false), + err: errors.New("no ready pod exists"), + expectIdx: 0, + doBackupBinlog: false, + warnings: 0, + }, + { + name: "triple-ready", + replicas: 3, + current: 0, + bkup: mocov1beta2.BackupStatus{}, + pods: makePod3(true, false, true), + err: nil, + expectIdx: 2, + doBackupBinlog: false, + warnings: 0, + }, + { + name: "triple-some-not-ready", + replicas: 3, + current: 1, + bkup: mocov1beta2.BackupStatus{}, + pods: makePod3(false, true, false), + err: nil, + expectIdx: 1, + doBackupBinlog: false, + warnings: 0, + }, + { + name: "triple-all-not-ready", + replicas: 3, + current: 1, + bkup: mocov1beta2.BackupStatus{}, + pods: makePod3(false, false, false), + err: errors.New("no ready pod exists"), + expectIdx: 0, + doBackupBinlog: false, + warnings: 0, + }, + { + name: "single-2nd", + replicas: 1, + current: 0, + bkup: makeBS(0, map[string]string{"0": "uuid-0"}), + pods: makePod1(true), + err: nil, + expectIdx: 0, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "single-2nd-uuid-changed", + replicas: 1, + current: 0, + bkup: makeBS(0, map[string]string{"0": "uuid-a"}), + pods: makePod1(true), + err: nil, + expectIdx: 0, + doBackupBinlog: false, + warnings: 1, + }, + { + name: "single-2nd-not-ready", + replicas: 1, + current: 0, + bkup: makeBS(0, map[string]string{"0": "uuid-0"}), + pods: makePod1(false), + err: errors.New("no ready pod exists"), + expectIdx: 0, + doBackupBinlog: false, + warnings: 1, + }, + { + name: "triple-2nd", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}), + pods: makePod3(true, true, true), + err: nil, + expectIdx: 1, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "triple-2nd-uuid-changed", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-b", "2": "uuid-2"}), + pods: makePod3(true, true, true), + err: nil, + expectIdx: 2, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "triple-2nd-last-index-not-ready", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}), + pods: makePod3(true, false, true), + err: nil, + expectIdx: 2, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "triple-2nd-primary", + replicas: 3, + current: 1, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}), + pods: makePod3(true, true, true), + err: nil, + expectIdx: 0, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "triple-2nd-all-not-ready", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-1", "2": "uuid-2"}), + pods: makePod3(false, false, false), + err: errors.New("no ready pod exists"), + expectIdx: 0, + doBackupBinlog: false, + warnings: 1, + }, + { + name: "triple-2nd-replica-uuid-changed", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-0", "1": "uuid-a", "2": "uuid-b"}), + pods: makePod3(true, true, true), + err: nil, + expectIdx: 0, + doBackupBinlog: true, + warnings: 0, + }, + { + name: "triple-2nd-some-not-ready-and-uuid-changed-1", + replicas: 3, + current: 0, + bkup: makeBS(1, map[string]string{"0": "uuid-a", "1": "uuid-1", "2": "uuid-b"}), + pods: makePod3(true, false, true), + err: nil, + expectIdx: 2, + doBackupBinlog: false, + warnings: 1, + }, + { + name: "triple-2nd-some-not-ready-and-uuid-changed-2", + replicas: 3, + current: 0, + bkup: makeBS(0, map[string]string{"0": "uuid-a", "1": "uuid-1", "2": "uuid-b"}), + pods: makePod3(true, false, false), + err: nil, + expectIdx: 0, + doBackupBinlog: false, + warnings: 1, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - lastOp = nil - bm := makeBM(tc.replicas, tc.current, tc.bkup) - idx, err := bm.ChoosePod(context.Background(), tc.pods) - if lastOp != nil && !lastOp.closed { - t.Error("op was not closed") - } + bm := makeBM(tc.replicas, tc.current, tc.bkup, tc.pods) + idx, doBackupBinlog, err := bm.ChoosePod(context.Background(), tc.pods) if err != nil { - t.Error("unexpected error", err) - return + if errors.Is(err, tc.err) { + t.Error("unexpected error", err) + return + } } - if idx != tc.expectIdx { t.Errorf("unexpected index %d, expected %d", idx, tc.expectIdx) } + if doBackupBinlog != tc.doBackupBinlog { + t.Errorf("unexpected doBackupBinlog %v, expected %v", doBackupBinlog, tc.doBackupBinlog) + } + if len(bm.warnings) != tc.warnings { + t.Errorf("unexpected warnings %d, expected %d", len(bm.warnings), tc.warnings) + } }) } } diff --git a/backup/integration_test.go b/backup/integration_test.go index 6d1e7bff3..d113b0042 100644 --- a/backup/integration_test.go +++ b/backup/integration_test.go @@ -142,6 +142,8 @@ var _ = Describe("Backup/Restore", func() { Expect(bs.Elapsed.Seconds()).To(BeNumerically(">", 0)) Expect(bs.SourceIndex).To(Equal(1)) Expect(bs.SourceUUID).To(Equal("123")) + uuidSet := map[string]string{"0": "123", "1": "123", "2": "123"} + Expect(bs.UUIDSet).To(Equal(uuidSet)) Expect(bs.BinlogFilename).To(Equal("binlog.000001")) Expect(bs.GTIDSet).To(Equal("gtid1")) Expect(bs.DumpSize).To(BeNumerically(">", 0)) @@ -229,6 +231,8 @@ var _ = Describe("Backup/Restore", func() { Expect(bs.Time.IsZero()).To(BeFalse()) Expect(bs.SourceIndex).To(Equal(1)) Expect(bs.SourceUUID).To(Equal("123")) + uuidSet := map[string]string{"0": "123", "1": "123", "2": "123"} + Expect(bs.UUIDSet).To(Equal(uuidSet)) Expect(bs.BinlogFilename).To(Equal("binlog.000002")) Expect(bs.GTIDSet).To(Equal("gtid2")) Expect(bs.DumpSize).To(BeNumerically(">", 0)) diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index eca33fc81..b2c504310 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -7753,6 +7753,11 @@ spec: format: date-time nullable: true type: string + uuidSet: + additionalProperties: + type: string + description: 'UUIDSet is the `server_uuid` set of all candidate ' + type: object warnings: description: Warnings are list of warnings from the last backup items: diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index 26b7a486a..404e62087 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -6224,6 +6224,11 @@ spec: format: date-time nullable: true type: string + uuidSet: + additionalProperties: + type: string + description: 'UUIDSet is the `server_uuid` set of all candidate ' + type: object warnings: description: Warnings are list of warnings from the last backup items: diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index f2bc713fd..cb526ca07 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -6224,6 +6224,11 @@ spec: format: date-time nullable: true type: string + uuidSet: + additionalProperties: + type: string + description: 'UUIDSet is the `server_uuid` set of all candidate ' + type: object warnings: description: Warnings are list of warnings from the last backup items: diff --git a/docs/backup.md b/docs/backup.md index b78ab1eaa..fee7d9ed5 100644 --- a/docs/backup.md +++ b/docs/backup.md @@ -145,6 +145,51 @@ Finally, the Job updates MySQLCluster status field with the following informatio - The maximum usage of the working directory - Warnings, if any +When executing an incremental backup, the backup source must be a pod whose server_uuid has not changed since the last backup. +If the server_uuid has changed, the pod may be missing some of the binlogs generated since the last backup. + +The following is how to choose a pod to be the backup source. + +```mermaid +flowchart TD +A{"first time?"} +A -->|"yes"| B +A -->|"no"| C["x ← Get the indexes of the pod whose server_uuid has not changed"] --> D + +B{Are replicas available?} +B -->|"yes"| B1["return\nreplicaIdx\ndoBackupBinlog=false"] +style B1 fill:#c1ffff +B -->|"no"| B2["return\nprimaryIdx\ndoBackupBinlog=false"] +style B2 fill:#ffffc1 + +D{"Is x empty?"} +D -->|"yes"| E["add warning to bm.warnings"] --> F +style E fill:#ffc1c1 +D -->|"no"| G + +F{"Are replicas available?"} +F -->|"yes"| F1["return\nreplicaIdx\ndoBackupBinlog=false"] +style F1 fill:#ffc1c1 +F -->|"no"| F2["return\nprimaryIdx\ndoBackupBinlog=false"] +style F2 fill:#ffc1c1 + +G{"Are there replica indexes in x?"} +G -->|"yes"| H +G -->|"no"| G1["return\nprimaryIdx\ndoBackupBinlog=true"] +style G1 fill:#ffffc1 + +H{"Is lastIndex included in x?"} +H -->|"yes"| I +H -->|"no"| H1["return\nreplicaIdx\ndoBackupBinlog=true"] +style H1 fill:#c1ffff + +I{"Is lastIndex primary?"} +I -->|"yes"| I1["return\nreplicaIdx\ndoBackupBinlog=true"] +style I1 fill:#c1ffff +I -->|"no"| I2["return\nlastIdx\ndoBackupBinlog=true"] +style I2 fill:#c1ffff +``` + ### Restore To restore MySQL data from a backup, users need to create a new MySQLCluster with appropriate `spec.restore` field. diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index 5b88b1a28..7fef139fd 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -29,6 +29,7 @@ BackupStatus represents the status of the last successful backup. | elapsed | Elapsed is the time spent on the backup. | [metav1.Duration](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration) | true | | sourceIndex | SourceIndex is the ordinal of the backup source instance. | int | true | | sourceUUID | SourceUUID is the `server_uuid` of the backup source instance. | string | true | +| uuidSet | UUIDSet is the `server_uuid` set of all candidate instances for the backup source. | map[string]string | true | | binlogFilename | BinlogFilename is the binlog filename that the backup source instance was writing to at the backup. | string | true | | gtidSet | GTIDSet is the GTID set of the full dump of database. | string | true | | dumpSize | DumpSize is the size in bytes of a full dump of database stored in an object storage bucket. | int64 | true |