From 0b12d82d207f5f2fb0923f5591c73fab328e4187 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 19 May 2024 23:44:18 -0400 Subject: [PATCH 01/34] Workflow Delete: remove workflow before artifacts Otherwise the cancel/delete and cleanup work can fail. Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 74 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 17b01736a77..7af3a3982ea 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1616,7 +1616,7 @@ func (s *Server) MoveTablesComplete(ctx context.Context, req *vtctldatapb.MoveTa span, ctx := trace.NewSpan(ctx, "workflow.Server.MoveTablesComplete") defer span.Finish() - ts, state, err := s.getWorkflowState(ctx, req.TargetKeyspace, req.Workflow) + ts, state, err := s.getWorkflowState(ctx, req.GetTargetKeyspace(), req.GetWorkflow()) if err != nil { return nil, err } @@ -1630,8 +1630,7 @@ func (s *Server) MoveTablesComplete(ctx context.Context, req *vtctldatapb.MoveTa var dryRunResults *[]string if state.WorkflowType == TypeMigrate { - dryRunResults, err = s.finalizeMigrateWorkflow(ctx, req.TargetKeyspace, req.Workflow, strings.Join(ts.tables, ","), - false, req.KeepData, req.KeepRoutingRules, req.DryRun) + dryRunResults, err = s.finalizeMigrateWorkflow(ctx, ts, strings.Join(ts.tables, ","), false, req.KeepData, req.KeepRoutingRules, req.DryRun) if err != nil { return nil, vterrors.Wrapf(err, "failed to finalize the %s workflow in the %s keyspace", req.Workflow, req.TargetKeyspace) @@ -1970,11 +1969,24 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe span.Annotate("keep_routing_rules", req.KeepRoutingRules) span.Annotate("shards", req.Shards) - // Cleanup related data and artifacts. - if _, err := s.DropTargets(ctx, req.Keyspace, req.Workflow, req.KeepData, req.KeepRoutingRules, false); err != nil { - if topo.IsErrType(err, topo.NoNode) { - return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.Keyspace) - } + ts, state, err := s.getWorkflowState(ctx, req.GetKeyspace(), req.GetWorkflow()) + if err != nil { + log.Errorf("Failed to get VReplication workflow state for %s.%s: %v", req.GetKeyspace(), req.GetWorkflow(), err) + return nil, err + } + + // There is nothing to drop for a LookupVindex workflow. + if ts.workflowType == binlogdatapb.VReplicationWorkflowType_CreateLookupIndex { + return nil, nil + } + + // Return an error if the workflow traffic is partially switched. + if state.WritesSwitched || len(state.ReplicaCellsSwitched) > 0 || len(state.RdonlyCellsSwitched) > 0 { + return nil, ErrWorkflowPartiallySwitched + } + + if state.WorkflowType == TypeMigrate { + _, err := s.finalizeMigrateWorkflow(ctx, ts, "", true, req.GetKeepData(), req.GetKeepRoutingRules(), false) return nil, err } @@ -2002,6 +2014,14 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "the %s workflow does not exist in the %s keyspace", req.Workflow, req.Keyspace) } + // Cleanup related data and artifacts. + if _, err := s.DropTargets(ctx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { + if topo.IsErrType(err, topo.NoNode) { + return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.Keyspace) + } + return nil, err + } + response := &vtctldatapb.WorkflowDeleteResponse{} response.Summary = fmt.Sprintf("Successfully cancelled the %s workflow in the %s keyspace", req.Workflow, req.Keyspace) details := make([]*vtctldatapb.WorkflowDeleteResponse_TabletInfo, 0, len(res)) @@ -2497,28 +2517,8 @@ func (s *Server) optimizeCopyStateTable(tablet *topodatapb.Tablet) { // DropTargets cleans up target tables, shards and denied tables if a MoveTables/Reshard // is cancelled. -func (s *Server) DropTargets(ctx context.Context, targetKeyspace, workflow string, keepData, keepRoutingRules, dryRun bool) (*[]string, error) { - ts, state, err := s.getWorkflowState(ctx, targetKeyspace, workflow) - if err != nil { - log.Errorf("Failed to get VReplication workflow state for %s.%s: %v", targetKeyspace, workflow, err) - return nil, err - } - - // There is nothing to drop for a LookupVindex workflow. - if ts.workflowType == binlogdatapb.VReplicationWorkflowType_CreateLookupIndex { - return nil, nil - } - - // Return an error if the workflow traffic is partially switched. - if state.WritesSwitched || len(state.ReplicaCellsSwitched) > 0 || len(state.RdonlyCellsSwitched) > 0 { - return nil, ErrWorkflowPartiallySwitched - } - - if state.WorkflowType == TypeMigrate { - _, err := s.finalizeMigrateWorkflow(ctx, targetKeyspace, workflow, "", true, keepData, keepRoutingRules, dryRun) - return nil, err - } - +func (s *Server) DropTargets(ctx context.Context, ts *trafficSwitcher, keepData, keepRoutingRules, dryRun bool) (*[]string, error) { + var err error ts.keepRoutingRules = keepRoutingRules var sw iswitcher if dryRun { @@ -2942,13 +2942,11 @@ func (s *Server) refreshPrimaryTablets(ctx context.Context, shards []*topo.Shard // finalizeMigrateWorkflow deletes the streams for the Migrate workflow. // We only cleanup the target for external sources. -func (s *Server) finalizeMigrateWorkflow(ctx context.Context, targetKeyspace, workflow, tableSpecs string, cancel, keepData, keepRoutingRules, dryRun bool) (*[]string, error) { - ts, err := s.buildTrafficSwitcher(ctx, targetKeyspace, workflow) - if err != nil { - ts.Logger().Errorf("buildTrafficSwitcher failed: %v", err) - return nil, err - } - var sw iswitcher +func (s *Server) finalizeMigrateWorkflow(ctx context.Context, ts *trafficSwitcher, tableSpecs string, cancel, keepData, keepRoutingRules, dryRun bool) (*[]string, error) { + var ( + sw iswitcher + err error + ) if dryRun { sw = &switcherDryRun{ts: ts, drLog: NewLogRecorder()} } else { @@ -2966,7 +2964,7 @@ func (s *Server) finalizeMigrateWorkflow(ctx context.Context, targetKeyspace, wo return nil, err } if !cancel { - if err := sw.addParticipatingTablesToKeyspace(ctx, targetKeyspace, tableSpecs); err != nil { + if err := sw.addParticipatingTablesToKeyspace(ctx, ts.targetKeyspace, tableSpecs); err != nil { return nil, err } if err := ts.TopoServer().RebuildSrvVSchema(ctx, nil); err != nil { From 4401a21a3d3ac09d0dc23deb5295dfe935b74471 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 20 May 2024 19:02:16 -0400 Subject: [PATCH 02/34] Add unit test Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server_test.go | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 174cc2aaf6a..b66e0283203 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -199,3 +199,41 @@ func TestVDiffCreate(t *testing.T) { }) } } + +func TestWorkflowDelete(t *testing.T) { + ctx := context.Background() + ksName := "ks1" + wfName := "wf1" + cellName := "cell1" + + ts := memorytopo.NewServer(ctx, cellName) + tmc := &fakeTMC{} + s := NewServer(vtenv.NewTestEnv(), ts, tmc) + err := s.ts.CreateKeyspace(ctx, ksName, &topodatapb.Keyspace{}) + require.NoError(t, err) + + testcases := []struct { + name string + req *vtctldatapb.WorkflowDeleteRequest + want *vtctldatapb.WorkflowDeleteResponse + wantErr bool + }{ + { + name: "no values", + req: &vtctldatapb.WorkflowDeleteRequest{ + Keyspace: ksName, + Workflow: wfName, + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := s.WorkflowDelete(ctx, tc.req) + if (err != nil) != tc.wantErr { + t.Errorf("Server.WorkflowDelete() error = %v, wantErr %v", err, tc.wantErr) + return + } + require.Equal(t, got, tc.want, "Server.WorkflowDelete() = %v, want %v", got, tc.want) + }) + } +} From 51b75c16257dc4113098340114693c8a62c6bb55 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 21 May 2024 10:04:13 -0400 Subject: [PATCH 03/34] Add timeouts to vx.CallbackContext usage Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 10 +++++++--- go/vt/vtctl/workflow/server_test.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 7af3a3982ea..00c36b2d3dc 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -2005,7 +2005,9 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe s.optimizeCopyStateTable(tablet.Tablet) return res.Result, err } - res, err := vx.CallbackContext(ctx, callback) + delCtx, delCancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout*2) + defer delCancel() + res, err := vx.CallbackContext(delCtx, callback) if err != nil { return nil, err } @@ -2015,7 +2017,7 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe } // Cleanup related data and artifacts. - if _, err := s.DropTargets(ctx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { + if _, err := s.DropTargets(delCtx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { if topo.IsErrType(err, topo.NoNode) { return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.Keyspace) } @@ -2289,7 +2291,9 @@ func (s *Server) WorkflowUpdate(ctx context.Context, req *vtctldatapb.WorkflowUp } return res.Result, err } - res, err := vx.CallbackContext(ctx, callback) + updCtx, updCancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout*2) + defer updCancel() + res, err := vx.CallbackContext(updCtx, callback) if err != nil { if topo.IsErrType(err, topo.NoNode) { return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.Keyspace) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index b66e0283203..a6a38fd3040 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -211,6 +211,18 @@ func TestWorkflowDelete(t *testing.T) { s := NewServer(vtenv.NewTestEnv(), ts, tmc) err := s.ts.CreateKeyspace(ctx, ksName, &topodatapb.Keyspace{}) require.NoError(t, err) + err = s.ts.CreateShard(ctx, ksName, "0") + require.NoError(t, err) + err = s.ts.CreateTablet(ctx, &topodatapb.Tablet{ + Hostname: "host1", + Keyspace: ksName, + Shard: "0", + Alias: &topodatapb.TabletAlias{ + Cell: cellName, + Uid: 100, + }, + }) + require.NoError(t, err) testcases := []struct { name string @@ -228,6 +240,13 @@ func TestWorkflowDelete(t *testing.T) { } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { + res, err := s.MoveTablesCreate(ctx, &vtctldatapb.MoveTablesCreateRequest{ + TargetKeyspace: ksName, + Workflow: wfName, + AllTables: true, + }) + require.NoError(t, err) + require.NotNil(t, res) got, err := s.WorkflowDelete(ctx, tc.req) if (err != nil) != tc.wantErr { t.Errorf("Server.WorkflowDelete() error = %v, wantErr %v", err, tc.wantErr) From 82005fb0f8099b2e76cb7b4f3860e13fbf378e00 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 08:30:18 -0400 Subject: [PATCH 04/34] Don't fail workflow cancel/cleanup if there's nothing to do Signed-off-by: Matt Lord --- go/vt/topo/shard.go | 3 +- go/vt/vtctl/workflow/server.go | 8 ++--- go/vt/vtctl/workflow/traffic_switcher.go | 37 +++++++++++++++++------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index b9554bf789f..827b531e6a5 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -451,7 +451,8 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr } if remove { if len(newTables) != 0 { - return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, dlTablesNotPresent) + // These tables did not exist in the denied list so we don't need to remove them. + log.Warningf("%s:%s", dlTablesNotPresent, strings.Join(newTables, ",")) } var newDenyList []string if len(tables) != 0 { // legacy uses diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 00c36b2d3dc..ebd4b4b2ec6 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1971,7 +1971,7 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe ts, state, err := s.getWorkflowState(ctx, req.GetKeyspace(), req.GetWorkflow()) if err != nil { - log.Errorf("Failed to get VReplication workflow state for %s.%s: %v", req.GetKeyspace(), req.GetWorkflow(), err) + log.Errorf("failed to get VReplication workflow state for %s.%s: %v", req.GetKeyspace(), req.GetWorkflow(), err) return nil, err } @@ -2019,7 +2019,7 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe // Cleanup related data and artifacts. if _, err := s.DropTargets(delCtx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { if topo.IsErrType(err, topo.NoNode) { - return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.Keyspace) + return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.GetKeyspace()) } return nil, err } @@ -2807,8 +2807,8 @@ func (s *Server) DeleteShard(ctx context.Context, keyspace, shard string, recurs shardInfo, err := s.ts.GetShard(ctx, keyspace, shard) if err != nil { if topo.IsErrType(err, topo.NoNode) { - log.Infof("Shard %v/%v doesn't seem to exist, cleaning up any potential leftover", keyspace, shard) - return s.ts.DeleteShard(ctx, keyspace, shard) + log.Warningf("Shard %v/%v did not exist when attempting to remove it", keyspace, shard) + return nil } return err } diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index 9ea1c8b609b..15c5439aa9c 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -26,12 +26,11 @@ import ( "sync" "time" - vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" - "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/sqlescape" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" @@ -53,6 +52,7 @@ import ( tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -428,6 +428,10 @@ func (ts *trafficSwitcher) deleteShardRoutingRules(ctx context.Context) error { } srr, err := topotools.GetShardRoutingRules(ctx, ts.TopoServer()) if err != nil { + if topo.IsErrType(err, topo.NoNode) { + log.Warningf("No shard routing rules found when attempting to delete the ones for the %s keyspace", ts.targetKeyspace) + return nil + } return err } for _, si := range ts.TargetShards() { @@ -520,14 +524,14 @@ func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType T query := fmt.Sprintf("drop table %s.%s", primaryDbName, tableNameEscaped) if removalType == DropTable { ts.Logger().Infof("%s: Dropping table %s.%s\n", - source.GetPrimary().String(), source.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName) } else { renameName, err := sqlescape.EnsureEscaped(getRenameFileName(tableName)) if err != nil { return err } ts.Logger().Infof("%s: Renaming table %s.%s to %s.%s\n", - source.GetPrimary().String(), source.GetPrimary().DbName(), tableName, source.GetPrimary().DbName(), renameName) + topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName, source.GetPrimary().DbName(), renameName) query = fmt.Sprintf("rename table %s.%s TO %s.%s", primaryDbName, tableNameEscaped, primaryDbName, renameName) } _, err = ts.ws.tmc.ExecuteFetchAsDba(ctx, source.GetPrimary().Tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ @@ -537,10 +541,14 @@ func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType T DisableForeignKeyChecks: true, }) if err != nil { - ts.Logger().Errorf("%s: Error removing table %s: %v", source.GetPrimary().String(), tableName, err) + if mysqlErr, ok := err.(*sqlerror.SQLError); ok && mysqlErr.Num == sqlerror.ERNoSuchTable { + ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(source.GetPrimary().Alias), tableName) + return nil + } + ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(source.GetPrimary().Alias), tableName, err) return err } - ts.Logger().Infof("%s: Removed table %s.%s\n", source.GetPrimary().String(), source.GetPrimary().DbName(), tableName) + ts.Logger().Infof("%s: Removed table %s.%s\n", topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName) } return nil @@ -833,6 +841,7 @@ func (ts *trafficSwitcher) deleteReverseVReplication(ctx context.Context) error return ts.ForAllSources(func(source *MigrationSource) error { query := fmt.Sprintf(sqlDeleteWorkflow, encodeString(source.GetPrimary().DbName()), encodeString(ts.reverseWorkflow)) if _, err := ts.TabletManagerClient().VReplicationExec(ctx, source.GetPrimary().Tablet, query); err != nil { + // vreplication.exec returns no error on delete if the rows do not exist. return err } ts.ws.deleteWorkflowVDiffData(ctx, source.GetPrimary().Tablet, ts.reverseWorkflow) @@ -1112,6 +1121,7 @@ func (ts *trafficSwitcher) dropTargetVReplicationStreams(ctx context.Context) er ts.Logger().Infof("Deleting target streams and related data for workflow %s db_name %s", ts.WorkflowName(), target.GetPrimary().DbName()) query := fmt.Sprintf(sqlDeleteWorkflow, encodeString(target.GetPrimary().DbName()), encodeString(ts.WorkflowName())) if _, err := ts.TabletManagerClient().VReplicationExec(ctx, target.GetPrimary().Tablet, query); err != nil { + // vreplication.exec returns no error on delete if the rows do not exist. return err } ts.ws.deleteWorkflowVDiffData(ctx, target.GetPrimary().Tablet, ts.WorkflowName()) @@ -1125,6 +1135,7 @@ func (ts *trafficSwitcher) dropSourceReverseVReplicationStreams(ctx context.Cont ts.Logger().Infof("Deleting reverse streams and related data for workflow %s db_name %s", ts.WorkflowName(), source.GetPrimary().DbName()) query := fmt.Sprintf(sqlDeleteWorkflow, encodeString(source.GetPrimary().DbName()), encodeString(ReverseWorkflowName(ts.WorkflowName()))) if _, err := ts.TabletManagerClient().VReplicationExec(ctx, source.GetPrimary().Tablet, query); err != nil { + // vreplication.exec returns no error on delete if the rows do not exist. return err } ts.ws.deleteWorkflowVDiffData(ctx, source.GetPrimary().Tablet, ReverseWorkflowName(ts.WorkflowName())) @@ -1147,7 +1158,7 @@ func (ts *trafficSwitcher) removeTargetTables(ctx context.Context) error { } query := fmt.Sprintf("drop table %s.%s", primaryDbName, tableName) ts.Logger().Infof("%s: Dropping table %s.%s\n", - target.GetPrimary().String(), target.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(target.GetPrimary().Alias), target.GetPrimary().DbName(), tableName) res, err := ts.ws.tmc.ExecuteFetchAsDba(ctx, target.GetPrimary().Tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ Query: []byte(query), MaxRows: 1, @@ -1156,12 +1167,16 @@ func (ts *trafficSwitcher) removeTargetTables(ctx context.Context) error { }) log.Infof("Removed target table with result: %+v", res) if err != nil { - ts.Logger().Errorf("%s: Error removing table %s: %v", - target.GetPrimary().String(), tableName, err) + if mysqlErr, ok := err.(*sqlerror.SQLError); ok && mysqlErr.Num == sqlerror.ERNoSuchTable { + // The table was already gone, so we can ignore the error. + ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(target.GetPrimary().Alias), tableName) + return nil + } + ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(target.GetPrimary().Alias), tableName, err) return err } ts.Logger().Infof("%s: Removed table %s.%s\n", - target.GetPrimary().String(), target.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(target.GetPrimary().Alias), target.GetPrimary().DbName(), tableName) } return nil @@ -1626,7 +1641,7 @@ func (ts *trafficSwitcher) resetSequences(ctx context.Context) error { } return ts.ForAllSources(func(source *MigrationSource) error { ts.Logger().Infof("Resetting sequences for source shard %s.%s on tablet %s", - source.GetShard().Keyspace(), source.GetShard().ShardName(), source.GetPrimary().String()) + source.GetShard().Keyspace(), source.GetShard().ShardName(), topoproto.TabletAliasString(source.GetPrimary().Alias)) return ts.TabletManagerClient().ResetSequences(ctx, source.GetPrimary().Tablet, ts.Tables()) }) } From 7dd69c23ec3f5386613be72cae399be308489831 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 09:09:20 -0400 Subject: [PATCH 05/34] Improve and unify logging for tablet reference Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/traffic_switcher.go | 40 ++++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index 15c5439aa9c..16027d1dba2 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -524,14 +524,14 @@ func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType T query := fmt.Sprintf("drop table %s.%s", primaryDbName, tableNameEscaped) if removalType == DropTable { ts.Logger().Infof("%s: Dropping table %s.%s\n", - topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(source.GetPrimary().GetAlias()), source.GetPrimary().DbName(), tableName) } else { renameName, err := sqlescape.EnsureEscaped(getRenameFileName(tableName)) if err != nil { return err } ts.Logger().Infof("%s: Renaming table %s.%s to %s.%s\n", - topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName, source.GetPrimary().DbName(), renameName) + topoproto.TabletAliasString(source.GetPrimary().GetAlias()), source.GetPrimary().DbName(), tableName, source.GetPrimary().DbName(), renameName) query = fmt.Sprintf("rename table %s.%s TO %s.%s", primaryDbName, tableNameEscaped, primaryDbName, renameName) } _, err = ts.ws.tmc.ExecuteFetchAsDba(ctx, source.GetPrimary().Tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ @@ -542,13 +542,13 @@ func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType T }) if err != nil { if mysqlErr, ok := err.(*sqlerror.SQLError); ok && mysqlErr.Num == sqlerror.ERNoSuchTable { - ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(source.GetPrimary().Alias), tableName) + ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(source.GetPrimary().GetAlias()), tableName) return nil } - ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(source.GetPrimary().Alias), tableName, err) + ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(source.GetPrimary().GetAlias()), tableName, err) return err } - ts.Logger().Infof("%s: Removed table %s.%s\n", topoproto.TabletAliasString(source.GetPrimary().Alias), source.GetPrimary().DbName(), tableName) + ts.Logger().Infof("%s: Removed table %s.%s\n", topoproto.TabletAliasString(source.GetPrimary().GetAlias()), source.GetPrimary().DbName(), tableName) } return nil @@ -647,7 +647,7 @@ func (ts *trafficSwitcher) startReverseVReplication(ctx context.Context) error { return ts.ForAllSources(func(source *MigrationSource) error { query := fmt.Sprintf("update _vt.vreplication set state='Running', message='' where db_name=%s and workflow=%s", encodeString(source.GetPrimary().DbName()), encodeString(ts.ReverseWorkflowName())) - _, err := ts.VReplicationExec(ctx, source.GetPrimary().Alias, query) + _, err := ts.VReplicationExec(ctx, source.GetPrimary().GetAlias(), query) return err }) } @@ -935,8 +935,8 @@ func (ts *trafficSwitcher) createReverseVReplication(ctx context.Context) error }) } log.Infof("Creating reverse workflow vreplication stream on tablet %s: workflow %s, startPos %s", - source.GetPrimary().Alias, ts.ReverseWorkflowName(), target.Position) - _, err = ts.VReplicationExec(ctx, source.GetPrimary().Alias, + source.GetPrimary().GetAlias(), ts.ReverseWorkflowName(), target.Position) + _, err = ts.VReplicationExec(ctx, source.GetPrimary().GetAlias(), binlogplayer.CreateVReplicationState(ts.ReverseWorkflowName(), reverseBls, target.Position, binlogdatapb.VReplicationWorkflowState_Stopped, source.GetPrimary().DbName(), ts.workflowType, ts.workflowSubType)) if err != nil { @@ -948,11 +948,11 @@ func (ts *trafficSwitcher) createReverseVReplication(ctx context.Context) error if err != nil { return err } - updateQuery := ts.getReverseVReplicationUpdateQuery(target.GetPrimary().Alias.Cell, - source.GetPrimary().Alias.Cell, source.GetPrimary().DbName(), string(optionsJSON)) + updateQuery := ts.getReverseVReplicationUpdateQuery(target.GetPrimary().GetAlias().GetCell(), + source.GetPrimary().GetAlias().GetCell(), source.GetPrimary().DbName(), string(optionsJSON)) if updateQuery != "" { - log.Infof("Updating vreplication stream entry on %s with: %s", source.GetPrimary().Alias, updateQuery) - _, err = ts.VReplicationExec(ctx, source.GetPrimary().Alias, updateQuery) + log.Infof("Updating vreplication stream entry on %s with: %s", source.GetPrimary().GetAlias(), updateQuery) + _, err = ts.VReplicationExec(ctx, source.GetPrimary().GetAlias(), updateQuery) return err } return nil @@ -993,7 +993,7 @@ func (ts *trafficSwitcher) waitForCatchup(ctx context.Context, filteredReplicati // Source writes have been stopped, wait for all streams on targets to catch up. if err := ts.ForAllUIDs(func(target *MigrationTarget, uid int32) error { ts.Logger().Infof("Before Catchup: uid: %d, target primary %s, target position %s, shard %s", uid, - target.GetPrimary().AliasString(), target.Position, target.GetShard().String()) + topoproto.TabletAliasString(target.GetPrimary().GetAlias()), target.Position, target.GetShard().String()) bls := target.Sources[uid] source := ts.Sources()[bls.Shard] ts.Logger().Infof("Before Catchup: waiting for keyspace:shard: %v:%v to reach source position %v, uid %d", @@ -1006,7 +1006,7 @@ func (ts *trafficSwitcher) waitForCatchup(ctx context.Context, filteredReplicati ts.Logger().Infof("After catchup: position for keyspace:shard: %v:%v reached, uid %d", ts.TargetKeyspaceName(), target.GetShard().ShardName(), uid) if _, err := ts.TabletManagerClient().VReplicationExec(ctx, target.GetPrimary().Tablet, binlogplayer.StopVReplication(uid, "stopped for cutover")); err != nil { - log.Infof("Error marking stopped for cutover on %s, uid %d", target.GetPrimary().AliasString(), uid) + log.Infof("Error marking stopped for cutover on %s, uid %d", topoproto.TabletAliasString(target.GetPrimary().GetAlias()), uid) return err } return nil @@ -1017,7 +1017,7 @@ func (ts *trafficSwitcher) waitForCatchup(ctx context.Context, filteredReplicati return ts.ForAllTargets(func(target *MigrationTarget) error { var err error target.Position, err = ts.TabletManagerClient().PrimaryPosition(ctx, target.GetPrimary().Tablet) - ts.Logger().Infof("After catchup, position for target primary %s, %v", target.GetPrimary().AliasString(), target.Position) + ts.Logger().Infof("After catchup, position for target primary %s, %v", topoproto.TabletAliasString(target.GetPrimary().GetAlias()), target.Position) return err }) } @@ -1158,7 +1158,7 @@ func (ts *trafficSwitcher) removeTargetTables(ctx context.Context) error { } query := fmt.Sprintf("drop table %s.%s", primaryDbName, tableName) ts.Logger().Infof("%s: Dropping table %s.%s\n", - topoproto.TabletAliasString(target.GetPrimary().Alias), target.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(target.GetPrimary().GetAlias()), target.GetPrimary().DbName(), tableName) res, err := ts.ws.tmc.ExecuteFetchAsDba(ctx, target.GetPrimary().Tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ Query: []byte(query), MaxRows: 1, @@ -1169,14 +1169,14 @@ func (ts *trafficSwitcher) removeTargetTables(ctx context.Context) error { if err != nil { if mysqlErr, ok := err.(*sqlerror.SQLError); ok && mysqlErr.Num == sqlerror.ERNoSuchTable { // The table was already gone, so we can ignore the error. - ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(target.GetPrimary().Alias), tableName) + ts.Logger().Warningf("%s: Table %s did not exist when attempting to remove it", topoproto.TabletAliasString(target.GetPrimary().GetAlias()), tableName) return nil } - ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(target.GetPrimary().Alias), tableName, err) + ts.Logger().Errorf("%s: Error removing table %s: %v", topoproto.TabletAliasString(target.GetPrimary().GetAlias()), tableName, err) return err } ts.Logger().Infof("%s: Removed table %s.%s\n", - topoproto.TabletAliasString(target.GetPrimary().Alias), target.GetPrimary().DbName(), tableName) + topoproto.TabletAliasString(target.GetPrimary().GetAlias()), target.GetPrimary().DbName(), tableName) } return nil @@ -1641,7 +1641,7 @@ func (ts *trafficSwitcher) resetSequences(ctx context.Context) error { } return ts.ForAllSources(func(source *MigrationSource) error { ts.Logger().Infof("Resetting sequences for source shard %s.%s on tablet %s", - source.GetShard().Keyspace(), source.GetShard().ShardName(), topoproto.TabletAliasString(source.GetPrimary().Alias)) + source.GetShard().Keyspace(), source.GetShard().ShardName(), topoproto.TabletAliasString(source.GetPrimary().GetAlias())) return ts.TabletManagerClient().ResetSequences(ctx, source.GetPrimary().Tablet, ts.Tables()) }) } From 371c90cac865c911ebaa6f3ce631f1d4185c1b29 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 09:25:34 -0400 Subject: [PATCH 06/34] Put initial denied tables entries in place on MoveTables create Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 31 +++++++++++++++++++++--- go/vt/vtctl/workflow/traffic_switcher.go | 16 ++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index ebd4b4b2ec6..5954e6c5ee5 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1460,6 +1460,9 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl if cerr != nil { err = vterrors.Wrapf(err, "failed to cleanup workflow artifacts: %v", cerr) } + if cerr = ts.dropTargetDeniedTables(ctx); cerr != nil { + err = vterrors.Wrapf(err, "failed to cleanup denied table entries: %v", cerr) + } if cerr := s.dropArtifacts(ctx, false, &switcher{s: s, ts: ts}); cerr != nil { err = vterrors.Wrapf(err, "failed to cleanup workflow artifacts: %v", cerr) } @@ -1473,9 +1476,9 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl }() // Now that the streams have been successfully created, let's put the associated - // routing rules in place. + // routing rules and denied tables entries in place. if externalTopo == nil { - if err := s.setupInitialRoutingRules(ctx, req, mz, tables, vschema); err != nil { + if err := s.setupInitialRoutingRules(ctx, req, mz, tables); err != nil { return nil, err } @@ -1484,6 +1487,9 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl return nil, err } } + if err := s.setupInitialDeniedTables(ctx, req); err != nil { + return nil, vterrors.Wrapf(err, "failed to put initial denied tables entries in place on the target shards") + } if err := s.ts.RebuildSrvVSchema(ctx, nil); err != nil { return nil, err } @@ -1547,7 +1553,26 @@ func (s *Server) validateRoutingRuleFlags(req *vtctldatapb.MoveTablesCreateReque return nil } -func (s *Server) setupInitialRoutingRules(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest, mz *materializer, tables []string, vschema *vschemapb.Keyspace) error { +func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest) error { + ts, err := s.buildTrafficSwitcher(ctx, req.GetTargetKeyspace(), req.GetWorkflow()) + if err != nil { + return err + } + err = ts.ForAllTargets(func(target *MigrationTarget) error { + if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) + }); err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) + defer cancel() + _, _, err := topotools.RefreshTabletsByShard(ctx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) + return err + }) + return err +} + +func (s *Server) setupInitialRoutingRules(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest, mz *materializer, tables []string) error { if err := s.validateRoutingRuleFlags(req, mz); err != nil { return err } diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index 16027d1dba2..ff9a0052c36 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -731,6 +731,22 @@ func (ts *trafficSwitcher) allowTableTargetWrites(ctx context.Context) error { }) } +// preventTableTargetWrites puts denied table entries in place for the given tables on the +// target tablets. This should be used when a MoveTables workflow is initially created. +func (ts *trafficSwitcher) preventTableTargetWrites(ctx context.Context) error { + return ts.ForAllTargets(func(target *MigrationTarget) error { + if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) + }); err != nil { + return err + } + rtbsCtx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) + defer cancel() + _, _, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) + return err + }) +} + func (ts *trafficSwitcher) changeRouting(ctx context.Context) error { if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { return ts.changeWriteRoute(ctx) From 7b87e59a6c6360dc150d3c9db6db15e0f5b956ad Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 09:59:36 -0400 Subject: [PATCH 07/34] Get target keyspace lock before adding denied tables entries Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 15 +++++++++++---- go/vt/vtctl/workflow/traffic_switcher.go | 16 ---------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 5954e6c5ee5..952e877d94a 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1558,15 +1558,22 @@ func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb. if err != nil { return err } + sw := &switcher{s: s, ts: ts} + lockCtx, targetUnlock, lockErr := sw.lockKeyspace(ctx, ts.TargetKeyspaceName(), "SetupInitialDeniedTables") + if lockErr != nil { + ts.Logger().Errorf("Locking target keyspace %s failed: %v", ts.TargetKeyspaceName(), lockErr) + return lockErr + } + defer targetUnlock(&err) err = ts.ForAllTargets(func(target *MigrationTarget) error { - if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) + if _, err := ts.TopoServer().UpdateShardFields(lockCtx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(lockCtx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) }); err != nil { return err } - ctx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) + strCtx, cancel := context.WithTimeout(lockCtx, shardTabletRefreshTimeout) defer cancel() - _, _, err := topotools.RefreshTabletsByShard(ctx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) + _, _, err := topotools.RefreshTabletsByShard(strCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) return err }) return err diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index ff9a0052c36..16027d1dba2 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -731,22 +731,6 @@ func (ts *trafficSwitcher) allowTableTargetWrites(ctx context.Context) error { }) } -// preventTableTargetWrites puts denied table entries in place for the given tables on the -// target tablets. This should be used when a MoveTables workflow is initially created. -func (ts *trafficSwitcher) preventTableTargetWrites(ctx context.Context) error { - return ts.ForAllTargets(func(target *MigrationTarget) error { - if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) - }); err != nil { - return err - } - rtbsCtx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) - defer cancel() - _, _, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) - return err - }) -} - func (ts *trafficSwitcher) changeRouting(ctx context.Context) error { if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { return ts.changeWriteRoute(ctx) From 8650d9576bfe59f1b72d475723547e289b23d384 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 10:01:00 -0400 Subject: [PATCH 08/34] Comment out broken unit test for now Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index a6a38fd3040..905240d59fb 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -200,6 +200,7 @@ func TestVDiffCreate(t *testing.T) { } } +/* TODO: get the framework in place for this and then get the test working func TestWorkflowDelete(t *testing.T) { ctx := context.Background() ksName := "ks1" @@ -256,3 +257,4 @@ func TestWorkflowDelete(t *testing.T) { }) } } +*/ From 4b381f50153e6022cad3344a55ddb5124a082e6b Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 16:31:38 -0400 Subject: [PATCH 09/34] Adjust e2e tests Signed-off-by: Matt Lord --- go/test/endtoend/cluster/vttablet_process.go | 7 ++- go/test/endtoend/vreplication/fk_test.go | 4 +- go/test/endtoend/vreplication/helper_test.go | 29 ++++++----- .../vreplication/initial_data_test.go | 30 ++++++------ .../endtoend/vreplication/materialize_test.go | 2 +- go/test/endtoend/vreplication/migrate_test.go | 22 ++++----- .../partial_movetables_seq_test.go | 6 +-- .../endtoend/vreplication/reference_test.go | 2 +- .../resharding_workflows_v2_test.go | 20 ++++---- .../endtoend/vreplication/time_zone_test.go | 4 +- go/test/endtoend/vreplication/vdiff2_test.go | 8 +-- .../vreplication/vdiff_helper_test.go | 4 +- .../vreplication/vdiff_online_ddl_test.go | 2 +- .../vreplication/vreplication_test.go | 49 ++++++++++--------- go/test/endtoend/vreplication/vstream_test.go | 8 +-- 15 files changed, 104 insertions(+), 93 deletions(-) diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 45db1dc4bd2..0c2852b569b 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -74,6 +74,7 @@ type VttabletProcess struct { SupportsBackup bool ExplicitServingStatus bool ServingStatus string + DbName string DbPassword string DbPort int DbFlavor string @@ -148,6 +149,8 @@ func (vttablet *VttabletProcess) Setup() (err error) { return } + vttablet.DbName = "vt_" + vttablet.Keyspace + vttablet.exit = make(chan error) go func() { if vttablet.proc != nil { @@ -442,8 +445,8 @@ func (vttablet *VttabletProcess) TearDownWithTimeout(timeout time.Duration) erro // CreateDB creates the database for keyspace func (vttablet *VttabletProcess) CreateDB(keyspace string) error { - _, _ = vttablet.QueryTablet(fmt.Sprintf("drop database IF EXISTS vt_%s", keyspace), keyspace, false) - _, err := vttablet.QueryTablet(fmt.Sprintf("create database IF NOT EXISTS vt_%s", keyspace), keyspace, false) + _, _ = vttablet.QueryTablet(fmt.Sprintf("drop database IF EXISTS %s", vttablet.DbName), keyspace, false) + _, err := vttablet.QueryTablet(fmt.Sprintf("create database IF NOT EXISTS %s", vttablet.DbName), keyspace, false) return err } diff --git a/go/test/endtoend/vreplication/fk_test.go b/go/test/endtoend/vreplication/fk_test.go index 09692930c5c..69f8b8a63be 100644 --- a/go/test/endtoend/vreplication/fk_test.go +++ b/go/test/endtoend/vreplication/fk_test.go @@ -222,7 +222,7 @@ func (ls *fkLoadSimulator) simulateLoad() { func (ls *fkLoadSimulator) getNumRowsParent(vtgateConn *mysql.Conn) int { t := ls.t - qr := execVtgateQuery(t, vtgateConn, "fksource", "SELECT COUNT(*) FROM parent") + qr := execQueryWithDatabase(t, vtgateConn, "fksource", "SELECT COUNT(*) FROM parent") require.NotNil(t, qr) numRows, err := strconv.Atoi(qr.Rows[0][0].ToString()) require.NoError(t, err) @@ -296,7 +296,7 @@ func (ls *fkLoadSimulator) exec(query string) *sqltypes.Result { t := ls.t vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execVtgateQuery(t, vtgateConn, "fksource", query) + qr := execQueryWithDatabase(t, vtgateConn, "fksource", query) require.NotNil(t, qr) return qr } diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 4764b213ad6..398ac42fc00 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -72,7 +72,7 @@ func execMultipleQueries(t *testing.T, conn *mysql.Conn, database string, lines if strings.HasPrefix(query, "--") { continue } - execVtgateQuery(t, conn, database, string(query)) + execQueryWithDatabase(t, conn, database, string(query)) } } @@ -134,7 +134,7 @@ func getConnection(t *testing.T, hostname string, port int) *mysql.Conn { return conn } -func execVtgateQuery(t *testing.T, conn *mysql.Conn, database string, query string) *sqltypes.Result { +func execQueryWithDatabase(t *testing.T, conn *mysql.Conn, database string, query string) *sqltypes.Result { if strings.TrimSpace(query) == "" { return nil } @@ -171,7 +171,7 @@ func waitForQueryResult(t *testing.T, conn *mysql.Conn, database string, query s timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - qr := execVtgateQuery(t, conn, database, query) + qr := execQueryWithDatabase(t, conn, database, query) require.NotNil(t, qr) if want == fmt.Sprintf("%v", qr.Rows) { return @@ -245,7 +245,7 @@ func waitForNoWorkflowLag(t *testing.T, vc *VitessCluster, keyspace, worfklow st // verifyNoInternalTables can e.g. be used to confirm that no internal tables were // copied from a source to a target during a MoveTables or Reshard operation. func verifyNoInternalTables(t *testing.T, conn *mysql.Conn, keyspaceShard string) { - qr := execVtgateQuery(t, conn, keyspaceShard, "show tables") + qr := execQueryWithDatabase(t, conn, keyspaceShard, "show tables") require.NotNil(t, qr) require.NotNil(t, qr.Rows) for _, row := range qr.Rows { @@ -260,7 +260,7 @@ func waitForRowCount(t *testing.T, conn *mysql.Conn, database string, table stri timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - qr := execVtgateQuery(t, conn, database, query) + qr := execQueryWithDatabase(t, conn, database, query) require.NotNil(t, qr) if wantRes == fmt.Sprintf("%v", qr.Rows) { return @@ -332,7 +332,7 @@ func executeOnTablet(t *testing.T, conn *mysql.Conn, tablet *cluster.VttabletPro count0, body0 := getQueryCount(t, queryStatsURL, matchQuery) - qr := execVtgateQuery(t, conn, ksName, query) + qr := execQueryWithDatabase(t, conn, ksName, query) require.NotNil(t, qr) count1, body1 := getQueryCount(t, queryStatsURL, matchQuery) @@ -602,12 +602,17 @@ func expectNumberOfStreams(t *testing.T, vtgateConn *mysql.Conn, name string, wo waitForQueryResult(t, vtgateConn, database, query, fmt.Sprintf(`[[INT64(%d)]]`, want)) } -// confirmAllStreamsRunning confirms that all of the migrated streams are running -// after a Reshard. -func confirmAllStreamsRunning(t *testing.T, vtgateConn *mysql.Conn, database string) { +// confirmAllStreamsRunning confirms that all of migrated streams are running +// for a workflow. +func confirmAllStreamsRunning(t *testing.T, keyspace, shard string) { query := sqlparser.BuildParsedQuery("select count(*) from %s.vreplication where state != '%s'", sidecarDBIdentifier, binlogdatapb.VReplicationWorkflowState_Running.String()).Query - waitForQueryResult(t, vtgateConn, database, query, `[[INT64(0)]]`) + tablet := vc.getPrimaryTablet(t, keyspace, shard) + // Query the tablet's mysqld directly as the target may have denied table entries. + dbc, err := tablet.TabletConn(keyspace, true) + require.NoError(t, err) + defer dbc.Close() + waitForQueryResult(t, dbc, sidecarDBName, query, `[[INT64(0)]]`) } func printShardPositions(vc *VitessCluster, ksShards []string) { @@ -801,7 +806,7 @@ func isBinlogRowImageNoBlob(t *testing.T, tablet *cluster.VttabletProcess) bool func getRowCount(t *testing.T, vtgateConn *mysql.Conn, table string) int { query := fmt.Sprintf("select count(*) from %s", table) - qr := execVtgateQuery(t, vtgateConn, "", query) + qr := execQuery(t, vtgateConn, query) numRows, _ := qr.Rows[0][0].ToInt() return numRows } @@ -1009,7 +1014,7 @@ func vexplain(t *testing.T, database, query string) *VExplainPlan { vtgateConn := vc.GetVTGateConn(t) defer vtgateConn.Close() - qr := execVtgateQuery(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) + qr := execQueryWithDatabase(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) require.NotNil(t, qr) require.Equal(t, 1, len(qr.Rows)) json := qr.Rows[0][0].ToString() diff --git a/go/test/endtoend/vreplication/initial_data_test.go b/go/test/endtoend/vreplication/initial_data_test.go index ea34ef7fddf..22cd9c50c33 100644 --- a/go/test/endtoend/vreplication/initial_data_test.go +++ b/go/test/endtoend/vreplication/initial_data_test.go @@ -32,9 +32,9 @@ func insertInitialData(t *testing.T) { log.Infof("Inserting initial data") lines, _ := os.ReadFile("unsharded_init_data.sql") execMultipleQueries(t, vtgateConn, "product:0", string(lines)) - execVtgateQuery(t, vtgateConn, "product:0", "insert into customer_seq(id, next_id, cache) values(0, 100, 100);") - execVtgateQuery(t, vtgateConn, "product:0", "insert into order_seq(id, next_id, cache) values(0, 100, 100);") - execVtgateQuery(t, vtgateConn, "product:0", "insert into customer_seq2(id, next_id, cache) values(0, 100, 100);") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into customer_seq(id, next_id, cache) values(0, 100, 100);") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into order_seq(id, next_id, cache) values(0, 100, 100);") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into customer_seq2(id, next_id, cache) values(0, 100, 100);") log.Infof("Done inserting initial data") waitForRowCount(t, vtgateConn, "product:0", "product", 2) @@ -52,12 +52,12 @@ func insertJSONValues(t *testing.T) { // insert null value combinations vtgateConn, closeConn := getVTGateConn() defer closeConn() - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(1, \"{}\")") - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j3) values(2, \"{}\", \"{}\")") - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j2, j3) values(3, \"{}\", \"{}\")") - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j2, j3) values(4, NULL, 'null', '\"null\"')") - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(5, JSON_QUOTE('null'))") - execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(6, '{}')") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(1, \"{}\")") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j3) values(2, \"{}\", \"{}\")") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j2, j3) values(3, \"{}\", \"{}\")") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j2, j3) values(4, NULL, 'null', '\"null\"')") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(5, JSON_QUOTE('null'))") + execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(6, '{}')") id := 8 // 6 inserted above and one after copy phase is done @@ -68,7 +68,7 @@ func insertJSONValues(t *testing.T) { j1 := rand.IntN(numJsonValues) j2 := rand.IntN(numJsonValues) query := fmt.Sprintf(q, id, jsonValues[j1], jsonValues[j2]) - execVtgateQuery(t, vtgateConn, "product:0", query) + execQueryWithDatabase(t, vtgateConn, "product:0", query) } } @@ -97,28 +97,28 @@ func insertMoreCustomers(t *testing.T, numCustomers int) { } cid++ } - execVtgateQuery(t, vtgateConn, "customer", sql) + execQueryWithDatabase(t, vtgateConn, "customer", sql) } func insertMoreProducts(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(3, 'cpu'),(4, 'camera'),(5, 'mouse');" - execVtgateQuery(t, vtgateConn, "product", sql) + execQueryWithDatabase(t, vtgateConn, "product", sql) } func insertMoreProductsForSourceThrottler(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(103, 'new-cpu'),(104, 'new-camera'),(105, 'new-mouse');" - execVtgateQuery(t, vtgateConn, "product", sql) + execQueryWithDatabase(t, vtgateConn, "product", sql) } func insertMoreProductsForTargetThrottler(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(203, 'new-cpu'),(204, 'new-camera'),(205, 'new-mouse');" - execVtgateQuery(t, vtgateConn, "product", sql) + execQueryWithDatabase(t, vtgateConn, "product", sql) } var blobTableQueries = []string{ @@ -137,6 +137,6 @@ func insertIntoBlobTable(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() for _, query := range blobTableQueries { - execVtgateQuery(t, vtgateConn, "product:0", query) + execQueryWithDatabase(t, vtgateConn, "product:0", query) } } diff --git a/go/test/endtoend/vreplication/materialize_test.go b/go/test/endtoend/vreplication/materialize_test.go index 486692a58ba..3297d83acd7 100644 --- a/go/test/endtoend/vreplication/materialize_test.go +++ b/go/test/endtoend/vreplication/materialize_test.go @@ -206,7 +206,7 @@ func testMaterialize(t *testing.T, useVtctldClient bool) { waitForQueryResult(t, vtgateConn, targetKs, "select id, val, ts, day, month, x from mat2", want) // insert data to test the replication phase - execVtgateQuery(t, vtgateConn, sourceKs, "insert into mat(id, val, ts) values (3, 'ghi', '2021-12-11 16:17:36')") + execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into mat(id, val, ts) values (3, 'ghi', '2021-12-11 16:17:36')") // validate data after the replication phase waitForQueryResult(t, vtgateConn, targetKs, "select count(*) from mat2", "[[INT64(3)]]") diff --git a/go/test/endtoend/vreplication/migrate_test.go b/go/test/endtoend/vreplication/migrate_test.go index 1f365c47600..f2d6b62024c 100644 --- a/go/test/endtoend/vreplication/migrate_test.go +++ b/go/test/endtoend/vreplication/migrate_test.go @@ -31,11 +31,11 @@ import ( func insertInitialDataIntoExternalCluster(t *testing.T, conn *mysql.Conn) { t.Run("insertInitialData", func(t *testing.T) { fmt.Printf("Inserting initial data\n") - execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(1, 1, 'review1');") - execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(2, 1, 'review2');") - execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(3, 2, 'review3');") - execVtgateQuery(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(1, 1, 4);") - execVtgateQuery(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(2, 2, 5);") + execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(1, 1, 'review1');") + execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(2, 1, 'review2');") + execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(3, 2, 'review3');") + execQueryWithDatabase(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(1, 1, 4);") + execQueryWithDatabase(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(2, 2, 5);") }) } @@ -109,8 +109,8 @@ func TestVtctlMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 1) waitForRowCount(t, vtgateConn, "product:0", "rating", 2) waitForRowCount(t, vtgateConn, "product:0", "review", 3) - execVtgateQuery(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") - execVtgateQuery(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") + execQueryWithDatabase(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") + execQueryWithDatabase(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") waitForRowCount(t, vtgateConn, "product:0", "rating", 3) waitForRowCount(t, vtgateConn, "product:0", "review", 4) vdiffSideBySide(t, ksWorkflow, "extcell1") @@ -122,7 +122,7 @@ func TestVtctlMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 0) }) t.Run("cancel migrate workflow", func(t *testing.T) { - execVtgateQuery(t, vtgateConn, "product", "drop table review,rating") + execQueryWithDatabase(t, vtgateConn, "product", "drop table review,rating") if output, err = vc.VtctlClient.ExecuteCommandWithOutput("Migrate", "--", "--all", "--auto_start=false", "--cells=extcell1", "--source=ext1.rating", "create", ksWorkflow); err != nil { @@ -234,8 +234,8 @@ func TestVtctldMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 1) waitForRowCount(t, vtgateConn, "product:0", "rating", 2) waitForRowCount(t, vtgateConn, "product:0", "review", 3) - execVtgateQuery(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") - execVtgateQuery(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") + execQueryWithDatabase(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") + execQueryWithDatabase(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") waitForRowCount(t, vtgateConn, "product:0", "rating", 3) waitForRowCount(t, vtgateConn, "product:0", "review", 4) vdiffSideBySide(t, ksWorkflow, "extcell1") @@ -261,7 +261,7 @@ func TestVtctldMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 0) }) t.Run("cancel migrate workflow", func(t *testing.T) { - execVtgateQuery(t, vtgateConn, "product", "drop table review,rating") + execQueryWithDatabase(t, vtgateConn, "product", "drop table review,rating") output, err = vc.VtctldClient.ExecuteCommandWithOutput("Migrate", "--target-keyspace", "product", "--workflow", "e1", "Create", "--source-keyspace", "rating", "--mount-name", "ext1", "--all-tables", "--auto-start=false", "--cells=extcell1") diff --git a/go/test/endtoend/vreplication/partial_movetables_seq_test.go b/go/test/endtoend/vreplication/partial_movetables_seq_test.go index eec304e0a4d..d57e18f4bf1 100644 --- a/go/test/endtoend/vreplication/partial_movetables_seq_test.go +++ b/go/test/endtoend/vreplication/partial_movetables_seq_test.go @@ -532,7 +532,7 @@ var lastCustomerId int64 func getCustomerCount(t *testing.T, msg string) int64 { vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execVtgateQuery(t, vtgateConn, "", "select count(*) from customer") + qr := execQueryWithDatabase(t, vtgateConn, "", "select count(*) from customer") require.NotNil(t, qr) count, err := qr.Rows[0][0].ToInt64() require.NoError(t, err) @@ -542,7 +542,7 @@ func getCustomerCount(t *testing.T, msg string) int64 { func confirmLastCustomerIdHasIncreased(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execVtgateQuery(t, vtgateConn, "", "select cid from customer order by cid desc limit 1") + qr := execQueryWithDatabase(t, vtgateConn, "", "select cid from customer order by cid desc limit 1") require.NotNil(t, qr) currentCustomerId, err := qr.Rows[0][0].ToInt64() require.NoError(t, err) @@ -554,7 +554,7 @@ func insertCustomers(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() for i := int64(1); i < newCustomerCount+1; i++ { - execVtgateQuery(t, vtgateConn, "customer@primary", fmt.Sprintf("insert into customer(name) values ('name-%d')", currentCustomerCount+i)) + execQueryWithDatabase(t, vtgateConn, "customer@primary", fmt.Sprintf("insert into customer(name) values ('name-%d')", currentCustomerCount+i)) } customerCount = getCustomerCount(t, "") require.Equal(t, currentCustomerCount+newCustomerCount, customerCount) diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index 8ff77de8708..969e0bd2c88 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -146,7 +146,7 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { execRefQuery(t, "update sks.mfg2 set name = concat(name, '-updated') where id = 4") waitForRowCount(t, vtgateConn, uks, "mfg", 8) - qr := execVtgateQuery(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") + qr := execQueryWithDatabase(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") require.NotNil(t, qr) require.Equal(t, "4", qr.Rows[0][0].ToString()) diff --git a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go index 34a4b269b53..90f20b34ce8 100644 --- a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go +++ b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go @@ -359,7 +359,7 @@ func validateWritesRouteToSource(t *testing.T) { insertQuery := "insert into customer(name, cid) values('tempCustomer2', 200)" matchInsertQuery := "insert into customer(`name`, cid) values" assertQueryExecutesOnTablet(t, vtgateConn, sourceTab, "customer", insertQuery, matchInsertQuery) - execVtgateQuery(t, vtgateConn, "customer", "delete from customer where cid = 200") + execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where cid = 200") } func validateWritesRouteToTarget(t *testing.T) { @@ -370,7 +370,7 @@ func validateWritesRouteToTarget(t *testing.T) { assertQueryExecutesOnTablet(t, vtgateConn, targetTab2, "customer", insertQuery, matchInsertQuery) insertQuery = "insert into customer(name, cid) values('tempCustomer3', 102)" assertQueryExecutesOnTablet(t, vtgateConn, targetTab1, "customer", insertQuery, matchInsertQuery) - execVtgateQuery(t, vtgateConn, "customer", "delete from customer where cid in (101, 102)") + execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where cid in (101, 102)") } func revert(t *testing.T, workflowType string) { @@ -470,7 +470,7 @@ func testVSchemaForSequenceAfterMoveTables(t *testing.T) { // ensure sequence is available to vtgate num := 5 for i := 0; i < num; i++ { - execVtgateQuery(t, vtgateConn, "customer", "insert into customer2(name) values('a')") + execQueryWithDatabase(t, vtgateConn, "customer", "insert into customer2(name) values('a')") } waitForRowCount(t, vtgateConn, "customer", "customer2", 3+num) want := fmt.Sprintf("[[INT32(%d)]]", 100+num-1) @@ -502,7 +502,7 @@ func testVSchemaForSequenceAfterMoveTables(t *testing.T) { // ensure sequence is available to vtgate for i := 0; i < num; i++ { - execVtgateQuery(t, vtgateConn, "product", "insert into customer2(name) values('a')") + execQueryWithDatabase(t, vtgateConn, "product", "insert into customer2(name) values('a')") } waitForRowCount(t, vtgateConn, "product", "customer2", 3+num+num) want = fmt.Sprintf("[[INT32(%d)]]", 100+num+num-1) @@ -525,10 +525,10 @@ func testReplicatingWithPKEnumCols(t *testing.T) { // typ is an enum, with soho having a stored and binlogged value of 2 deleteQuery := "delete from customer where cid = 2 and typ = 'soho'" insertQuery := "insert into customer(cid, name, typ, sport, meta) values(2, 'Paül','soho','cricket',convert(x'7b7d' using utf8mb4))" - execVtgateQuery(t, vtgateConn, sourceKs, deleteQuery) + execQueryWithDatabase(t, vtgateConn, sourceKs, deleteQuery) waitForNoWorkflowLag(t, vc, targetKs, workflowName) vdiffSideBySide(t, ksWorkflow, "") - execVtgateQuery(t, vtgateConn, sourceKs, insertQuery) + execQueryWithDatabase(t, vtgateConn, sourceKs, insertQuery) waitForNoWorkflowLag(t, vc, targetKs, workflowName) vdiffSideBySide(t, ksWorkflow, "") } @@ -557,7 +557,7 @@ func testReshardV2Workflow(t *testing.T) { return default: // Use a random customer type for each record. - _ = execVtgateQuery(t, dataGenConn, "customer", fmt.Sprintf("insert into customer (cid, name, typ) values (%d, 'tempCustomer%d', %s)", + _ = execQueryWithDatabase(t, dataGenConn, "customer", fmt.Sprintf("insert into customer (cid, name, typ) values (%d, 'tempCustomer%d', %s)", id, id, customerTypes[rand.IntN(len(customerTypes))])) } time.Sleep(1 * time.Millisecond) @@ -588,17 +588,17 @@ func testReshardV2Workflow(t *testing.T) { // Confirm that we lost no customer related writes during the Reshard. dataGenCancel() dataGenWg.Wait() - cres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from customer") + cres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from customer") require.Len(t, cres.Rows, 1) waitForNoWorkflowLag(t, vc, "customer", "customer_name") - cnres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from customer_name") + cnres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from customer_name") require.Len(t, cnres.Rows, 1) require.EqualValues(t, cres.Rows, cnres.Rows) if debugMode { // We expect the row count to differ in enterprise_customer because it is // using a `where typ='enterprise'` filter. So the count is only for debug // info. - ecres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from enterprise_customer") + ecres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from enterprise_customer") t.Logf("Done inserting customer data. Record counts in customer: %s, customer_name: %s, enterprise_customer: %s", cres.Rows[0][0].ToString(), cnres.Rows[0][0].ToString(), ecres.Rows[0][0].ToString()) } diff --git a/go/test/endtoend/vreplication/time_zone_test.go b/go/test/endtoend/vreplication/time_zone_test.go index 2c0a9a4f5a5..cb271e04c8a 100644 --- a/go/test/endtoend/vreplication/time_zone_test.go +++ b/go/test/endtoend/vreplication/time_zone_test.go @@ -190,7 +190,7 @@ func TestMoveTablesTZ(t *testing.T) { } // inserts to test date conversions in reverse replication - execVtgateQuery(t, vtgateConn, "customer", "insert into datze(id, dt2) values (13, '2022-01-01 18:20:30')") - execVtgateQuery(t, vtgateConn, "customer", "insert into datze(id, dt2) values (14, '2022-04-01 12:06:07')") + execQueryWithDatabase(t, vtgateConn, "customer", "insert into datze(id, dt2) values (13, '2022-01-01 18:20:30')") + execQueryWithDatabase(t, vtgateConn, "customer", "insert into datze(id, dt2) values (14, '2022-04-01 12:06:07')") vdiffSideBySide(t, ksReverseWorkflow, "") } diff --git a/go/test/endtoend/vreplication/vdiff2_test.go b/go/test/endtoend/vreplication/vdiff2_test.go index 08f5bb8926d..7fb37430768 100644 --- a/go/test/endtoend/vreplication/vdiff2_test.go +++ b/go/test/endtoend/vreplication/vdiff2_test.go @@ -164,13 +164,13 @@ func TestVDiff2(t *testing.T) { // Insert null and empty enum values for testing vdiff comparisons for those values. // If we add this to the initial data list, the counts in several other tests will need to change query := `insert into customer(cid, name, typ, sport) values(1001, null, 'soho','')` - execVtgateQuery(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) + execQueryWithDatabase(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) generateMoreCustomers(t, sourceKs, 1000) // Create rows in the nopk table using the customer names and random ages between 20 and 100. query = "insert into nopk(name, age) select name, floor(rand()*80)+20 from customer" - execVtgateQuery(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) + execQueryWithDatabase(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) // The primary tablet is only added in the first cell. // We ONLY add primary tablets in this test. @@ -502,7 +502,7 @@ func testResume(t *testing.T, tc *testCase, cells string) { expectedNewRows := int64(0) if tc.resumeInsert != "" { - res := execVtgateQuery(t, vtgateConn, tc.sourceKs, tc.resumeInsert) + res := execQueryWithDatabase(t, vtgateConn, tc.sourceKs, tc.resumeInsert) expectedNewRows = int64(res.RowsAffected) } expectedRows := rowsCompared + expectedNewRows @@ -549,7 +549,7 @@ func testAutoRetryError(t *testing.T, tc *testCase, cells string) { // compared is cumulative. expectedNewRows := int64(0) if tc.retryInsert != "" { - res := execVtgateQuery(t, vtgateConn, tc.sourceKs, tc.retryInsert) + res := execQueryWithDatabase(t, vtgateConn, tc.sourceKs, tc.retryInsert) expectedNewRows = int64(res.RowsAffected) } expectedRows := rowsCompared + expectedNewRows diff --git a/go/test/endtoend/vreplication/vdiff_helper_test.go b/go/test/endtoend/vreplication/vdiff_helper_test.go index 53e19e56731..a2a967df3f1 100644 --- a/go/test/endtoend/vreplication/vdiff_helper_test.go +++ b/go/test/endtoend/vreplication/vdiff_helper_test.go @@ -354,7 +354,7 @@ func generateMoreCustomers(t *testing.T, keyspace string, numCustomers int64) { vtgateConn, closeConn := getVTGateConn() defer closeConn() log.Infof("Generating more test data with an additional %d customers", numCustomers) - res := execVtgateQuery(t, vtgateConn, keyspace, "select max(cid) from customer") + res := execQueryWithDatabase(t, vtgateConn, keyspace, "select max(cid) from customer") startingID, _ := res.Rows[0][0].ToInt64() insert := strings.Builder{} insert.WriteString("insert into customer(cid, name, typ) values ") @@ -366,5 +366,5 @@ func generateMoreCustomers(t *testing.T, keyspace string, numCustomers int64) { insert.WriteString(", ") } } - execVtgateQuery(t, vtgateConn, keyspace, insert.String()) + execQueryWithDatabase(t, vtgateConn, keyspace, insert.String()) } diff --git a/go/test/endtoend/vreplication/vdiff_online_ddl_test.go b/go/test/endtoend/vreplication/vdiff_online_ddl_test.go index 92977111294..1650ea50e0f 100644 --- a/go/test/endtoend/vreplication/vdiff_online_ddl_test.go +++ b/go/test/endtoend/vreplication/vdiff_online_ddl_test.go @@ -132,7 +132,7 @@ func waitForAdditionalRows(t *testing.T, keyspace, table string, count int) { } func getNumRows(t *testing.T, vtgateConn *mysql.Conn, keyspace, table string) int { - qr := execVtgateQuery(t, vtgateConn, keyspace, fmt.Sprintf("SELECT COUNT(*) FROM %s", table)) + qr := execQueryWithDatabase(t, vtgateConn, keyspace, fmt.Sprintf("SELECT COUNT(*) FROM %s", table)) require.NotNil(t, qr) numRows, err := strconv.Atoi(qr.Rows[0][0].ToString()) require.NoError(t, err) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index c06489006f8..0d975e018f0 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -340,7 +340,7 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string // the Lead and Lead-1 tables tested a specific case with binary sharding keys. Drop it now so that we don't // have to update the rest of the tests - execVtgateQuery(t, vtgateConn, "customer", "drop table `Lead`,`Lead-1`") + execQueryWithDatabase(t, vtgateConn, "customer", "drop table `Lead`,`Lead-1`") validateRollupReplicates(t) shardOrders(t) shardMerchant(t) @@ -360,13 +360,13 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string insertMoreCustomers(t, 16) reshardCustomer2to4Split(t, nil, "") - confirmAllStreamsRunning(t, vtgateConn, "customer:-40") + confirmAllStreamsRunning(t, "customer", "-40") expectNumberOfStreams(t, vtgateConn, "Customer2to4", "sales", "product:0", 4) reshardCustomer3to2SplitMerge(t) - confirmAllStreamsRunning(t, vtgateConn, "customer:-60") + confirmAllStreamsRunning(t, "customer", "-60") expectNumberOfStreams(t, vtgateConn, "Customer3to2", "sales", "product:0", 3) reshardCustomer3to1Merge(t) - confirmAllStreamsRunning(t, vtgateConn, "customer:0") + confirmAllStreamsRunning(t, "customer", "0") expectNumberOfStreams(t, vtgateConn, "Customer3to1", "sales", "product:0", 1) t.Run("Verify CopyState Is Optimized Afterwards", func(t *testing.T) { @@ -717,15 +717,18 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl defer vtgateConn.Close() // Confirm that the 0 scale decimal field, dec80, is replicated correctly dec80Replicated := false - execVtgateQuery(t, vtgateConn, sourceKs, "update customer set dec80 = 0") - execVtgateQuery(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") - execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") - execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") + execQueryWithDatabase(t, vtgateConn, sourceKs, "update customer set dec80 = 0") + execQueryWithDatabase(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") + execQueryWithDatabase(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") + execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") waitForNoWorkflowLag(t, vc, targetKs, workflow) - for _, shard := range []string{"-80", "80-"} { - shardTarget := fmt.Sprintf("%s:%s", targetKs, shard) - if res := execVtgateQuery(t, vtgateConn, shardTarget, "select cid from customer"); len(res.Rows) > 0 { - waitForQueryResult(t, vtgateConn, shardTarget, "select distinct dec80 from customer", `[[DECIMAL(0)]]`) + for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { + // Query the tablet's mysqld directly as the target may have denied table entries. + dbc, err := tablet.TabletConn(targetKs, true) + require.NoError(t, err) + defer dbc.Close() + if res := execQuery(t, dbc, "select cid from customer"); len(res.Rows) > 0 { + waitForQueryResult(t, dbc, tablet.DbName, "select distinct dec80 from customer", `[[DECIMAL(0)]]`) dec80Replicated = true } } @@ -734,8 +737,8 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // Insert multiple rows in the loadtest table and immediately delete them to confirm that bulk delete // works the same way with the vplayer optimization enabled and disabled. Currently this optimization // is disabled by default, but enabled in TestCellAliasVreplicationWorkflow. - execVtgateQuery(t, vtgateConn, sourceKs, "insert into loadtest(id, name) values(10001, 'tempCustomer'), (10002, 'tempCustomer2'), (10003, 'tempCustomer3'), (10004, 'tempCustomer4')") - execVtgateQuery(t, vtgateConn, sourceKs, "delete from loadtest where id > 10000") + execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into loadtest(id, name) values(10001, 'tempCustomer'), (10002, 'tempCustomer2'), (10003, 'tempCustomer3'), (10004, 'tempCustomer4')") + execQueryWithDatabase(t, vtgateConn, sourceKs, "delete from loadtest where id > 10000") // Confirm that all partial query metrics get updated when we are testing the noblob mode. t.Run("validate partial query counts", func(t *testing.T) { @@ -779,7 +782,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl if err != nil { require.FailNow(t, output) } - execVtgateQuery(t, vtgateConn, "product", fmt.Sprintf("update `%s` set name='xyz'", tbl)) + execQueryWithDatabase(t, vtgateConn, "product", fmt.Sprintf("update `%s` set name='xyz'", tbl)) } } vdiffSideBySide(t, ksWorkflow, "") @@ -800,7 +803,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // The original unsharded customer data included an insert with the // vindex column (cid) of 999999, so the backing sequence table should // now have a next_id of 1000000 after SwitchTraffic. - res := execVtgateQuery(t, vtgateConn, sourceKs, "select next_id from customer_seq where id = 0") + res := execQueryWithDatabase(t, vtgateConn, sourceKs, "select next_id from customer_seq where id = 0") require.Equal(t, "1000000", res.Rows[0][0].ToString()) if withOpenTx && commit != nil { @@ -811,7 +814,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl vdiffSideBySide(t, "product.p2c_reverse", "") if withOpenTx { - execVtgateQuery(t, vtgateConn, "", deleteOpenTxQuery) + execQueryWithDatabase(t, vtgateConn, "", deleteOpenTxQuery) } ksShards := []string{"product/0", "customer/-80", "customer/80-"} @@ -826,7 +829,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl insertQuery2 = "insert into customer(name, cid) values('tempCustomer4', 102)" // ID 102, hence due to reverse_bits in shard -80 assertQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery2, matchInsertQuery2) - execVtgateQuery(t, vtgateConn, "customer", "update customer set meta = convert(x'7b7d' using utf8mb4) where cid = 1") + execQueryWithDatabase(t, vtgateConn, "customer", "update customer set meta = convert(x'7b7d' using utf8mb4) where cid = 1") if testReverse { // Reverse Replicate switchReads(t, workflowType, cellNames, ksWorkflow, true) @@ -885,13 +888,13 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl insertQuery2 = "insert into customer(name, cid) values('tempCustomer9', 105)" // ID 104, hence due to reverse_bits in shard 80- assertQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery2, matchInsertQuery2) - execVtgateQuery(t, vtgateConn, "customer", "delete from customer where name like 'tempCustomer%'") + execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where name like 'tempCustomer%'") waitForRowCountInTablet(t, customerTab1, "customer", "customer", 1) waitForRowCountInTablet(t, customerTab2, "customer", "customer", 2) waitForRowCount(t, vtgateConn, "customer", "customer.customer", 3) query = "insert into customer (name, cid) values('george', 5)" - execVtgateQuery(t, vtgateConn, "customer", query) + execQueryWithDatabase(t, vtgateConn, "customer", query) waitForRowCountInTablet(t, customerTab1, "customer", "customer", 1) waitForRowCountInTablet(t, customerTab2, "customer", "customer", 3) waitForRowCount(t, vtgateConn, "customer", "customer.customer", 4) @@ -920,7 +923,7 @@ func reshardCustomer2to4Split(t *testing.T, cells []*Cell, sourceCellOrAlias str 600, counts, nil, nil, cells, sourceCellOrAlias, 1) waitForRowCount(t, vtgateConn, ksName, "customer", 20) query := "insert into customer (name) values('yoko')" - execVtgateQuery(t, vtgateConn, ksName, query) + execQueryWithDatabase(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "customer", 21) }) } @@ -935,7 +938,7 @@ func reshardMerchant2to3SplitMerge(t *testing.T) { 1600, counts, dryRunResultsSwitchReadM2m3, dryRunResultsSwitchWritesM2m3, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 2) query := "insert into merchant (mname, category) values('amazon', 'electronics')" - execVtgateQuery(t, vtgateConn, ksName, query) + execQueryWithDatabase(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "merchant", 3) var output string @@ -984,7 +987,7 @@ func reshardMerchant3to1Merge(t *testing.T) { 2000, counts, nil, nil, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 3) query := "insert into merchant (mname, category) values('flipkart', 'electronics')" - execVtgateQuery(t, vtgateConn, ksName, query) + execQueryWithDatabase(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "merchant", 4) }) } diff --git a/go/test/endtoend/vreplication/vstream_test.go b/go/test/endtoend/vreplication/vstream_test.go index e13c3e24e80..b92dbeb9937 100644 --- a/go/test/endtoend/vreplication/vstream_test.go +++ b/go/test/endtoend/vreplication/vstream_test.go @@ -95,7 +95,7 @@ func testVStreamWithFailover(t *testing.T, failover bool) { } insertMu.Lock() id++ - execVtgateQuery(t, vtgateConn, "product", fmt.Sprintf("insert into customer (cid, name) values (%d, 'customer%d')", id+100, id)) + execQueryWithDatabase(t, vtgateConn, "product", fmt.Sprintf("insert into customer (cid, name) values (%d, 'customer%d')", id+100, id)) insertMu.Unlock() } }() @@ -164,7 +164,7 @@ func testVStreamWithFailover(t *testing.T, failover bool) { } } - qr := execVtgateQuery(t, vtgateConn, "product", "select count(*) from customer") + qr := execQueryWithDatabase(t, vtgateConn, "product", "select count(*) from customer") require.NotNil(t, qr) // total number of row events found by the VStream API should match the rows inserted insertedRows, err := qr.Rows[0][0].ToCastInt64() @@ -507,7 +507,7 @@ func testVStreamCopyMultiKeyspaceReshard(t *testing.T, baseTabletID int) numEven // because the keyspace remains unsharded and the number of rows in the customer_seq table is always 1. // We believe that checking the number of row events for the unsharded keyspace, which should always be greater than 0 before and after resharding, // is sufficient to confirm that the resharding of one keyspace does not affect another keyspace, while keeping the test straightforward. - customerResult := execVtgateQuery(t, vtgateConn, "sharded", "select count(*) from customer") + customerResult := execQueryWithDatabase(t, vtgateConn, "sharded", "select count(*) from customer") insertedCustomerRows, err := customerResult.Rows[0][0].ToCastInt64() require.NoError(t, err) require.Equal(t, insertedCustomerRows, ne.numDash80Events+ne.num80DashEvents+ne.numDash40Events+ne.num40DashEvents) @@ -698,7 +698,7 @@ func TestMultiVStreamsKeyspaceReshard(t *testing.T) { require.NotZero(t, newShardRowEvents) // The number of row events streamed by the VStream API should match the number of rows inserted. - customerResult := execVtgateQuery(t, vtgateConn, ks, "select count(*) from customer") + customerResult := execQueryWithDatabase(t, vtgateConn, ks, "select count(*) from customer") customerCount, err := customerResult.Rows[0][0].ToInt64() require.NoError(t, err) require.Equal(t, customerCount, int64(oldShardRowEvents+newShardRowEvents)) From 9fe930fd2d6da6c894fb45c513002897e5a64650 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 19:03:13 -0400 Subject: [PATCH 10/34] Adjust test after merging origin/main Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/resharding_workflows_v2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go index 99dc94cb287..862ab31cb4d 100644 --- a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go +++ b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go @@ -505,7 +505,7 @@ func testVSchemaForSequenceAfterMoveTables(t *testing.T) { execQueryWithDatabase(t, vtgateConn, "product", "insert into customer2(name) values('a')") } waitForRowCount(t, vtgateConn, "product", "customer2", 3+num+num) - res := execVtgateQuery(t, vtgateConn, "product", "select max(cid) from customer2") + res := execQueryWithDatabase(t, vtgateConn, "product", "select max(cid) from customer2") cid, err := res.Rows[0][0].ToInt() require.NoError(t, err) require.GreaterOrEqual(t, cid, 100+num+num-1) From 4ea4dbe6935e09e99d40de98c947c32b53fa3591 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 22 May 2024 19:09:34 -0400 Subject: [PATCH 11/34] Get vrepl e2e tests working Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/helper_test.go | 16 +----- go/vt/topo/shard.go | 8 ++- go/vt/vtctl/workflow/server.go | 54 +++++++++++--------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 398ac42fc00..4cefec97b84 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -89,10 +89,11 @@ func execQueryWithRetry(t *testing.T, conn *mysql.Conn, query string, timeout ti if err == nil { return qr } + log.Infof("Error executing query (will retry): %s: %v", query, err) select { case <-ctx.Done(): require.FailNow(t, fmt.Sprintf("query %q did not succeed before the timeout of %s; last seen result: %v", - query, timeout, qr.Rows)) + query, timeout, qr)) case <-ticker.C: log.Infof("query %q failed with error %v, retrying in %ds", query, err, defaultTick) } @@ -147,19 +148,6 @@ func execQueryWithDatabase(t *testing.T, conn *mysql.Conn, database string, quer return qr } -func execVtgateQueryWithRetry(t *testing.T, conn *mysql.Conn, database string, query string, timeout time.Duration) *sqltypes.Result { - if strings.TrimSpace(query) == "" { - return nil - } - if database != "" { - execQuery(t, conn, "use `"+database+"`;") - } - execQuery(t, conn, "begin") - qr := execQueryWithRetry(t, conn, query, timeout) - execQuery(t, conn, "commit") - return qr -} - func checkHealth(t *testing.T, url string) bool { resp, err := http.Get(url) require.NoError(t, err) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 827b531e6a5..379957797b8 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -22,6 +22,7 @@ import ( "fmt" "path" "reflect" + "slices" "sort" "strings" "sync" @@ -476,7 +477,12 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr return nil } if len(newTables) != len(tables) { - return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, dlTablesAlreadyPresent) + // Some of the tables already existed in the denied list so we don't need to add them. + log.Warningf("%s:%s", dlTablesNotPresent, strings.Join(newTables, ",")) + // We do need to merge the two lists, however. + tables = append(tables, newTables...) + // And be sure to remove any duplicates. + tables = slices.Compact(tables) } tc.DeniedTables = append(tc.DeniedTables, tables...) return nil diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 952e877d94a..054647bb9b0 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1452,16 +1452,31 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl return nil, err } + isStandardMoveTables := func() bool { + return !mz.IsMultiTenantMigration() && !mz.isPartial + } + + ts, err := s.buildTrafficSwitcher(ctx, req.GetTargetKeyspace(), req.GetWorkflow()) + if err != nil { + return nil, err + } + sw := &switcher{s: s, ts: ts} + lockCtx, targetUnlock, lockErr := sw.lockKeyspace(ctx, ts.TargetKeyspaceName(), "MoveTablesCreate") + if lockErr != nil { + ts.Logger().Errorf("Locking target keyspace %s failed: %v", ts.TargetKeyspaceName(), lockErr) + return nil, lockErr + } + defer targetUnlock(&err) + ctx = lockCtx + // If we get an error after this point, where the vreplication streams/records // have been created, then we clean up the workflow's artifacts. defer func() { if err != nil { - ts, cerr := s.buildTrafficSwitcher(ctx, ms.TargetKeyspace, ms.Workflow) - if cerr != nil { - err = vterrors.Wrapf(err, "failed to cleanup workflow artifacts: %v", cerr) - } - if cerr = ts.dropTargetDeniedTables(ctx); cerr != nil { - err = vterrors.Wrapf(err, "failed to cleanup denied table entries: %v", cerr) + if isStandardMoveTables() { // Non-standard ones do not use shard scoped mechanisms + if cerr := ts.dropTargetDeniedTables(ctx); cerr != nil { + err = vterrors.Wrapf(err, "failed to cleanup denied table entries: %v", cerr) + } } if cerr := s.dropArtifacts(ctx, false, &switcher{s: s, ts: ts}); cerr != nil { err = vterrors.Wrapf(err, "failed to cleanup workflow artifacts: %v", cerr) @@ -1487,8 +1502,10 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl return nil, err } } - if err := s.setupInitialDeniedTables(ctx, req); err != nil { - return nil, vterrors.Wrapf(err, "failed to put initial denied tables entries in place on the target shards") + if isStandardMoveTables() { // Non-standard ones do not use shard scoped mechanisms + if err := s.setupInitialDeniedTables(ctx, req, ts); err != nil { + return nil, vterrors.Wrapf(err, "failed to put initial denied tables entries in place on the target shards") + } } if err := s.ts.RebuildSrvVSchema(ctx, nil); err != nil { return nil, err @@ -1553,25 +1570,14 @@ func (s *Server) validateRoutingRuleFlags(req *vtctldatapb.MoveTablesCreateReque return nil } -func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest) error { - ts, err := s.buildTrafficSwitcher(ctx, req.GetTargetKeyspace(), req.GetWorkflow()) - if err != nil { - return err - } - sw := &switcher{s: s, ts: ts} - lockCtx, targetUnlock, lockErr := sw.lockKeyspace(ctx, ts.TargetKeyspaceName(), "SetupInitialDeniedTables") - if lockErr != nil { - ts.Logger().Errorf("Locking target keyspace %s failed: %v", ts.TargetKeyspaceName(), lockErr) - return lockErr - } - defer targetUnlock(&err) - err = ts.ForAllTargets(func(target *MigrationTarget) error { - if _, err := ts.TopoServer().UpdateShardFields(lockCtx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateDeniedTables(lockCtx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) +func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest, ts *trafficSwitcher) error { + err := ts.ForAllTargets(func(target *MigrationTarget) error { + if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) }); err != nil { return err } - strCtx, cancel := context.WithTimeout(lockCtx, shardTabletRefreshTimeout) + strCtx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) defer cancel() _, _, err := topotools.RefreshTabletsByShard(strCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) return err From dddf1c20e008a3803de0655d83415c27d217977d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 23 May 2024 15:56:38 -0400 Subject: [PATCH 12/34] Fix backup tests Signed-off-by: Matt Lord --- go/test/endtoend/cluster/vttablet_process.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index 0c2852b569b..6bd60b63191 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -445,6 +445,9 @@ func (vttablet *VttabletProcess) TearDownWithTimeout(timeout time.Duration) erro // CreateDB creates the database for keyspace func (vttablet *VttabletProcess) CreateDB(keyspace string) error { + if vttablet.DbName == "" { + vttablet.DbName = "vt_" + keyspace + } _, _ = vttablet.QueryTablet(fmt.Sprintf("drop database IF EXISTS %s", vttablet.DbName), keyspace, false) _, err := vttablet.QueryTablet(fmt.Sprintf("create database IF NOT EXISTS %s", vttablet.DbName), keyspace, false) return err From 3758430121b2a0fa6c58ac13ae0378fc6998174b Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 28 May 2024 20:54:28 -0400 Subject: [PATCH 13/34] Correct updatePrimaryTabletControl and test Signed-off-by: Matt Lord --- go/vt/topo/shard.go | 23 ++++++++++++----------- go/vt/topo/shard_test.go | 9 +++++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 379957797b8..2650e962610 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "fmt" "path" - "reflect" "slices" "sort" "strings" @@ -398,16 +397,15 @@ func (si *ShardInfo) UpdateDeniedTables(ctx context.Context, tabletType topodata } tc := si.GetTabletControl(tabletType) if tc == nil { - - // handle the case where the TabletControl object is new + // Handle the case where the TabletControl object is new if remove { - // we try to remove from something that doesn't exist, - // log, but we're done. + // We tried to remove something that doesn't exist, log a warning. + // But we know that our work is done. log.Warningf("Trying to remove TabletControl.DeniedTables for missing type %v in shard %v/%v", tabletType, si.keyspace, si.shardName) return nil } - // trying to add more constraints with no existing record + // Add constraints to the new record. si.TabletControls = append(si.TabletControls, &topodatapb.Shard_TabletControl{ TabletType: tabletType, Cells: cells, @@ -423,16 +421,16 @@ func (si *ShardInfo) UpdateDeniedTables(ctx context.Context, tabletType topodata return nil } - // we have an existing record, check table lists matches and + // We have an existing record, update the table lists. if remove { si.removeCellsFromTabletControl(tc, tabletType, cells) } else { - if !reflect.DeepEqual(tc.DeniedTables, tables) { + if !slices.Equal(tc.DeniedTables, tables) { return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "trying to use two different sets of denied tables for shard %v/%v: %v and %v", si.keyspace, si.shardName, tc.DeniedTables, tables) } - tc.Cells = addCells(tc.Cells, cells) } + return nil } @@ -479,10 +477,13 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr if len(newTables) != len(tables) { // Some of the tables already existed in the denied list so we don't need to add them. log.Warningf("%s:%s", dlTablesNotPresent, strings.Join(newTables, ",")) - // We do need to merge the two lists, however. + // We do need to merge the lists, however. tables = append(tables, newTables...) + tc.DeniedTables = append(tc.DeniedTables, tables...) // And be sure to remove any duplicates. - tables = slices.Compact(tables) + slices.Sort(tc.DeniedTables) + tc.DeniedTables = slices.Compact(tc.DeniedTables) + return nil } tc.DeniedTables = append(tc.DeniedTables, tables...) return nil diff --git a/go/vt/topo/shard_test.go b/go/vt/topo/shard_test.go index ccef80944a9..c7ee685024b 100644 --- a/go/vt/topo/shard_test.go +++ b/go/vt/topo/shard_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/utils" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) @@ -121,14 +122,14 @@ func TestUpdateSourcePrimaryDeniedTables(t *testing.T) { require.NoError(t, addToDenyList(ctx, si, primary, nil, tables2)) validateDenyList(t, si, primary, nil, append(tables1, tables2...)) - require.Error(t, addToDenyList(ctx, si, primary, nil, tables2), dlTablesAlreadyPresent) - require.Error(t, addToDenyList(ctx, si, primary, nil, []string{t1}), dlTablesAlreadyPresent) + require.NoError(t, addToDenyList(ctx, si, primary, nil, tables2)) + require.NoError(t, addToDenyList(ctx, si, primary, nil, []string{t1})) require.NoError(t, removeFromDenyList(ctx, si, primary, nil, tables2)) validateDenyList(t, si, primary, nil, tables1) - require.Error(t, removeFromDenyList(ctx, si, primary, nil, tables2), dlTablesNotPresent) - require.Error(t, removeFromDenyList(ctx, si, primary, nil, []string{t3}), dlTablesNotPresent) + require.NoError(t, removeFromDenyList(ctx, si, primary, nil, tables2)) + require.NoError(t, removeFromDenyList(ctx, si, primary, nil, []string{t3})) validateDenyList(t, si, primary, nil, tables1) require.NoError(t, removeFromDenyList(ctx, si, primary, nil, []string{t1})) From 0c6e12ab27c104b10ddd1638062e1c6acf9580ac Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 29 May 2024 09:36:23 -0400 Subject: [PATCH 14/34] Complete/Cancel fixups Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 28 ++-- .../tabletmanager/rpc_vreplication_test.go | 121 ++++++++++-------- 2 files changed, 78 insertions(+), 71 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 054647bb9b0..b7934b172f0 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "reflect" "slices" "sort" "strings" @@ -2013,14 +2012,11 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe return nil, err } - // There is nothing to drop for a LookupVindex workflow. - if ts.workflowType == binlogdatapb.VReplicationWorkflowType_CreateLookupIndex { - return nil, nil - } - - // Return an error if the workflow traffic is partially switched. - if state.WritesSwitched || len(state.ReplicaCellsSwitched) > 0 || len(state.RdonlyCellsSwitched) > 0 { - return nil, ErrWorkflowPartiallySwitched + if ts.workflowType != binlogdatapb.VReplicationWorkflowType_CreateLookupIndex { + // Return an error if the workflow traffic is partially switched. + if state.WritesSwitched || len(state.ReplicaCellsSwitched) > 0 || len(state.RdonlyCellsSwitched) > 0 { + return nil, ErrWorkflowPartiallySwitched + } } if state.WorkflowType == TypeMigrate { @@ -2054,12 +2050,14 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "the %s workflow does not exist in the %s keyspace", req.Workflow, req.Keyspace) } - // Cleanup related data and artifacts. - if _, err := s.DropTargets(delCtx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { - if topo.IsErrType(err, topo.NoNode) { - return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.GetKeyspace()) + // Cleanup related data and artifacts. There are none for a LookupVindex workflow. + if ts.workflowType != binlogdatapb.VReplicationWorkflowType_CreateLookupIndex { + if _, err := s.DropTargets(delCtx, ts, req.GetKeepData(), req.GetKeepRoutingRules(), false); err != nil { + if topo.IsErrType(err, topo.NoNode) { + return nil, vterrors.Wrapf(err, "%s keyspace does not exist", req.GetKeyspace()) + } + return nil, err } - return nil, err } response := &vtctldatapb.WorkflowDeleteResponse{} @@ -2671,7 +2669,7 @@ func (s *Server) buildTrafficSwitcher(ctx context.Context, targetKeyspace, workf tables = append(tables, rule.Match) } sort.Strings(tables) - if !reflect.DeepEqual(ts.tables, tables) { + if !slices.Equal(ts.tables, tables) { return nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "table lists are mismatched across streams: %v vs %v", ts.tables, tables) } } diff --git a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go index 42e5129b40e..5f8accb543e 100644 --- a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go +++ b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go @@ -50,7 +50,6 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" - vttimepb "vitess.io/vitess/go/vt/proto/vttime" ) const ( @@ -412,6 +411,13 @@ func TestMoveTables(t *testing.T) { ), fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Stopped||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), ), nil) + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Stopped||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), + ), nil) ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowConfig, wf), sqltypes.MakeTestResult( sqltypes.MakeTestFields( "id|source|cell|tablet_types|state|message", @@ -470,70 +476,73 @@ func TestMoveTables(t *testing.T) { AutoStart: true, }) require.NoError(t, err) + /* - for _, ftc := range targetShards { - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, wf), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", wf, vreplID, bls, position, targetKs), - ), nil) - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( + for _, ftc := range targetShards { + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, wf), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", wf, vreplID, bls, position, targetKs), + ), nil) + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), + ), nil) + } + + _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKs, + Workflow: wf, + Cells: tenv.cells, + MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, + EnableReverseReplication: true, + InitializeTargetSequences: true, + Direction: int32(workflow.DirectionForward), + }) + + require.NoError(t, err) + + for _, ftc := range targetShards { + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), + ), nil) + } + addInvariants(sourceTablet.vrdbClient, vreplID, sourceTabletUID, position, workflow.ReverseWorkflowName(wf), tenv.cells[0]) + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, workflow.ReverseWorkflowName(wf), tenv.dbName), sqltypes.MakeTestResult( sqltypes.MakeTestFields( "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, sourceKs), ), nil) - } - - _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ - Keyspace: targetKs, - Workflow: wf, - Cells: tenv.cells, - MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, - EnableReverseReplication: true, - InitializeTargetSequences: true, - Direction: int32(workflow.DirectionForward), - }) - require.NoError(t, err) - - for _, ftc := range targetShards { - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, workflow.ReverseWorkflowName(wf)), sqltypes.MakeTestResult( sqltypes.MakeTestFields( - "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), + fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", workflow.ReverseWorkflowName(wf), vreplID, bls, position, sourceKs), ), nil) - } - addInvariants(sourceTablet.vrdbClient, vreplID, sourceTabletUID, position, workflow.ReverseWorkflowName(wf), tenv.cells[0]) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, workflow.ReverseWorkflowName(wf), tenv.dbName), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, sourceKs), - ), nil) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, workflow.ReverseWorkflowName(wf)), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", workflow.ReverseWorkflowName(wf), vreplID, bls, position, sourceKs), - ), nil) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), &sqltypes.Result{}, nil) - - _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ - Keyspace: targetKs, - Workflow: wf, - Cells: tenv.cells, - MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, - EnableReverseReplication: true, - Direction: int32(workflow.DirectionBackward), - }) - require.NoError(t, err) + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), &sqltypes.Result{}, nil) + + _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKs, + Workflow: wf, + Cells: tenv.cells, + MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, + EnableReverseReplication: true, + Direction: int32(workflow.DirectionBackward), + }) + require.NoError(t, err) + */ } func TestUpdateVReplicationWorkflow(t *testing.T) { From 862bdbacd5466d8f43b7b9442ccf3ed18fdc4d04 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 30 May 2024 16:25:50 -0400 Subject: [PATCH 15/34] Uncomment test Signed-off-by: Matt Lord --- .../tabletmanager/rpc_vreplication_test.go | 116 +++++++++--------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go index 5f8accb543e..691f5a00d63 100644 --- a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go +++ b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go @@ -50,6 +50,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + vttimepb "vitess.io/vitess/go/vt/proto/vttime" ) const ( @@ -476,73 +477,70 @@ func TestMoveTables(t *testing.T) { AutoStart: true, }) require.NoError(t, err) - /* - - for _, ftc := range targetShards { - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, wf), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", wf, vreplID, bls, position, targetKs), - ), nil) - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), - ), nil) - } - - _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ - Keyspace: targetKs, - Workflow: wf, - Cells: tenv.cells, - MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, - EnableReverseReplication: true, - InitializeTargetSequences: true, - Direction: int32(workflow.DirectionForward), - }) - - require.NoError(t, err) - - for _, ftc := range targetShards { - ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( - sqltypes.MakeTestFields( - "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", - ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), - ), nil) - } - addInvariants(sourceTablet.vrdbClient, vreplID, sourceTabletUID, position, workflow.ReverseWorkflowName(wf), tenv.cells[0]) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, workflow.ReverseWorkflowName(wf), tenv.dbName), sqltypes.MakeTestResult( + for _, ftc := range targetShards { + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, wf), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", wf, vreplID, bls, position, targetKs), + ), nil) + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( sqltypes.MakeTestFields( "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", ), - fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, sourceKs), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), ), nil) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, workflow.ReverseWorkflowName(wf)), sqltypes.MakeTestResult( + } + + _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKs, + Workflow: wf, + Cells: tenv.cells, + MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, + EnableReverseReplication: true, + InitializeTargetSequences: true, + Direction: int32(workflow.DirectionForward), + }) + + require.NoError(t, err) + + for _, ftc := range targetShards { + ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), sqltypes.MakeTestResult( sqltypes.MakeTestFields( - "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", - "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", ), - fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", workflow.ReverseWorkflowName(wf), vreplID, bls, position, sourceKs), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, targetKs), ), nil) - sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), &sqltypes.Result{}, nil) - - _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ - Keyspace: targetKs, - Workflow: wf, - Cells: tenv.cells, - MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, - EnableReverseReplication: true, - Direction: int32(workflow.DirectionBackward), - }) - require.NoError(t, err) - */ + } + addInvariants(sourceTablet.vrdbClient, vreplID, sourceTabletUID, position, workflow.ReverseWorkflowName(wf), tenv.cells[0]) + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, workflow.ReverseWorkflowName(wf), tenv.dbName), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", vreplID, bls, position, sourceKs), + ), nil) + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, workflow.ReverseWorkflowName(wf)), sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "workflow|id|source|pos|stop_pos|max_tps|max_replication_lag|cell|tablet_types|time_updated|transaction_timestamp|state|message|db_name|rows_copied|tags|time_heartbeat|workflow_type|time_throttled|component_throttled|workflow_sub_type|defer_secondary_keys", + "workflow|int64|varchar|blob|varchar|int64|int64|varchar|varchar|int64|int64|varchar|varchar|varchar|int64|varchar|int64|int64|int64|varchar|int64|int64", + ), + fmt.Sprintf("%s|%d|%s|%s|NULL|0|0|||1686577659|0|Running||%s|1||0|0|0||0|1", workflow.ReverseWorkflowName(wf), vreplID, bls, position, sourceKs), + ), nil) + sourceTablet.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflow, wf, tenv.dbName), &sqltypes.Result{}, nil) + + _, err = ws.WorkflowSwitchTraffic(ctx, &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKs, + Workflow: wf, + Cells: tenv.cells, + MaxReplicationLagAllowed: &vttimepb.Duration{Seconds: 922337203}, + EnableReverseReplication: true, + Direction: int32(workflow.DirectionBackward), + }) + require.NoError(t, err) } func TestUpdateVReplicationWorkflow(t *testing.T) { From 5b4c288b39f8c4310e2eba37f13233b0d1460847 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 3 Jun 2024 22:25:17 -0400 Subject: [PATCH 16/34] Unify denied table rules management when doing traffic switches Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server.go | 10 +-- go/vt/vtctl/workflow/traffic_switcher.go | 84 +++++++++++++----------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index b7934b172f0..f1c5323b8db 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -1502,7 +1502,7 @@ func (s *Server) moveTablesCreate(ctx context.Context, req *vtctldatapb.MoveTabl } } if isStandardMoveTables() { // Non-standard ones do not use shard scoped mechanisms - if err := s.setupInitialDeniedTables(ctx, req, ts); err != nil { + if err := s.setupInitialDeniedTables(ctx, ts); err != nil { return nil, vterrors.Wrapf(err, "failed to put initial denied tables entries in place on the target shards") } } @@ -1569,8 +1569,11 @@ func (s *Server) validateRoutingRuleFlags(req *vtctldatapb.MoveTablesCreateReque return nil } -func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest, ts *trafficSwitcher) error { - err := ts.ForAllTargets(func(target *MigrationTarget) error { +func (s *Server) setupInitialDeniedTables(ctx context.Context, ts *trafficSwitcher) error { + if ts.MigrationType() != binlogdatapb.MigrationType_TABLES { + return nil + } + return ts.ForAllTargets(func(target *MigrationTarget) error { if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) }); err != nil { @@ -1581,7 +1584,6 @@ func (s *Server) setupInitialDeniedTables(ctx context.Context, req *vtctldatapb. _, _, err := topotools.RefreshTabletsByShard(strCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) return err }) - return err } func (s *Server) setupInitialRoutingRules(ctx context.Context, req *vtctldatapb.MoveTablesCreateRequest, mz *materializer, tables []string) error { diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index 16027d1dba2..d3d479beec4 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -712,25 +712,11 @@ func (ts *trafficSwitcher) changeShardsAccess(ctx context.Context, keyspace stri func (ts *trafficSwitcher) allowTargetWrites(ctx context.Context) error { if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { - return ts.allowTableTargetWrites(ctx) + return ts.switchDeniedTables(ctx) } return ts.changeShardsAccess(ctx, ts.TargetKeyspaceName(), ts.TargetShards(), allowWrites) } -func (ts *trafficSwitcher) allowTableTargetWrites(ctx context.Context) error { - return ts.ForAllTargets(func(target *MigrationTarget) error { - if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, true, ts.Tables()) - }); err != nil { - return err - } - rtbsCtx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) - defer cancel() - _, _, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) - return err - }) -} - func (ts *trafficSwitcher) changeRouting(ctx context.Context) error { if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { return ts.changeWriteRoute(ctx) @@ -1025,7 +1011,7 @@ func (ts *trafficSwitcher) waitForCatchup(ctx context.Context, filteredReplicati func (ts *trafficSwitcher) stopSourceWrites(ctx context.Context) error { var err error if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { - err = ts.changeTableSourceWrites(ctx, disallowWrites) + err = ts.switchDeniedTables(ctx) } else { err = ts.changeShardsAccess(ctx, ts.SourceKeyspaceName(), ts.SourceShards(), disallowWrites) } @@ -1045,36 +1031,60 @@ func (ts *trafficSwitcher) stopSourceWrites(ctx context.Context) error { }) } -func (ts *trafficSwitcher) changeTableSourceWrites(ctx context.Context, access accessType) error { - err := ts.ForAllSources(func(source *MigrationSource) error { - if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.SourceKeyspaceName(), source.GetShard().ShardName(), func(si *topo.ShardInfo) error { - return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, access == allowWrites /* remove */, ts.Tables()) - }); err != nil { +// switchDeniedTables switches the denied tables rules for the traffic switch. +// They are removed on the source side and added on the target side. +func (ts *trafficSwitcher) switchDeniedTables(ctx context.Context) error { + if ts.MigrationType() != binlogdatapb.MigrationType_TABLES { + return nil + } + + egrp, ectx := errgroup.WithContext(ctx) + egrp.Go(func() error { + return ts.ForAllSources(func(source *MigrationSource) error { + if _, err := ts.TopoServer().UpdateShardFields(ctx, ts.SourceKeyspaceName(), source.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(ectx, topodatapb.TabletType_PRIMARY, nil, false, ts.Tables()) + }); err != nil { + return err + } + rtbsCtx, cancel := context.WithTimeout(ectx, shardTabletRefreshTimeout) + defer cancel() + isPartial, partialDetails, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), source.GetShard(), nil, ts.Logger()) + if isPartial { + err = fmt.Errorf("failed to successfully refresh all tablets in the %s/%s source shard (%v):\n %v", + source.GetShard().Keyspace(), source.GetShard().ShardName(), err, partialDetails) + } return err - } - rtbsCtx, cancel := context.WithTimeout(ctx, shardTabletRefreshTimeout) - defer cancel() - isPartial, partialDetails, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), source.GetShard(), nil, ts.Logger()) - if isPartial { - err = fmt.Errorf("failed to successfully refresh all tablets in the %s/%s source shard (%v):\n %v", - source.GetShard().Keyspace(), source.GetShard().ShardName(), err, partialDetails) - } - return err + }) }) - if err != nil { - log.Warningf("Error in changeTableSourceWrites: %s", err) + egrp.Go(func() error { + return ts.ForAllTargets(func(target *MigrationTarget) error { + if _, err := ts.TopoServer().UpdateShardFields(ectx, ts.TargetKeyspaceName(), target.GetShard().ShardName(), func(si *topo.ShardInfo) error { + return si.UpdateDeniedTables(ctx, topodatapb.TabletType_PRIMARY, nil, true, ts.Tables()) + }); err != nil { + return err + } + rtbsCtx, cancel := context.WithTimeout(ectx, shardTabletRefreshTimeout) + defer cancel() + isPartial, partialDetails, err := topotools.RefreshTabletsByShard(rtbsCtx, ts.TopoServer(), ts.TabletManagerClient(), target.GetShard(), nil, ts.Logger()) + if isPartial { + err = fmt.Errorf("failed to successfully refresh all tablets in the %s/%s target shard (%v):\n %v", + target.GetShard().Keyspace(), target.GetShard().ShardName(), err, partialDetails) + } + return err + }) + }) + if err := egrp.Wait(); err != nil { + log.Warningf("Error in switchDeniedTables: %s", err) return err } - // Note that the denied tables, which are being updated in this method, are not part of the SrvVSchema in the topo. - // However, we are using the notification of a SrvVSchema change in VTGate to recompute the state of a - // MoveTables workflow (which also looks up denied tables from the topo). So we need to trigger a SrvVSchema change here. - return ts.TopoServer().RebuildSrvVSchema(ctx, nil) + + return nil } func (ts *trafficSwitcher) cancelMigration(ctx context.Context, sm *StreamMigrator) { var err error if ts.MigrationType() == binlogdatapb.MigrationType_TABLES { - err = ts.changeTableSourceWrites(ctx, allowWrites) + err = ts.switchDeniedTables(ctx) } else { err = ts.changeShardsAccess(ctx, ts.SourceKeyspaceName(), ts.SourceShards(), allowWrites) } From e5017114fd106b13254c4a9b21cbf8e05953c8a6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 4 Jun 2024 15:10:59 -0400 Subject: [PATCH 17/34] Add unit test framework and test cases (scaffolding) Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 415 ++++++++++++++++++ go/vt/vtctl/workflow/materializer_env_test.go | 16 +- go/vt/vtctl/workflow/server.go | 3 + go/vt/vtctl/workflow/server_test.go | 122 +++-- 4 files changed, 503 insertions(+), 53 deletions(-) create mode 100644 go/vt/vtctl/workflow/framework_test.go diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go new file mode 100644 index 00000000000..65c3835ca19 --- /dev/null +++ b/go/vt/vtctl/workflow/framework_test.go @@ -0,0 +1,415 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "context" + "fmt" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" + "google.golang.org/protobuf/proto" + + "vitess.io/vitess/go/protoutil" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/mysqlctl/tmutils" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vtenv" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vttablet/tmclient" + + _flag "vitess.io/vitess/go/internal/flag" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" +) + +const defaultCellName = "cell" + +type testKeyspace struct { + KeyspaceName string + ShardNames []string +} + +type queryResult struct { + query string + result *querypb.QueryResult +} + +type testEnv struct { + ws *Server + ts *topo.Server + tmc *testTMClient + sourceKeyspace, targetKeyspace *testKeyspace + // Keyed first by keyspace name, then tablet UID. + tablets map[string]map[int]*topodatapb.Tablet + cell string +} + +//---------------------------------------------- +// testEnv + +func TestMain(m *testing.M) { + _flag.ParseFlagsForTest() + os.Exit(m.Run()) +} + +func newTestEnv(t *testing.T, ctx context.Context, cell string, sourceKeyspace, targetKeyspace *testKeyspace) *testEnv { + t.Helper() + env := &testEnv{ + ts: memorytopo.NewServer(ctx, defaultCellName), + sourceKeyspace: sourceKeyspace, + targetKeyspace: targetKeyspace, + tablets: make(map[string]map[int]*topodatapb.Tablet), + cell: cell, + } + venv := vtenv.NewTestEnv() + env.tmc = newTestTMClient(env) + env.ws = NewServer(venv, env.ts, env.tmc) + + tabletID := 100 + for _, shardName := range sourceKeyspace.ShardNames { + _ = env.addTablet(tabletID, sourceKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) + tabletID += 10 + } + if sourceKeyspace.KeyspaceName != targetKeyspace.KeyspaceName { + tabletID = 200 + for _, shardName := range targetKeyspace.ShardNames { + _ = env.addTablet(tabletID, targetKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) + tabletID += 10 + } + } + err := env.ts.RebuildSrvVSchema(ctx, nil) + require.NoError(t, err) + + return env +} + +func (env *testEnv) close() { + for _, k := range maps.Values(env.tablets) { + for _, t := range maps.Values(k) { + env.deleteTablet(t) + } + } +} + +func (env *testEnv) addTablet(id int, keyspace, shard string, tabletType topodatapb.TabletType) *topodatapb.Tablet { + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: env.cell, + Uid: uint32(id), + }, + Keyspace: keyspace, + Shard: shard, + KeyRange: &topodatapb.KeyRange{}, + Type: tabletType, + PortMap: map[string]int32{ + "test": int32(id), + }, + } + if env.tablets[keyspace] == nil { + env.tablets[keyspace] = make(map[int]*topodatapb.Tablet) + } + env.tablets[keyspace][id] = tablet + if err := env.ws.ts.InitTablet(context.Background(), tablet, false /* allowPrimaryOverride */, true /* createShardAndKeyspace */, false /* allowUpdate */); err != nil { + panic(err) + } + if tabletType == topodatapb.TabletType_PRIMARY { + _, err := env.ws.ts.UpdateShardFields(context.Background(), keyspace, shard, func(si *topo.ShardInfo) error { + si.PrimaryAlias = tablet.Alias + si.IsPrimaryServing = true + return nil + }) + if err != nil { + panic(err) + } + } + return tablet +} + +func (env *testEnv) deleteTablet(tablet *topodatapb.Tablet) { + _ = env.ts.DeleteTablet(context.Background(), tablet.Alias) + delete(env.tablets[tablet.Keyspace], int(tablet.Alias.Uid)) +} + +//---------------------------------------------- +// testTMClient + +type testTMClient struct { + tmclient.TabletManagerClient + schema map[string]*tabletmanagerdatapb.SchemaDefinition + + mu sync.Mutex + vrQueries map[int][]*queryResult + createVReplicationWorkflowRequests map[uint32]*tabletmanagerdatapb.CreateVReplicationWorkflowRequest + + // Used to confirm the number of times WorkflowDelete was called. + workflowDeleteCalls int + + env *testEnv +} + +func newTestTMClient(env *testEnv) *testTMClient { + return &testTMClient{ + schema: make(map[string]*tabletmanagerdatapb.SchemaDefinition), + vrQueries: make(map[int][]*queryResult), + createVReplicationWorkflowRequests: make(map[uint32]*tabletmanagerdatapb.CreateVReplicationWorkflowRequest), + env: env, + } +} + +func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) (*tabletmanagerdatapb.CreateVReplicationWorkflowResponse, error) { + if expect := tmc.createVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { + if !proto.Equal(expect, request) { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected CreateVReplicationWorkflow request: got %+v, want %+v", request, expect) + } + } + res := sqltypes.MakeTestResult(sqltypes.MakeTestFields("rowsaffected", "int64"), "1") + return &tabletmanagerdatapb.CreateVReplicationWorkflowResponse{Result: sqltypes.ResultToProto3(res)}, nil +} + +func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.ReadVReplicationWorkflowRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowResponse, error) { + workflowType := binlogdatapb.VReplicationWorkflowType_MoveTables + if strings.Contains(request.Workflow, "lookup") { + workflowType = binlogdatapb.VReplicationWorkflowType_CreateLookupIndex + } + res := &tabletmanagerdatapb.ReadVReplicationWorkflowResponse{ + Workflow: request.Workflow, + WorkflowType: workflowType, + Streams: make([]*tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream, len(tmc.env.sourceKeyspace.ShardNames)), + } + rules := make([]*binlogdatapb.Rule, len(tmc.schema)) + for i, table := range maps.Keys(tmc.schema) { + rules[i] = &binlogdatapb.Rule{ + Match: table, + Filter: fmt.Sprintf("select * from %s", table), + } + } + for i, shard := range tmc.env.sourceKeyspace.ShardNames { + res.Streams[i] = &tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream{ + Id: int32(i + 1), + Bls: &binlogdatapb.BinlogSource{ + Keyspace: tmc.env.sourceKeyspace.KeyspaceName, + Shard: shard, + Tables: maps.Keys(tmc.schema), + Filter: &binlogdatapb.Filter{ + Rules: rules, + }, + }, + } + } + + return res, nil +} + +func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (response *tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, err error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + tmc.workflowDeleteCalls++ + return &tabletmanagerdatapb.DeleteVReplicationWorkflowResponse{ + Result: &querypb.QueryResult{ + RowsAffected: 1, + }, + }, nil +} + +func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.GetSchemaRequest) (*tabletmanagerdatapb.SchemaDefinition, error) { + schemaDefn := &tabletmanagerdatapb.SchemaDefinition{} + for _, table := range request.Tables { + if table == "/.*/" { + // Special case of all tables in keyspace. + for key, tableDefn := range tmc.schema { + if strings.HasPrefix(key, tablet.Keyspace+".") { + schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) + } + } + break + } + + key := tablet.Keyspace + "." + table + tableDefn := tmc.schema[key] + if tableDefn == nil { + continue + } + schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) + } + return schemaDefn, nil +} + +func (tmc *testTMClient) expectVRQuery(tabletID int, query string, result *sqltypes.Result) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + tmc.vrQueries[tabletID] = append(tmc.vrQueries[tabletID], &queryResult{ + query: query, + result: sqltypes.ResultToProto3(result), + }) +} + +func (tmc *testTMClient) expectVRQueryResultOnKeyspaceTablets(keyspace string, queryResult *queryResult) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + for uid := range tmc.env.tablets[keyspace] { + tmc.vrQueries[uid] = append(tmc.vrQueries[uid], queryResult) + } +} + +func (tmc *testTMClient) expectCreateVReplicationWorkflowRequest(tabletID uint32, req *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + tmc.createVReplicationWorkflowRequests[tabletID] = req +} + +func (tmc *testTMClient) verifyQueries(t *testing.T) { + t.Helper() + tmc.mu.Lock() + defer tmc.mu.Unlock() + + for tabletID, qrs := range tmc.vrQueries { + if len(qrs) != 0 { + var list []string + for _, qr := range qrs { + list = append(list, qr.query) + } + t.Errorf("tablet %v: found queries that were expected but never got executed by the test: %v", tabletID, list) + } + } +} + +func (tmc *testTMClient) VReplicationExec(ctx context.Context, tablet *topodatapb.Tablet, query string) (*querypb.QueryResult, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + + qrs := tmc.vrQueries[int(tablet.Alias.Uid)] + if len(qrs) == 0 { + return nil, fmt.Errorf("tablet %v does not expect any more queries: %s", tablet, query) + } + matched := false + if qrs[0].query[0] == '/' { + matched = regexp.MustCompile(qrs[0].query[1:]).MatchString(query) + } else { + matched = query == qrs[0].query + } + if !matched { + return nil, fmt.Errorf("tablet %v:\nunexpected query\n%s\nwant:\n%s", tablet, query, qrs[0].query) + } + tmc.vrQueries[int(tablet.Alias.Uid)] = qrs[1:] + return qrs[0].result, nil +} + +func (tmc *testTMClient) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteFetchAsDbaRequest) (*querypb.QueryResult, error) { + // Reuse VReplicationExec + return tmc.VReplicationExec(ctx, tablet, string(req.Query)) +} + +func (tmc *testTMClient) ExecuteFetchAsAllPrivs(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.ExecuteFetchAsAllPrivsRequest) (*querypb.QueryResult, error) { + return nil, nil +} + +// Note: ONLY breaks up change.SQL into individual statements and executes it. Does NOT fully implement ApplySchema. +func (tmc *testTMClient) ApplySchema(ctx context.Context, tablet *topodatapb.Tablet, change *tmutils.SchemaChange) (*tabletmanagerdatapb.SchemaChangeResult, error) { + stmts := strings.Split(change.SQL, ";") + + for _, stmt := range stmts { + _, err := tmc.ExecuteFetchAsDba(ctx, tablet, false, &tabletmanagerdatapb.ExecuteFetchAsDbaRequest{ + Query: []byte(stmt), + MaxRows: 0, + ReloadSchema: true, + }) + if err != nil { + return nil, err + } + } + + return nil, nil +} + +func (tmc *testTMClient) VDiff(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.VDiffRequest) (*tabletmanagerdatapb.VDiffResponse, error) { + return &tabletmanagerdatapb.VDiffResponse{ + Id: 1, + VdiffUuid: req.VdiffUuid, + Output: &querypb.QueryResult{ + RowsAffected: 1, + }, + }, nil +} + +func (tmc *testTMClient) HasVReplicationWorkflows(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.HasVReplicationWorkflowsRequest) (*tabletmanagerdatapb.HasVReplicationWorkflowsResponse, error) { + return &tabletmanagerdatapb.HasVReplicationWorkflowsResponse{ + Has: false, + }, nil +} + +func (tmc *testTMClient) ReadVReplicationWorkflows(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.ReadVReplicationWorkflowsRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowsResponse, error) { + workflowType := binlogdatapb.VReplicationWorkflowType_MoveTables + if len(req.IncludeWorkflows) > 0 { + for _, wf := range req.IncludeWorkflows { + if strings.Contains(wf, "lookup") { + workflowType = binlogdatapb.VReplicationWorkflowType_CreateLookupIndex + } + } + return &tabletmanagerdatapb.ReadVReplicationWorkflowsResponse{ + Workflows: []*tabletmanagerdatapb.ReadVReplicationWorkflowResponse{ + { + Workflow: req.IncludeWorkflows[0], + WorkflowType: workflowType, + Streams: []*tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream{ + { + Id: 1, + State: binlogdatapb.VReplicationWorkflowState_Running, + Bls: &binlogdatapb.BinlogSource{ + Keyspace: tmc.env.sourceKeyspace.KeyspaceName, + Shard: tmc.env.sourceKeyspace.ShardNames[0], + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{ + { + Match: "/.*/", + }, + }, + }, + }, + Pos: "MySQL56/" + position, + TimeUpdated: protoutil.TimeToProto(time.Now()), + TimeHeartbeat: protoutil.TimeToProto(time.Now()), + }, + }, + }, + }, + }, nil + } else { + return &tabletmanagerdatapb.ReadVReplicationWorkflowsResponse{}, nil + } +} + +func (tmc *testTMClient) UpdateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.UpdateVReplicationWorkflowRequest) (*tabletmanagerdatapb.UpdateVReplicationWorkflowResponse, error) { + return &tabletmanagerdatapb.UpdateVReplicationWorkflowResponse{ + Result: &querypb.QueryResult{ + RowsAffected: 1, + }, + }, nil +} diff --git a/go/vt/vtctl/workflow/materializer_env_test.go b/go/vt/vtctl/workflow/materializer_env_test.go index 587712f620c..fe49b24a10d 100644 --- a/go/vt/vtctl/workflow/materializer_env_test.go +++ b/go/vt/vtctl/workflow/materializer_env_test.go @@ -19,7 +19,6 @@ package workflow import ( "context" "fmt" - "os" "regexp" "strings" "sync" @@ -37,7 +36,6 @@ import ( "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" - _flag "vitess.io/vitess/go/internal/flag" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" @@ -46,11 +44,6 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -type queryResult struct { - query string - result *querypb.QueryResult -} - type testMaterializerEnv struct { ws *Server ms *vtctldatapb.MaterializeSettings @@ -65,11 +58,6 @@ type testMaterializerEnv struct { //---------------------------------------------- // testMaterializerEnv -func TestMain(m *testing.M) { - _flag.ParseFlagsForTest() - os.Exit(m.Run()) -} - func newTestMaterializerEnv(t *testing.T, ctx context.Context, ms *vtctldatapb.MaterializeSettings, sources, targets []string) *testMaterializerEnv { t.Helper() env := &testMaterializerEnv{ @@ -77,8 +65,8 @@ func newTestMaterializerEnv(t *testing.T, ctx context.Context, ms *vtctldatapb.M sources: sources, targets: targets, tablets: make(map[int]*topodatapb.Tablet), - topoServ: memorytopo.NewServer(ctx, "cell"), - cell: "cell", + topoServ: memorytopo.NewServer(ctx, defaultCellName), + cell: defaultCellName, tmc: newTestMaterializerTMClient(), } venv := vtenv.NewTestEnv() diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index f1c5323b8db..0dc9be308ae 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -2072,6 +2072,9 @@ func (s *Server) WorkflowDelete(ctx context.Context, req *vtctldatapb.WorkflowDe } details = append(details, result) } + sort.Slice(details, func(i, j int) bool { // Ensure deterministic output + return topoproto.TabletAliasString(details[i].Tablet) < topoproto.TabletAliasString(details[j].Tablet) + }) response.Details = details return response, nil } diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 905240d59fb..f97fe82c200 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -34,6 +35,7 @@ import ( binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -200,61 +202,103 @@ func TestVDiffCreate(t *testing.T) { } } -/* TODO: get the framework in place for this and then get the test working func TestWorkflowDelete(t *testing.T) { - ctx := context.Background() - ksName := "ks1" - wfName := "wf1" - cellName := "cell1" + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() - ts := memorytopo.NewServer(ctx, cellName) - tmc := &fakeTMC{} - s := NewServer(vtenv.NewTestEnv(), ts, tmc) - err := s.ts.CreateKeyspace(ctx, ksName, &topodatapb.Keyspace{}) - require.NoError(t, err) - err = s.ts.CreateShard(ctx, ksName, "0") - require.NoError(t, err) - err = s.ts.CreateTablet(ctx, &topodatapb.Tablet{ - Hostname: "host1", - Keyspace: ksName, - Shard: "0", - Alias: &topodatapb.TabletAlias{ - Cell: cellName, - Uid: 100, + workflowName := "wf1" + tableName := "t1" + sourceKeyspaceName := "sourceks" + targetKeyspaceName := "targetks" + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + "t1": { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, }, - }) - require.NoError(t, err) + } testcases := []struct { - name string - req *vtctldatapb.WorkflowDeleteRequest - want *vtctldatapb.WorkflowDeleteResponse - wantErr bool + name string + sourceKeyspace, targetKeyspace *testKeyspace + req *vtctldatapb.WorkflowDeleteRequest + expectedSourceQueries []*queryResult + expectedTargetQueries []*queryResult + want *vtctldatapb.WorkflowDeleteResponse + wantErr bool }{ { - name: "no values", + name: "basic", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, req: &vtctldatapb.WorkflowDeleteRequest{ - Keyspace: ksName, - Workflow: wfName, + Keyspace: targetKeyspaceName, + Workflow: workflowName, + }, + expectedSourceQueries: []*queryResult{ + { + query: fmt.Sprintf("delete from _vt.vreplication where db_name = 'vt_%s' and workflow = '%s'", + sourceKeyspaceName, ReverseWorkflowName(workflowName)), + result: &querypb.QueryResult{}, + }, + }, + expectedTargetQueries: []*queryResult{ + { + query: fmt.Sprintf("drop table `vt_%s`.`%s`", targetKeyspaceName, tableName), + result: &querypb.QueryResult{}, + }, + }, + want: &vtctldatapb.WorkflowDeleteResponse{ + Summary: fmt.Sprintf("Successfully cancelled the %s workflow in the %s keyspace", + workflowName, targetKeyspaceName), + Details: []*vtctldatapb.WorkflowDeleteResponse_TabletInfo{ + { + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: 200}, + Deleted: true, + }, + { + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: 210}, + Deleted: true, + }, + }, }, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - res, err := s.MoveTablesCreate(ctx, &vtctldatapb.MoveTablesCreateRequest{ - TargetKeyspace: ksName, - Workflow: wfName, - AllTables: true, - }) - require.NoError(t, err) - require.NotNil(t, res) - got, err := s.WorkflowDelete(ctx, tc.req) + require.NotNil(t, tc.sourceKeyspace) + require.NotNil(t, tc.targetKeyspace) + require.NotNil(t, tc.req) + env := newTestEnv(t, ctx, defaultCellName, tc.sourceKeyspace, tc.targetKeyspace) + defer env.close() + env.tmc.schema = schema + if tc.expectedSourceQueries != nil { + require.NotNil(t, env.tablets[tc.sourceKeyspace.KeyspaceName]) + for _, eq := range tc.expectedSourceQueries { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, eq) + } + } + if tc.expectedTargetQueries != nil { + require.NotNil(t, env.tablets[tc.targetKeyspace.KeyspaceName]) + for _, eq := range tc.expectedTargetQueries { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, eq) + } + } + got, err := env.ws.WorkflowDelete(ctx, tc.req) if (err != nil) != tc.wantErr { - t.Errorf("Server.WorkflowDelete() error = %v, wantErr %v", err, tc.wantErr) + require.Fail(t, "unexpected error value", "Server.WorkflowDelete() error = %v, wantErr %v", err, tc.wantErr) return } - require.Equal(t, got, tc.want, "Server.WorkflowDelete() = %v, want %v", got, tc.want) + require.EqualValues(t, got, tc.want, "Server.WorkflowDelete() = %v, want %v", got, tc.want) }) } } -*/ From e5f8acd545526cb2066f5b54b3a7afd18adec86e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 4 Jun 2024 18:52:05 -0400 Subject: [PATCH 18/34] Ongoing improvements Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 15 ++-- go/vt/vtctl/workflow/server_test.go | 94 +++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index 65c3835ca19..a605a7ff477 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -47,7 +47,12 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -const defaultCellName = "cell" +const ( + defaultCellName = "cell" + startingSourceTabletUID = 100 + startingTargetTabletUID = 200 + tabletUIDStep = 10 +) type testKeyspace struct { KeyspaceName string @@ -90,16 +95,16 @@ func newTestEnv(t *testing.T, ctx context.Context, cell string, sourceKeyspace, env.tmc = newTestTMClient(env) env.ws = NewServer(venv, env.ts, env.tmc) - tabletID := 100 + tabletID := startingSourceTabletUID for _, shardName := range sourceKeyspace.ShardNames { _ = env.addTablet(tabletID, sourceKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) - tabletID += 10 + tabletID += tabletUIDStep } if sourceKeyspace.KeyspaceName != targetKeyspace.KeyspaceName { - tabletID = 200 + tabletID = startingTargetTabletUID for _, shardName := range targetKeyspace.ShardNames { _ = env.addTablet(tabletID, targetKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) - tabletID += 10 + tabletID += tabletUIDStep } } err := env.ts.RebuildSrvVSchema(ctx, nil) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index f97fe82c200..08bd7e17862 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -28,6 +28,7 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtenv" @@ -224,11 +225,13 @@ func TestWorkflowDelete(t *testing.T) { testcases := []struct { name string sourceKeyspace, targetKeyspace *testKeyspace + preFunc func(t *testing.T, env *testEnv) req *vtctldatapb.WorkflowDeleteRequest expectedSourceQueries []*queryResult expectedTargetQueries []*queryResult want *vtctldatapb.WorkflowDeleteResponse wantErr bool + postFunc func(t *testing.T, env *testEnv) }{ { name: "basic", @@ -262,16 +265,82 @@ func TestWorkflowDelete(t *testing.T) { workflowName, targetKeyspaceName), Details: []*vtctldatapb.WorkflowDeleteResponse_TabletInfo{ { - Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: 200}, + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: startingTargetTabletUID}, Deleted: true, }, { - Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: 210}, + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: startingTargetTabletUID + tabletUIDStep}, Deleted: true, }, }, }, }, + { + name: "basic with existing denied table entries", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, + preFunc: func(t *testing.T, env *testEnv) { + lockCtx, targetUnlock, lockErr := env.ts.LockKeyspace(ctx, targetKeyspaceName, "test") + require.NoError(t, lockErr) + var err error + defer require.NoError(t, err) + defer targetUnlock(&err) + for _, shard := range env.targetKeyspace.ShardNames { + _, err := env.ts.UpdateShardFields(lockCtx, targetKeyspaceName, shard, func(si *topo.ShardInfo) error { + err := si.UpdateDeniedTables(lockCtx, topodatapb.TabletType_PRIMARY, nil, false, []string{tableName, "t2", "t3"}) + return err + }) + require.NoError(t, err) + } + }, + req: &vtctldatapb.WorkflowDeleteRequest{ + Keyspace: targetKeyspaceName, + Workflow: workflowName, + }, + expectedSourceQueries: []*queryResult{ + { + query: fmt.Sprintf("delete from _vt.vreplication where db_name = 'vt_%s' and workflow = '%s'", + sourceKeyspaceName, ReverseWorkflowName(workflowName)), + result: &querypb.QueryResult{}, + }, + }, + expectedTargetQueries: []*queryResult{ + { + query: fmt.Sprintf("drop table `vt_%s`.`%s`", targetKeyspaceName, tableName), + result: &querypb.QueryResult{}, + }, + }, + want: &vtctldatapb.WorkflowDeleteResponse{ + Summary: fmt.Sprintf("Successfully cancelled the %s workflow in the %s keyspace", + workflowName, targetKeyspaceName), + Details: []*vtctldatapb.WorkflowDeleteResponse_TabletInfo{ + { + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: startingTargetTabletUID}, + Deleted: true, + }, + { + Tablet: &topodatapb.TabletAlias{Cell: defaultCellName, Uid: startingTargetTabletUID + tabletUIDStep}, + Deleted: true, + }, + }, + }, + postFunc: func(t *testing.T, env *testEnv) { + for _, shard := range env.targetKeyspace.ShardNames { + si, err := env.ts.GetShard(ctx, targetKeyspaceName, shard) + require.NoError(t, err) + require.NotNil(t, si) + tc := si.GetTabletControl(topodatapb.TabletType_PRIMARY) + require.NotNil(t, tc) + require.EqualValues(t, []string{"t2", "t3"}, tc.DeniedTables) + } + }, + }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -293,12 +362,33 @@ func TestWorkflowDelete(t *testing.T) { env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, eq) } } + if tc.preFunc != nil { + tc.preFunc(t, env) + } got, err := env.ws.WorkflowDelete(ctx, tc.req) if (err != nil) != tc.wantErr { require.Fail(t, "unexpected error value", "Server.WorkflowDelete() error = %v, wantErr %v", err, tc.wantErr) return } require.EqualValues(t, got, tc.want, "Server.WorkflowDelete() = %v, want %v", got, tc.want) + if tc.postFunc != nil { + tc.postFunc(t, env) + } else { // Default post checks + // Confirm that we have no routing rules. + rr, err := env.ts.GetRoutingRules(ctx) + require.NoError(t, err) + require.Zero(t, rr.Rules) + + // Confirm that we have no shard tablet controls, which is where + // DeniedTables live. + for _, keyspace := range []*testKeyspace{tc.sourceKeyspace, tc.targetKeyspace} { + for _, shardName := range keyspace.ShardNames { + si, err := env.ts.GetShard(ctx, keyspace.KeyspaceName, shardName) + require.NoError(t, err) + require.Zero(t, si.Shard.TabletControls) + } + } + } }) } } From c80f8a3ce3e5bc1c3be726df5f5b240c370c0adc Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 4 Jun 2024 21:04:05 -0400 Subject: [PATCH 19/34] Add scaffolding for MoveTables traffic switching unit test Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 7 ++ go/vt/vtctl/workflow/server_test.go | 95 ++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index a605a7ff477..4b96ef3a42f 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -170,6 +170,7 @@ type testTMClient struct { mu sync.Mutex vrQueries map[int][]*queryResult createVReplicationWorkflowRequests map[uint32]*tabletmanagerdatapb.CreateVReplicationWorkflowRequest + readVReplicationWorkflowRequests map[uint32]*tabletmanagerdatapb.ReadVReplicationWorkflowRequest // Used to confirm the number of times WorkflowDelete was called. workflowDeleteCalls int @@ -182,6 +183,7 @@ func newTestTMClient(env *testEnv) *testTMClient { schema: make(map[string]*tabletmanagerdatapb.SchemaDefinition), vrQueries: make(map[int][]*queryResult), createVReplicationWorkflowRequests: make(map[uint32]*tabletmanagerdatapb.CreateVReplicationWorkflowRequest), + readVReplicationWorkflowRequests: make(map[uint32]*tabletmanagerdatapb.ReadVReplicationWorkflowRequest), env: env, } } @@ -197,6 +199,11 @@ func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet } func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.ReadVReplicationWorkflowRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowResponse, error) { + if expect := tmc.readVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { + if !proto.Equal(expect, request) { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ReadVReplicationWorkflow request: got %+v, want %+v", request, expect) + } + } workflowType := binlogdatapb.VReplicationWorkflowType_MoveTables if strings.Contains(request.Workflow, "lookup") { workflowType = binlogdatapb.VReplicationWorkflowType_CreateLookupIndex diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 08bd7e17862..173c9aa1765 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -392,3 +392,98 @@ func TestWorkflowDelete(t *testing.T) { }) } } + +func TestMoveTablesTrafficSwitching(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + sourceKeyspaceName := "sourceks" + targetKeyspaceName := "targetks" + vrID := 1 + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + "t1": { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + copyTableQR := &queryResult{ + query: fmt.Sprintf("select vrepl_id, table_name, lastpk from _vt.copy_state where vrepl_id in (%d) and id in (select max(id) from _vt.copy_state where vrepl_id in (%d) group by vrepl_id, table_name)", + vrID, vrID), + result: &querypb.QueryResult{}, + } + + testcases := []struct { + name string + sourceKeyspace, targetKeyspace *testKeyspace + preFunc func(t *testing.T, env *testEnv) + req *vtctldatapb.WorkflowSwitchTrafficRequest + expectedSourceQueries []*queryResult + expectedTargetQueries []*queryResult + want *vtctldatapb.WorkflowSwitchTrafficResponse + wantErr bool + postFunc func(t *testing.T, env *testEnv) + }{ + { + name: "basic", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, + req: &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKeyspaceName, + Workflow: workflowName, + Direction: int32(DirectionForward), + }, + want: &vtctldatapb.WorkflowSwitchTrafficResponse{ + Summary: fmt.Sprintf("SwitchTraffic was successful for workflow %s.%s", targetKeyspaceName, workflowName), + StartState: "Reads Not Switched. Writes Not Switched", + CurrentState: "Reads Not Switched. Writes Not Switched", // Obviously not right yet... + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + require.NotNil(t, tc.sourceKeyspace) + require.NotNil(t, tc.targetKeyspace) + require.NotNil(t, tc.req) + env := newTestEnv(t, ctx, defaultCellName, tc.sourceKeyspace, tc.targetKeyspace) + defer env.close() + env.tmc.schema = schema + if tc.expectedSourceQueries != nil { + require.NotNil(t, env.tablets[tc.sourceKeyspace.KeyspaceName]) + for _, eq := range tc.expectedSourceQueries { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, eq) + } + } + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, copyTableQR) + if tc.expectedTargetQueries != nil { + require.NotNil(t, env.tablets[tc.targetKeyspace.KeyspaceName]) + for _, eq := range tc.expectedTargetQueries { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, eq) + } + } + if tc.preFunc != nil { + tc.preFunc(t, env) + } + got, err := env.ws.WorkflowSwitchTraffic(ctx, tc.req) + if (err != nil) != tc.wantErr { + require.Fail(t, "unexpected error value", "Server.WorkflowSwitchTraffic() error = %v, wantErr %v", err, tc.wantErr) + return + } + require.Equal(t, got.String(), tc.want.String(), "Server.WorkflowSwitchTraffic() = %v, want %v", got, tc.want) + if tc.postFunc != nil { + tc.postFunc(t, env) + } + }) + } +} From 4cf58c64e5e423cc4d0247744b655f25de3bdd89 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 5 Jun 2024 11:19:58 -0400 Subject: [PATCH 20/34] Finish up new unit tests Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 56 ++++--- go/vt/vtctl/workflow/server.go | 6 +- go/vt/vtctl/workflow/server_test.go | 182 +++++++++++++++++++++-- go/vt/vtctl/workflow/traffic_switcher.go | 16 +- 4 files changed, 222 insertions(+), 38 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index 4b96ef3a42f..1b7180eaac5 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Vitess Authors. +Copyright 2024 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "testing" "time" @@ -64,6 +65,11 @@ type queryResult struct { result *querypb.QueryResult } +func TestMain(m *testing.M) { + _flag.ParseFlagsForTest() + os.Exit(m.Run()) +} + type testEnv struct { ws *Server ts *topo.Server @@ -74,14 +80,6 @@ type testEnv struct { cell string } -//---------------------------------------------- -// testEnv - -func TestMain(m *testing.M) { - _flag.ParseFlagsForTest() - os.Exit(m.Run()) -} - func newTestEnv(t *testing.T, ctx context.Context, cell string, sourceKeyspace, targetKeyspace *testKeyspace) *testEnv { t.Helper() env := &testEnv{ @@ -160,9 +158,6 @@ func (env *testEnv) deleteTablet(tablet *topodatapb.Tablet) { delete(env.tablets[tablet.Keyspace], int(tablet.Alias.Uid)) } -//---------------------------------------------- -// testTMClient - type testTMClient struct { tmclient.TabletManagerClient schema map[string]*tabletmanagerdatapb.SchemaDefinition @@ -176,6 +171,8 @@ type testTMClient struct { workflowDeleteCalls int env *testEnv + + reverse atomic.Bool // Are we reversing traffic? } func newTestTMClient(env *testEnv) *testTMClient { @@ -211,7 +208,7 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t res := &tabletmanagerdatapb.ReadVReplicationWorkflowResponse{ Workflow: request.Workflow, WorkflowType: workflowType, - Streams: make([]*tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream, len(tmc.env.sourceKeyspace.ShardNames)), + Streams: make([]*tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream, 0, 2), } rules := make([]*binlogdatapb.Rule, len(tmc.schema)) for i, table := range maps.Keys(tmc.schema) { @@ -220,11 +217,15 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t Filter: fmt.Sprintf("select * from %s", table), } } - for i, shard := range tmc.env.sourceKeyspace.ShardNames { - res.Streams[i] = &tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream{ + blsKs := tmc.env.sourceKeyspace + if tmc.reverse.Load() && tablet.Keyspace == tmc.env.sourceKeyspace.KeyspaceName { + blsKs = tmc.env.targetKeyspace + } + for i, shard := range blsKs.ShardNames { + stream := &tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream{ Id: int32(i + 1), Bls: &binlogdatapb.BinlogSource{ - Keyspace: tmc.env.sourceKeyspace.KeyspaceName, + Keyspace: blsKs.KeyspaceName, Shard: shard, Tables: maps.Keys(tmc.schema), Filter: &binlogdatapb.Filter{ @@ -232,6 +233,7 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t }, }, } + res.Streams = append(res.Streams, stream) } return res, nil @@ -308,7 +310,7 @@ func (tmc *testTMClient) verifyQueries(t *testing.T) { for _, qr := range qrs { list = append(list, qr.query) } - t.Errorf("tablet %v: found queries that were expected but never got executed by the test: %v", tabletID, list) + require.Failf(t, "missing query", "tablet %v: found queries that were expected but never got executed by the test: %v", tabletID, list) } } } @@ -385,6 +387,10 @@ func (tmc *testTMClient) ReadVReplicationWorkflows(ctx context.Context, tablet * workflowType = binlogdatapb.VReplicationWorkflowType_CreateLookupIndex } } + ks := tmc.env.sourceKeyspace + if tmc.reverse.Load() { + ks = tmc.env.targetKeyspace + } return &tabletmanagerdatapb.ReadVReplicationWorkflowsResponse{ Workflows: []*tabletmanagerdatapb.ReadVReplicationWorkflowResponse{ { @@ -395,8 +401,8 @@ func (tmc *testTMClient) ReadVReplicationWorkflows(ctx context.Context, tablet * Id: 1, State: binlogdatapb.VReplicationWorkflowState_Running, Bls: &binlogdatapb.BinlogSource{ - Keyspace: tmc.env.sourceKeyspace.KeyspaceName, - Shard: tmc.env.sourceKeyspace.ShardNames[0], + Keyspace: ks.KeyspaceName, + Shard: ks.ShardNames[0], Filter: &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{ { @@ -425,3 +431,15 @@ func (tmc *testTMClient) UpdateVReplicationWorkflow(ctx context.Context, tablet }, }, nil } + +func (tmc *testTMClient) PrimaryPosition(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + return position, nil +} + +func (tmc *testTMClient) WaitForPosition(ctx context.Context, tablet *topodatapb.Tablet, pos string) error { + return nil +} + +func (tmc *testTMClient) VReplicationWaitForPos(ctx context.Context, tablet *topodatapb.Tablet, id int32, pos string) error { + return nil +} diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index 0dc9be308ae..cb732fbfb22 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -3044,6 +3044,8 @@ func (s *Server) WorkflowSwitchTraffic(ctx context.Context, req *vtctldatapb.Wor if err != nil { return nil, err } + log.Errorf("req: %+v", req) + log.Errorf("ts: %+v", ts) if startState.WorkflowType == TypeMigrate { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid action for Migrate workflow: SwitchTraffic") @@ -3115,7 +3117,7 @@ func (s *Server) WorkflowSwitchTraffic(ctx context.Context, req *vtctldatapb.Wor resp.Summary = fmt.Sprintf("%s dry run results for workflow %s.%s at %v", cmd, req.Keyspace, req.Workflow, time.Now().UTC().Format(time.RFC822)) resp.DryRunResults = dryRunResults } else { - log.Infof("SwitchTraffic done for workflow %s.%s", req.Keyspace, req.Workflow) + log.Infof("%s done for workflow %s.%s", cmd, req.Keyspace, req.Workflow) resp.Summary = fmt.Sprintf("%s was successful for workflow %s.%s", cmd, req.Keyspace, req.Workflow) // Reload the state after the SwitchTraffic operation // and return that as a string. @@ -3133,7 +3135,7 @@ func (s *Server) WorkflowSwitchTraffic(ctx context.Context, req *vtctldatapb.Wor } else { resp.CurrentState = currentState.String() } - log.Infof("SwitchTraffic done for workflow %s.%s, returning response %v", req.Keyspace, req.Workflow, resp) + log.Infof("%s done for workflow %s.%s, returning response %v", cmd, req.Keyspace, req.Workflow, resp) } return resp, nil } diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 173c9aa1765..1f64e188b40 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -19,6 +19,8 @@ package workflow import ( "context" "fmt" + "slices" + "strings" "testing" "time" @@ -31,6 +33,7 @@ import ( "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -402,6 +405,11 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { sourceKeyspaceName := "sourceks" targetKeyspaceName := "targetks" vrID := 1 + tabletTypes := []topodatapb.TabletType{ + topodatapb.TabletType_PRIMARY, + topodatapb.TabletType_REPLICA, + topodatapb.TabletType_RDONLY, + } schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ "t1": { TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ @@ -417,6 +425,56 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { vrID, vrID), result: &querypb.QueryResult{}, } + journalQR := &queryResult{ + query: "/select val from _vt.resharding_journal.*", + result: &querypb.QueryResult{}, + } + lockTableQR := &queryResult{ + query: fmt.Sprintf("LOCK TABLES `%s` READ", tableName), + result: &querypb.QueryResult{}, + } + cutoverQR := &queryResult{ + query: "/update _vt.vreplication set state='Stopped', message='stopped for cutover' where id=.*", + result: &querypb.QueryResult{}, + } + createWFQR := &queryResult{ + query: "/insert into _vt.vreplication.*", + result: &querypb.QueryResult{}, + } + deleteWFQR := &queryResult{ + query: fmt.Sprintf("delete from _vt.vreplication where db_name = 'vt_%s' and workflow = '%s'", targetKeyspaceName, workflowName), + result: &querypb.QueryResult{}, + } + deleteReverseWFQR := &queryResult{ + query: fmt.Sprintf("delete from _vt.vreplication where db_name = 'vt_%s' and workflow = '%s'", sourceKeyspaceName, ReverseWorkflowName(workflowName)), + result: &querypb.QueryResult{}, + } + createReverseWFQR := &queryResult{ + query: "/insert into _vt.vreplication.*", + result: &querypb.QueryResult{}, + } + createJournalQR := &queryResult{ + query: "/insert into _vt.resharding_journal.*", + result: &querypb.QueryResult{}, + } + freezeWFQR := &queryResult{ + query: fmt.Sprintf("update _vt.vreplication set message = 'FROZEN' where db_name='vt_%s' and workflow='%s'", targetKeyspaceName, workflowName), + result: &querypb.QueryResult{}, + } + freezeReverseWFQR := &queryResult{ + query: fmt.Sprintf("update _vt.vreplication set message = 'FROZEN' where db_name='vt_%s' and workflow='%s'", sourceKeyspaceName, ReverseWorkflowName(workflowName)), + result: &querypb.QueryResult{}, + } + + hasDeniedTableEntry := func(si *topo.ShardInfo) bool { + if si == nil || len(si.TabletControls) == 0 { + return false + } + for _, tc := range si.Shard.TabletControls { + return slices.Equal(tc.DeniedTables, []string{tableName}) + } + return false + } testcases := []struct { name string @@ -430,7 +488,7 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { postFunc func(t *testing.T, env *testEnv) }{ { - name: "basic", + name: "basic forward", sourceKeyspace: &testKeyspace{ KeyspaceName: sourceKeyspaceName, ShardNames: []string{"0"}, @@ -440,14 +498,37 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { ShardNames: []string{"-80", "80-"}, }, req: &vtctldatapb.WorkflowSwitchTrafficRequest{ - Keyspace: targetKeyspaceName, - Workflow: workflowName, - Direction: int32(DirectionForward), + Keyspace: targetKeyspaceName, + Workflow: workflowName, + Direction: int32(DirectionForward), + TabletTypes: tabletTypes, }, want: &vtctldatapb.WorkflowSwitchTrafficResponse{ Summary: fmt.Sprintf("SwitchTraffic was successful for workflow %s.%s", targetKeyspaceName, workflowName), StartState: "Reads Not Switched. Writes Not Switched", - CurrentState: "Reads Not Switched. Writes Not Switched", // Obviously not right yet... + CurrentState: "All Reads Switched. Writes Switched", + }, + }, + { + name: "basic backward", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, + req: &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKeyspaceName, + Workflow: workflowName, + Direction: int32(DirectionBackward), + TabletTypes: tabletTypes, + }, + want: &vtctldatapb.WorkflowSwitchTrafficResponse{ + Summary: fmt.Sprintf("ReverseTraffic was successful for workflow %s.%s", targetKeyspaceName, workflowName), + StartState: "All Reads Switched. Writes Switched", + CurrentState: "Reads Not Switched. Writes Not Switched", }, }, } @@ -459,30 +540,111 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { env := newTestEnv(t, ctx, defaultCellName, tc.sourceKeyspace, tc.targetKeyspace) defer env.close() env.tmc.schema = schema + if tc.preFunc != nil { + tc.preFunc(t, env) + } if tc.expectedSourceQueries != nil { require.NotNil(t, env.tablets[tc.sourceKeyspace.KeyspaceName]) for _, eq := range tc.expectedSourceQueries { env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, eq) } } - env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, copyTableQR) + if tc.req.Direction == int32(DirectionForward) { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, copyTableQR) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, cutoverQR) + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, journalQR) + } + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, lockTableQR) + } + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, deleteReverseWFQR) + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, createReverseWFQR) + } + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, createJournalQR) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, freezeWFQR) + } else { + env.tmc.reverse.Store(true) + // Setup the routing rules as they would be after having previously done SwitchTraffic. + ks := env.targetKeyspace.KeyspaceName + toTarget := []string{ks + "." + tableName} + rules := make(map[string][]string) + for _, tabletType := range tabletTypes { + tt := strings.ToLower(tabletType.String()) + if tabletType == topodatapb.TabletType_PRIMARY { + rules[tableName] = toTarget + rules[ks+"."+tableName] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget + } else { + rules[tableName+"@"+tt] = toTarget + rules[ks+"."+tableName+"@"+tt] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget + } + } + err := topotools.SaveRoutingRules(ctx, env.ts, rules) + require.NoError(t, err) + err = env.ts.RebuildSrvVSchema(ctx, nil) + require.NoError(t, err) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, copyTableQR) + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, cutoverQR) + } + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, journalQR) + } + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, lockTableQR) + } + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, deleteWFQR) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, createWFQR) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, createJournalQR) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, freezeReverseWFQR) + } if tc.expectedTargetQueries != nil { require.NotNil(t, env.tablets[tc.targetKeyspace.KeyspaceName]) for _, eq := range tc.expectedTargetQueries { env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, eq) } } - if tc.preFunc != nil { - tc.preFunc(t, env) - } got, err := env.ws.WorkflowSwitchTraffic(ctx, tc.req) if (err != nil) != tc.wantErr { require.Fail(t, "unexpected error value", "Server.WorkflowSwitchTraffic() error = %v, wantErr %v", err, tc.wantErr) return } - require.Equal(t, got.String(), tc.want.String(), "Server.WorkflowSwitchTraffic() = %v, want %v", got, tc.want) + require.Equal(t, tc.want.String(), got.String(), "Server.WorkflowSwitchTraffic() = %v, want %v", got, tc.want) if tc.postFunc != nil { tc.postFunc(t, env) + } else { // Default post checks + // Confirm that we have the expected routing rules. + rr, err := env.ts.GetRoutingRules(ctx) + require.NoError(t, err) + to := fmt.Sprintf("%s.%s", tc.targetKeyspace.KeyspaceName, tableName) + if tc.req.Direction == int32(DirectionBackward) { + to = fmt.Sprintf("%s.%s", tc.sourceKeyspace.KeyspaceName, tableName) + } + for _, rr := range rr.Rules { + for _, tt := range rr.ToTables { + require.Equal(t, to, tt) + } + } + // Confirm that we have the expected denied tables entires. + for _, keyspace := range []*testKeyspace{tc.sourceKeyspace, tc.targetKeyspace} { + for _, shardName := range keyspace.ShardNames { + si, err := env.ts.GetShard(ctx, keyspace.KeyspaceName, shardName) + require.NoError(t, err) + switch { + case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionForward): + require.True(t, hasDeniedTableEntry(si)) + case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionBackward): + require.False(t, hasDeniedTableEntry(si)) + case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionForward): + require.False(t, hasDeniedTableEntry(si)) + case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionBackward): + require.True(t, hasDeniedTableEntry(si)) + } + } + } } }) } diff --git a/go/vt/vtctl/workflow/traffic_switcher.go b/go/vt/vtctl/workflow/traffic_switcher.go index d3d479beec4..42f097f35b0 100644 --- a/go/vt/vtctl/workflow/traffic_switcher.go +++ b/go/vt/vtctl/workflow/traffic_switcher.go @@ -104,6 +104,13 @@ const ( // TrafficSwitchDirection specifies the switching direction. type TrafficSwitchDirection int +func (tsd TrafficSwitchDirection) String() string { + if tsd == DirectionForward { + return "forward" + } + return "backward" +} + // TableRemovalType specifies the way the a table will be removed during a // DropSource for a MoveTables workflow. type TableRemovalType int @@ -606,7 +613,7 @@ func (ts *trafficSwitcher) switchShardReads(ctx context.Context, cells []string, } func (ts *trafficSwitcher) switchTableReads(ctx context.Context, cells []string, servedTypes []topodatapb.TabletType, rebuildSrvVSchema bool, direction TrafficSwitchDirection) error { - log.Infof("switchTableReads: cells: %s, tablet types: %+v, direction %d", strings.Join(cells, ","), servedTypes, direction) + log.Infof("switchTableReads: cells: %s, tablet types: %+v, direction: %s", strings.Join(cells, ","), servedTypes, direction) rules, err := topotools.GetRoutingRules(ctx, ts.TopoServer()) if err != nil { return err @@ -623,11 +630,6 @@ func (ts *trafficSwitcher) switchTableReads(ctx context.Context, cells []string, tt := strings.ToLower(servedType.String()) for _, table := range ts.Tables() { - if direction == DirectionForward { - log.Infof("Route direction forward") - } else { - log.Infof("Route direction backwards") - } toTarget := []string{ts.TargetKeyspaceName() + "." + table} rules[table+"@"+tt] = toTarget rules[ts.TargetKeyspaceName()+"."+table+"@"+tt] = toTarget @@ -1106,7 +1108,7 @@ func (ts *trafficSwitcher) cancelMigration(ctx context.Context, sm *StreamMigrat err = ts.deleteReverseVReplication(ctx) if err != nil { - ts.Logger().Errorf("Cancel migration failed: could not delete revers vreplication entries: %v", err) + ts.Logger().Errorf("Cancel migration failed: could not delete reverse vreplication streams: %v", err) } } From 0a945ef99322f81fc4fd5ec20a6ed29cec03e35d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 6 Jun 2024 12:36:33 -0400 Subject: [PATCH 21/34] Move basic test to vtctldclient for static client uses Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/helper_test.go | 18 ++++++---- .../vreplication/vreplication_test.go | 35 +++++++++++-------- .../vreplication/vreplication_test_env.go | 23 ++++++------ go/vt/vtctl/workflow/switcher_dry_run.go | 21 +++++++++-- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 4cefec97b84..203acef6b84 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -504,20 +504,24 @@ func validateDryRunResults(t *testing.T, output string, want []string) { require.NotEmpty(t, output) gotDryRun := strings.Split(output, "\n") require.True(t, len(gotDryRun) > 3) - startRow := 3 - if strings.Contains(gotDryRun[0], "deprecated") { + var startRow int + if strings.HasPrefix(gotDryRun[1], "Parameters:") { // vtctlclient + startRow = 3 + } else if strings.Contains(gotDryRun[0], "deprecated") { startRow = 4 + } else { + startRow = 2 } gotDryRun = gotDryRun[startRow : len(gotDryRun)-1] if len(want) != len(gotDryRun) { - t.Fatalf("want and got: lengths don't match, \nwant\n%s\n\ngot\n%s", strings.Join(want, "\n"), strings.Join(gotDryRun, "\n")) + require.Fail(t, "invalid dry run results", "want and got: lengths don't match, \nwant\n%s\n\ngot\n%s", strings.Join(want, "\n"), strings.Join(gotDryRun, "\n")) } var match, fail bool fail = false for i, w := range want { w = strings.TrimSpace(w) g := strings.TrimSpace(gotDryRun[i]) - if w[0] == '/' { + if len(w) > 0 && w[0] == '/' { w = strings.TrimSpace(w[1:]) result := strings.HasPrefix(g, w) match = result @@ -526,11 +530,11 @@ func validateDryRunResults(t *testing.T, output string, want []string) { } if !match { fail = true - t.Fatalf("want %s, got %s\n", w, gotDryRun[i]) + require.Fail(t, "invlaid dry run results", "want %s, got %s\n", w, gotDryRun[i]) } } if fail { - t.Fatalf("Dry run results don't match, want %s, got %s", want, gotDryRun) + require.Fail(t, "invalid dry run results", "Dry run results don't match, want %s, got %s", want, gotDryRun) } } @@ -566,7 +570,7 @@ func isTableInDenyList(t *testing.T, vc *VitessCluster, ksShard string, table st var err error found := false if output, err = vc.VtctlClient.ExecuteCommandWithOutput("GetShard", ksShard); err != nil { - t.Fatalf("%v %v", err, output) + require.Fail(t, "GetShard error", "%v %v", err, output) return false, err } jsonparser.ArrayEach([]byte(output), func(value []byte, dataType jsonparser.ValueType, offset int, err error) { diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 0d975e018f0..b8f48ca28c4 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -605,7 +605,7 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) { vc.AddKeyspace(t, []*Cell{cell1, cell2}, keyspace, shard, initialProductVSchema, initialProductSchema, defaultReplicas, defaultRdonly, 100, sourceKsOpts) // Add cell alias containing only zone2 - result, err := vc.VtctlClient.ExecuteCommandWithOutput("AddCellsAlias", "--", "--cells", "zone2", "alias") + result, err := vc.VtctldClient.ExecuteCommandWithOutput("AddCellsAlias", "--cells", "zone2", "alias") require.NoError(t, err, "command failed with output: %v", result) verifyClusterHealth(t, vc) @@ -836,7 +836,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl printShardPositions(vc, ksShards) switchWrites(t, workflowType, ksWorkflow, true) - output, err := vc.VtctlClient.ExecuteCommandWithOutput("Workflow", ksWorkflow, "show") + output, err := vc.VtctldClient.ExecuteCommandWithOutput("Workflow", "--keyspace", targetKs, "show", "--workflow", workflow) require.NoError(t, err) require.Contains(t, output, "'customer.reverse_bits'") require.Contains(t, output, "'customer.bmd5'") @@ -945,7 +945,7 @@ func reshardMerchant2to3SplitMerge(t *testing.T) { var err error for _, shard := range strings.Split("-80,80-", ",") { - output, err = vc.VtctlClient.ExecuteCommandWithOutput("GetShard", "merchant:"+shard) + output, err = vc.VtctldClient.ExecuteCommandWithOutput("GetShard", "merchant:"+shard) if err == nil { t.Fatal("GetShard merchant:-80 failed") } @@ -954,7 +954,7 @@ func reshardMerchant2to3SplitMerge(t *testing.T) { for _, shard := range strings.Split("-40,40-c0,c0-", ",") { ksShard := fmt.Sprintf("%s:%s", merchantKeyspace, shard) - output, err = vc.VtctlClient.ExecuteCommandWithOutput("GetShard", ksShard) + output, err = vc.VtctldClient.ExecuteCommandWithOutput("GetShard", ksShard) if err != nil { t.Fatalf("GetShard merchant failed for: %s: %v", shard, err) } @@ -1403,7 +1403,7 @@ func waitForLowLag(t *testing.T, keyspace, workflow string) { waitDuration := 500 * time.Millisecond duration := maxWait for duration > 0 { - output, err := vc.VtctlClient.ExecuteCommandWithOutput("Workflow", fmt.Sprintf("%s.%s", keyspace, workflow), "Show") + output, err := vc.VtctldClient.ExecuteCommandWithOutput("Workflow", "--keyspace", "show", "--workflow", workflow) require.NoError(t, err) lagSeconds, err = jsonparser.GetInt([]byte(output), "MaxVReplicationTransactionLag") @@ -1486,7 +1486,7 @@ func reshardAction(t *testing.T, action, workflow, keyspaceName, sourceShards, t } func applyVSchema(t *testing.T, vschema, keyspace string) { - err := vc.VtctlClient.ExecuteCommand("ApplyVSchema", "--", "--vschema", vschema, keyspace) + err := vc.VtctldClient.ExecuteCommand("ApplyVSchema", "--vschema", vschema, keyspace) require.NoError(t, err) } @@ -1497,8 +1497,10 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry "workflow type specified: %s", workflowType) } ensureCanSwitch(t, workflowType, cells, ksWorkflow) - output, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=rdonly,replica", - "--dry_run", "SwitchTraffic", ksWorkflow) + ks, wf, ok := strings.Cut(ksWorkflow, ".") + require.True(t, ok) + output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--cells="+cells, "--tablet-types=rdonly,replica", + "--dry-run") require.NoError(t, err, fmt.Sprintf("Switching Reads DryRun Error: %s: %s", err, output)) if dryRunResults != nil { validateDryRunResults(t, output, dryRunResults) @@ -1506,10 +1508,12 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry } func ensureCanSwitch(t *testing.T, workflowType, cells, ksWorkflow string) { + ks, wf, ok := strings.Cut(ksWorkflow, ".") + require.True(t, ok) timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - _, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--dry_run", "SwitchTraffic", ksWorkflow) + _, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--cells="+cells, "--dry-run") if err == nil { return } @@ -1535,11 +1539,11 @@ func switchReads(t *testing.T, workflowType, cells, ksWorkflow string, reverse b command = "ReverseTraffic" } ensureCanSwitch(t, workflowType, cells, ksWorkflow) - output, err = vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=rdonly", - command, ksWorkflow) + ks, wf, ok := strings.Cut(ksWorkflow, ".") + require.True(t, ok) + output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, "--cells="+cells, "--tablet-types=rdonly") require.NoError(t, err, fmt.Sprintf("%s Error: %s: %s", command, err, output)) - output, err = vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=replica", - command, ksWorkflow) + output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, "--cells="+cells, "--tablet-types=replica") require.NoError(t, err, fmt.Sprintf("%s Error: %s: %s", command, err, output)) } @@ -1578,8 +1582,9 @@ func switchWritesDryRun(t *testing.T, workflowType, ksWorkflow string, dryRunRes require.FailNowf(t, "Invalid workflow type for SwitchTraffic, must be MoveTables or Reshard", "workflow type specified: %s", workflowType) } - output, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--tablet_types=primary", "--dry_run", - "SwitchTraffic", ksWorkflow) + ks, wf, ok := strings.Cut(ksWorkflow, ".") + require.True(t, ok) + output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--tablet-types=primary", "--dry-run") require.NoError(t, err, fmt.Sprintf("Switch writes DryRun Error: %s: %s", err, output)) validateDryRunResults(t, output, dryRunResults) } diff --git a/go/test/endtoend/vreplication/vreplication_test_env.go b/go/test/endtoend/vreplication/vreplication_test_env.go index 4500a98868c..6073cfac6ab 100644 --- a/go/test/endtoend/vreplication/vreplication_test_env.go +++ b/go/test/endtoend/vreplication/vreplication_test_env.go @@ -19,31 +19,28 @@ package vreplication var dryRunResultsSwitchWritesCustomerShard = []string{ "Lock keyspace product", "Lock keyspace customer", - "Stop writes on keyspace product, tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]:", - "/ Keyspace product, Shard 0 at Position", - "Wait for VReplication on stopped streams to catchup for up to 30s", - "Create reverse replication workflow p2c_reverse", + "/Stop writes on keyspace product for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]: [keyspace:product;shard:0;position:", + "Wait for vreplication on stopped streams to catchup for up to 30s", + "Create reverse vreplication workflow p2c_reverse", "Create journal entries on source databases", - "Enable writes on keyspace customer tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]", + "Enable writes on keyspace customer for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order]", "Switch routing from keyspace product to keyspace customer", "Routing rules for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] will be updated", - "Switch writes completed, freeze and delete vreplication streams on:", - " tablet 200 ", - " tablet 300 ", - "Start reverse replication streams on:", - " tablet 100 ", - "Mark vreplication streams frozen on:", - " Keyspace customer, Shard -80, Tablet 200, Workflow p2c, DbName vt_customer", - " Keyspace customer, Shard 80-, Tablet 300, Workflow p2c, DbName vt_customer", + "Switch writes completed, freeze and delete vreplication streams on: [tablet:200,tablet:300]", + "Start reverse vreplication streams on: [tablet:100]", + "Mark vreplication streams frozen on: [keyspace:customer;shard:-80;tablet:200;workflow:p2c;dbname:vt_customer,keyspace:customer;shard:80-;tablet:300;workflow:p2c;dbname:vt_customer]", "Unlock keyspace customer", "Unlock keyspace product", + "", // Additional empty newline in the output } var dryRunResultsReadCustomerShard = []string{ "Lock keyspace product", "Switch reads for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] to keyspace customer for tablet types [RDONLY,REPLICA]", "Routing rules for tables [Lead,Lead-1,blüb_tbl,customer,db_order_test,geom_tbl,json_tbl,loadtest,reftable,vdiff_order] will be updated", + "Serving VSchema will be rebuilt for the customer keyspace", "Unlock keyspace product", + "", // Additional empty newline in the output } var dryRunResultsSwitchWritesM2m3 = []string{ diff --git a/go/vt/vtctl/workflow/switcher_dry_run.go b/go/vt/vtctl/workflow/switcher_dry_run.go index 03faa4c4ca2..e784c688989 100644 --- a/go/vt/vtctl/workflow/switcher_dry_run.go +++ b/go/vt/vtctl/workflow/switcher_dry_run.go @@ -135,10 +135,20 @@ func (dr *switcherDryRun) changeRouting(ctx context.Context) error { } deleteLogs = nil addLogs = nil - for _, source := range dr.ts.Sources() { + sources := maps.Values(dr.ts.Sources()) + // Sort the slice for deterministic output. + sort.Slice(sources, func(i, j int) bool { + return sources[i].GetPrimary().Alias.Uid < sources[j].GetPrimary().Alias.Uid + }) + for _, source := range sources { deleteLogs = append(deleteLogs, fmt.Sprintf("shard:%s;tablet:%d", source.GetShard().ShardName(), source.GetShard().PrimaryAlias.Uid)) } - for _, target := range dr.ts.Targets() { + targets := maps.Values(dr.ts.Targets()) + // Sort the slice for deterministic output. + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, target := range targets { addLogs = append(addLogs, fmt.Sprintf("shard:%s;tablet:%d", target.GetShard().ShardName(), target.GetShard().PrimaryAlias.Uid)) } if len(deleteLogs) > 0 { @@ -150,7 +160,12 @@ func (dr *switcherDryRun) changeRouting(ctx context.Context) error { func (dr *switcherDryRun) streamMigraterfinalize(ctx context.Context, ts *trafficSwitcher, workflows []string) error { logs := make([]string, 0) - for _, t := range ts.Targets() { + targets := maps.Values(ts.Targets()) + // Sort the slice for deterministic output. + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, t := range targets { logs = append(logs, fmt.Sprintf("tablet:%d", t.GetPrimary().Alias.Uid)) } dr.drLog.Logf("Switch writes completed, freeze and delete vreplication streams on: [%s]", strings.Join(logs, ",")) From aa572b8dd62836b93cd22f692adc59afa2d08013 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 6 Jun 2024 20:03:54 -0400 Subject: [PATCH 22/34] Changes from self review Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/helper_test.go | 5 ++-- .../vreplication/vreplication_test.go | 16 +++++++---- go/vt/topo/shard.go | 9 +++--- go/vt/vtctl/workflow/framework_test.go | 28 +++++++------------ go/vt/vtctl/workflow/server.go | 2 -- go/vt/vtctl/workflow/server_test.go | 2 +- 6 files changed, 28 insertions(+), 34 deletions(-) diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 203acef6b84..8126b41fa76 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -89,7 +89,6 @@ func execQueryWithRetry(t *testing.T, conn *mysql.Conn, query string, timeout ti if err == nil { return qr } - log.Infof("Error executing query (will retry): %s: %v", query, err) select { case <-ctx.Done(): require.FailNow(t, fmt.Sprintf("query %q did not succeed before the timeout of %s; last seen result: %v", @@ -594,8 +593,8 @@ func expectNumberOfStreams(t *testing.T, vtgateConn *mysql.Conn, name string, wo waitForQueryResult(t, vtgateConn, database, query, fmt.Sprintf(`[[INT64(%d)]]`, want)) } -// confirmAllStreamsRunning confirms that all of migrated streams are running -// for a workflow. +// confirmAllStreamsRunning confirms that all of the workflow's streams are +// in the running state. func confirmAllStreamsRunning(t *testing.T, keyspace, shard string) { query := sqlparser.BuildParsedQuery("select count(*) from %s.vreplication where state != '%s'", sidecarDBIdentifier, binlogdatapb.VReplicationWorkflowState_Running.String()).Query diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index b8f48ca28c4..726f91cfbd0 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -1499,8 +1499,8 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry ensureCanSwitch(t, workflowType, cells, ksWorkflow) ks, wf, ok := strings.Cut(ksWorkflow, ".") require.True(t, ok) - output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--cells="+cells, "--tablet-types=rdonly,replica", - "--dry-run") + output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", + "--cells="+cells, "--tablet-types=rdonly,replica", "--dry-run") require.NoError(t, err, fmt.Sprintf("Switching Reads DryRun Error: %s: %s", err, output)) if dryRunResults != nil { validateDryRunResults(t, output, dryRunResults) @@ -1513,7 +1513,8 @@ func ensureCanSwitch(t *testing.T, workflowType, cells, ksWorkflow string) { timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - _, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--cells="+cells, "--dry-run") + _, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", + "--cells="+cells, "--dry-run") if err == nil { return } @@ -1541,9 +1542,11 @@ func switchReads(t *testing.T, workflowType, cells, ksWorkflow string, reverse b ensureCanSwitch(t, workflowType, cells, ksWorkflow) ks, wf, ok := strings.Cut(ksWorkflow, ".") require.True(t, ok) - output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, "--cells="+cells, "--tablet-types=rdonly") + output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, + "--cells="+cells, "--tablet-types=rdonly") require.NoError(t, err, fmt.Sprintf("%s Error: %s: %s", command, err, output)) - output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, "--cells="+cells, "--tablet-types=replica") + output, err = vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, command, + "--cells="+cells, "--tablet-types=replica") require.NoError(t, err, fmt.Sprintf("%s Error: %s: %s", command, err, output)) } @@ -1584,7 +1587,8 @@ func switchWritesDryRun(t *testing.T, workflowType, ksWorkflow string, dryRunRes } ks, wf, ok := strings.Cut(ksWorkflow, ".") require.True(t, ok) - output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, "SwitchTraffic", "--tablet-types=primary", "--dry-run") + output, err := vc.VtctldClient.ExecuteCommandWithOutput(workflowType, "--workflow", wf, "--target-keyspace", ks, + "SwitchTraffic", "--tablet-types=primary", "--dry-run") require.NoError(t, err, fmt.Sprintf("Switch writes DryRun Error: %s: %s", err, output)) validateDryRunResults(t, output, dryRunResults) } diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index 2650e962610..e7cefb5e90a 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -44,8 +44,8 @@ import ( ) const ( - dlTablesAlreadyPresent = "one or more tables are already present in the denylist" - dlTablesNotPresent = "cannot remove tables since one or more do not exist in the denylist" + dlTablesAlreadyPresent = "one or more tables were already present in the denylist" + dlTablesNotPresent = "one or more tables did not exist in the denylist" dlNoCellsForPrimary = "you cannot specify cells for a primary's tablet control" ) @@ -397,7 +397,7 @@ func (si *ShardInfo) UpdateDeniedTables(ctx context.Context, tabletType topodata } tc := si.GetTabletControl(tabletType) if tc == nil { - // Handle the case where the TabletControl object is new + // Handle the case where the TabletControl object is new. if remove { // We tried to remove something that doesn't exist, log a warning. // But we know that our work is done. @@ -475,7 +475,8 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr return nil } if len(newTables) != len(tables) { - // Some of the tables already existed in the denied list so we don't need to add them. + // Some of the tables already existed in the DeniedTables list so we don't + // need to add them. log.Warningf("%s:%s", dlTablesNotPresent, strings.Join(newTables, ",")) // We do need to merge the lists, however. tables = append(tables, newTables...) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index 1b7180eaac5..e183d3b0f6b 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -83,7 +83,7 @@ type testEnv struct { func newTestEnv(t *testing.T, ctx context.Context, cell string, sourceKeyspace, targetKeyspace *testKeyspace) *testEnv { t.Helper() env := &testEnv{ - ts: memorytopo.NewServer(ctx, defaultCellName), + ts: memorytopo.NewServer(ctx, cell), sourceKeyspace: sourceKeyspace, targetKeyspace: targetKeyspace, tablets: make(map[string]map[int]*topodatapb.Tablet), @@ -95,13 +95,13 @@ func newTestEnv(t *testing.T, ctx context.Context, cell string, sourceKeyspace, tabletID := startingSourceTabletUID for _, shardName := range sourceKeyspace.ShardNames { - _ = env.addTablet(tabletID, sourceKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) + _ = env.addTablet(t, ctx, tabletID, sourceKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) tabletID += tabletUIDStep } if sourceKeyspace.KeyspaceName != targetKeyspace.KeyspaceName { tabletID = startingTargetTabletUID for _, shardName := range targetKeyspace.ShardNames { - _ = env.addTablet(tabletID, targetKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) + _ = env.addTablet(t, ctx, tabletID, targetKeyspace.KeyspaceName, shardName, topodatapb.TabletType_PRIMARY) tabletID += tabletUIDStep } } @@ -119,7 +119,7 @@ func (env *testEnv) close() { } } -func (env *testEnv) addTablet(id int, keyspace, shard string, tabletType topodatapb.TabletType) *topodatapb.Tablet { +func (env *testEnv) addTablet(t *testing.T, ctx context.Context, id int, keyspace, shard string, tabletType topodatapb.TabletType) *topodatapb.Tablet { tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: env.cell, @@ -137,18 +137,15 @@ func (env *testEnv) addTablet(id int, keyspace, shard string, tabletType topodat env.tablets[keyspace] = make(map[int]*topodatapb.Tablet) } env.tablets[keyspace][id] = tablet - if err := env.ws.ts.InitTablet(context.Background(), tablet, false /* allowPrimaryOverride */, true /* createShardAndKeyspace */, false /* allowUpdate */); err != nil { - panic(err) - } + err := env.ws.ts.InitTablet(ctx, tablet, false /* allowPrimaryOverride */, true /* createShardAndKeyspace */, false /* allowUpdate */) + require.NoError(t, err) if tabletType == topodatapb.TabletType_PRIMARY { - _, err := env.ws.ts.UpdateShardFields(context.Background(), keyspace, shard, func(si *topo.ShardInfo) error { + _, err = env.ws.ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { si.PrimaryAlias = tablet.Alias si.IsPrimaryServing = true return nil }) - if err != nil { - panic(err) - } + require.NoError(t, err) } return tablet } @@ -167,11 +164,7 @@ type testTMClient struct { createVReplicationWorkflowRequests map[uint32]*tabletmanagerdatapb.CreateVReplicationWorkflowRequest readVReplicationWorkflowRequests map[uint32]*tabletmanagerdatapb.ReadVReplicationWorkflowRequest - // Used to confirm the number of times WorkflowDelete was called. - workflowDeleteCalls int - - env *testEnv - + env *testEnv // For access to the env config from tmc methods. reverse atomic.Bool // Are we reversing traffic? } @@ -242,7 +235,6 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (response *tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, err error) { tmc.mu.Lock() defer tmc.mu.Unlock() - tmc.workflowDeleteCalls++ return &tabletmanagerdatapb.DeleteVReplicationWorkflowResponse{ Result: &querypb.QueryResult{ RowsAffected: 1, @@ -337,7 +329,7 @@ func (tmc *testTMClient) VReplicationExec(ctx context.Context, tablet *topodatap } func (tmc *testTMClient) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb.Tablet, usePool bool, req *tabletmanagerdatapb.ExecuteFetchAsDbaRequest) (*querypb.QueryResult, error) { - // Reuse VReplicationExec + // Reuse VReplicationExec. return tmc.VReplicationExec(ctx, tablet, string(req.Query)) } diff --git a/go/vt/vtctl/workflow/server.go b/go/vt/vtctl/workflow/server.go index cb732fbfb22..587caff3c8c 100644 --- a/go/vt/vtctl/workflow/server.go +++ b/go/vt/vtctl/workflow/server.go @@ -3044,8 +3044,6 @@ func (s *Server) WorkflowSwitchTraffic(ctx context.Context, req *vtctldatapb.Wor if err != nil { return nil, err } - log.Errorf("req: %+v", req) - log.Errorf("ts: %+v", ts) if startState.WorkflowType == TypeMigrate { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid action for Migrate workflow: SwitchTraffic") diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 1f64e188b40..618bdac292c 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -450,7 +450,7 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { result: &querypb.QueryResult{}, } createReverseWFQR := &queryResult{ - query: "/insert into _vt.vreplication.*", + query: "/insert into _vt.vreplication.*_reverse.*", result: &querypb.QueryResult{}, } createJournalQR := &queryResult{ From 0c8a9ab393f8a2a5b909be064d214d4426c607fa Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 7 Jun 2024 01:26:09 -0400 Subject: [PATCH 23/34] Correct warning message Signed-off-by: Matt Lord --- go/vt/topo/shard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index e7cefb5e90a..f3d88099230 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -477,7 +477,7 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr if len(newTables) != len(tables) { // Some of the tables already existed in the DeniedTables list so we don't // need to add them. - log.Warningf("%s:%s", dlTablesNotPresent, strings.Join(newTables, ",")) + log.Warningf("%s:%s", dlTablesAlreadyPresent, strings.Join(newTables, ",")) // We do need to merge the lists, however. tables = append(tables, newTables...) tc.DeniedTables = append(tc.DeniedTables, tables...) From 1955b9ab5aebfe4f2e72cfc0dfa279db91184128 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 7 Jun 2024 02:32:02 -0400 Subject: [PATCH 24/34] Sort all the things in dry run switcher Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/switcher_dry_run.go | 127 +++++++++++++++++++---- 1 file changed, 108 insertions(+), 19 deletions(-) diff --git a/go/vt/vtctl/workflow/switcher_dry_run.go b/go/vt/vtctl/workflow/switcher_dry_run.go index e784c688989..b8b1369bdf7 100644 --- a/go/vt/vtctl/workflow/switcher_dry_run.go +++ b/go/vt/vtctl/workflow/switcher_dry_run.go @@ -82,6 +82,7 @@ func (dr *switcherDryRun) switchShardReads(ctx context.Context, cells []string, for _, target := range dr.ts.Targets() { targetShards = append(targetShards, target.GetShard().ShardName()) } + // Sort the slices for deterministic output. sort.Strings(sourceShards) sort.Strings(targetShards) if direction == DirectionForward { @@ -103,6 +104,7 @@ func (dr *switcherDryRun) switchTableReads(ctx context.Context, cells []string, for _, servedType := range servedTypes { tabletTypes = append(tabletTypes, servedType.String()) } + sort.Strings(dr.ts.Tables()) // For deterministic output tables := strings.Join(dr.ts.Tables(), ",") dr.drLog.Logf("Switch reads for tables [%s] to keyspace %s for tablet types [%s]", tables, ks, strings.Join(tabletTypes, ",")) dr.drLog.Logf("Routing rules for tables [%s] will be updated", tables) @@ -114,6 +116,7 @@ func (dr *switcherDryRun) switchTableReads(ctx context.Context, cells []string, func (dr *switcherDryRun) createJournals(ctx context.Context, sourceWorkflows []string) error { dr.drLog.Log("Create journal entries on source databases") + sort.Strings(sourceWorkflows) // For deterministic output if len(sourceWorkflows) > 0 { dr.drLog.Logf("Source workflows found: [%s]", strings.Join(sourceWorkflows, ",")) } @@ -121,6 +124,7 @@ func (dr *switcherDryRun) createJournals(ctx context.Context, sourceWorkflows [] } func (dr *switcherDryRun) allowTargetWrites(ctx context.Context) error { + sort.Strings(dr.ts.Tables()) // For deterministic output dr.drLog.Logf("Enable writes on keyspace %s for tables [%s]", dr.ts.TargetKeyspaceName(), strings.Join(dr.ts.Tables(), ",")) return nil } @@ -129,6 +133,7 @@ func (dr *switcherDryRun) changeRouting(ctx context.Context) error { dr.drLog.Logf("Switch routing from keyspace %s to keyspace %s", dr.ts.SourceKeyspaceName(), dr.ts.TargetKeyspaceName()) var deleteLogs, addLogs []string if dr.ts.MigrationType() == binlogdatapb.MigrationType_TABLES { + sort.Strings(dr.ts.Tables()) // For deterministic output tables := strings.Join(dr.ts.Tables(), ",") dr.drLog.Logf("Routing rules for tables [%s] will be updated", tables) return nil @@ -174,7 +179,12 @@ func (dr *switcherDryRun) streamMigraterfinalize(ctx context.Context, ts *traffi func (dr *switcherDryRun) startReverseVReplication(ctx context.Context) error { logs := make([]string, 0) - for _, t := range dr.ts.Sources() { + sources := maps.Values(dr.ts.Sources()) + // Sort the slice for deterministic output. + sort.Slice(sources, func(i, j int) bool { + return sources[i].GetPrimary().Alias.Uid < sources[j].GetPrimary().Alias.Uid + }) + for _, t := range sources { logs = append(logs, fmt.Sprintf("tablet:%d", t.GetPrimary().Alias.Uid)) } dr.drLog.Logf("Start reverse vreplication streams on: [%s]", strings.Join(logs, ",")) @@ -195,17 +205,33 @@ func (dr *switcherDryRun) migrateStreams(ctx context.Context, sm *StreamMigrator logs := make([]string, 0) dr.drLog.Logf("Migrate streams to %s:", dr.ts.TargetKeyspaceName()) - for key, streams := range sm.Streams() { - for _, stream := range streams { - logs = append(logs, fmt.Sprintf("shard:%s;id:%d;workflow:%s;position:%s;binlogsource:%v", key, stream.ID, stream.Workflow, replication.EncodePosition(stream.Position), stream.BinlogSource)) + allStreams := sm.Streams() + // Sort the keys and slices for deterministic output. + shards := maps.Keys(sm.Streams()) + sort.Strings(shards) + for _, shard := range shards { + shardStreams := allStreams[shard] + sort.Slice(shardStreams, func(i, j int) bool { + return shardStreams[i].ID < shardStreams[j].ID + }) + for _, stream := range shardStreams { + logs = append(logs, fmt.Sprintf("shard:%s;id:%d;workflow:%s;position:%s;binlogsource:%v", shard, stream.ID, stream.Workflow, replication.EncodePosition(stream.Position), stream.BinlogSource)) } } if len(logs) > 0 { dr.drLog.Logf("Migrate source streams: [%s]", strings.Join(logs, ",")) logs = nil } - for _, target := range dr.ts.Targets() { + // Sort the keys and slices for deterministic output. + targets := maps.Values(dr.ts.Targets()) + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, target := range targets { tabletStreams := templates + sort.Slice(tabletStreams, func(i, j int) bool { + return tabletStreams[i].ID < tabletStreams[j].ID + }) for _, vrs := range tabletStreams { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;tablet:%d;workflow:%s;id:%d,position:%v;binlogsource:%s", vrs.BinlogSource.Keyspace, vrs.BinlogSource.Shard, target.GetPrimary().Alias.Uid, vrs.Workflow, vrs.ID, replication.EncodePosition(vrs.Position), vrs.BinlogSource)) @@ -224,10 +250,16 @@ func (dr *switcherDryRun) waitForCatchup(ctx context.Context, filteredReplicatio func (dr *switcherDryRun) stopSourceWrites(ctx context.Context) error { logs := make([]string, 0) - for _, source := range dr.ts.Sources() { + sources := maps.Values(dr.ts.Sources()) + // Sort the slice for deterministic output. + sort.Slice(sources, func(i, j int) bool { + return sources[i].GetPrimary().Alias.Uid < sources[j].GetPrimary().Alias.Uid + }) + for _, source := range sources { position, _ := dr.ts.TabletManagerClient().PrimaryPosition(ctx, source.GetPrimary().Tablet) logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;position:%s", dr.ts.SourceKeyspaceName(), source.GetShard().ShardName(), position)) } + sort.Strings(dr.ts.Tables()) // For deterministic output if len(logs) > 0 { dr.drLog.Logf("Stop writes on keyspace %s for tables [%s]: [%s]", dr.ts.SourceKeyspaceName(), strings.Join(dr.ts.Tables(), ","), strings.Join(logs, ",")) @@ -237,8 +269,16 @@ func (dr *switcherDryRun) stopSourceWrites(ctx context.Context) error { func (dr *switcherDryRun) stopStreams(ctx context.Context, sm *StreamMigrator) ([]string, error) { logs := make([]string, 0) - for _, streams := range sm.Streams() { - for _, stream := range streams { + allStreams := sm.Streams() + // Sort the keys and slices for deterministic output. + shards := maps.Keys(sm.Streams()) + sort.Strings(shards) + for _, shard := range shards { + shardStreams := allStreams[shard] + sort.Slice(shardStreams, func(i, j int) bool { + return shardStreams[i].ID < shardStreams[j].ID + }) + for _, stream := range shardStreams { logs = append(logs, fmt.Sprintf("id:%d;keyspace:%s;shard:%s;rules:%s;position:%v", stream.ID, stream.BinlogSource.Keyspace, stream.BinlogSource.Shard, stream.BinlogSource.Filter, stream.Position)) } @@ -262,7 +302,13 @@ func (dr *switcherDryRun) lockKeyspace(ctx context.Context, keyspace, _ string) func (dr *switcherDryRun) removeSourceTables(ctx context.Context, removalType TableRemovalType) error { logs := make([]string, 0) - for _, source := range dr.ts.Sources() { + sort.Strings(dr.ts.Tables()) // For deterministic output + sources := maps.Values(dr.ts.Sources()) + // Sort the slice for deterministic output. + sort.Slice(sources, func(i, j int) bool { + return sources[i].GetPrimary().Alias.Uid < sources[j].GetPrimary().Alias.Uid + }) + for _, source := range sources { for _, tableName := range dr.ts.Tables() { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;dbname:%s;tablet:%d;table:%s", source.GetPrimary().Keyspace, source.GetPrimary().Shard, source.GetPrimary().DbName(), source.GetPrimary().Alias.Uid, tableName)) @@ -282,7 +328,12 @@ func (dr *switcherDryRun) removeSourceTables(ctx context.Context, removalType Ta func (dr *switcherDryRun) dropSourceShards(ctx context.Context) error { logs := make([]string, 0) tabletsList := make(map[string][]string) - for _, si := range dr.ts.SourceShards() { + // Sort the slice for deterministic output. + sourceShards := dr.ts.SourceShards() + sort.Slice(sourceShards, func(i, j int) bool { + return sourceShards[i].PrimaryAlias.Uid < sourceShards[j].PrimaryAlias.Uid + }) + for _, si := range sourceShards { tabletAliases, err := dr.ts.TopoServer().FindAllTabletAliasesInShard(ctx, si.Keyspace(), si.ShardName()) if err != nil { return err @@ -291,7 +342,7 @@ func (dr *switcherDryRun) dropSourceShards(ctx context.Context) error { for _, t := range tabletAliases { tabletsList[si.ShardName()] = append(tabletsList[si.ShardName()], fmt.Sprintf("%d", t.Uid)) } - sort.Strings(tabletsList[si.ShardName()]) + sort.Strings(tabletsList[si.ShardName()]) // For deterministic output logs = append(logs, fmt.Sprintf("cell:%s;keyspace:%s;shards:[%s]", si.Shard.PrimaryAlias.Cell, si.Keyspace(), si.ShardName()), strings.Join(tabletsList[si.ShardName()], ",")) } @@ -308,7 +359,12 @@ func (dr *switcherDryRun) validateWorkflowHasCompleted(ctx context.Context) erro func (dr *switcherDryRun) dropTargetVReplicationStreams(ctx context.Context) error { logs := make([]string, 0) - for _, t := range dr.ts.Targets() { + // Sort the keys and slices for deterministic output. + targets := maps.Values(dr.ts.Targets()) + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, t := range targets { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;workflow:%s;dbname:%s;tablet:%d", t.GetShard().Keyspace(), t.GetShard().ShardName(), dr.ts.WorkflowName(), t.GetPrimary().DbName(), t.GetPrimary().Alias.Uid)) } @@ -318,7 +374,12 @@ func (dr *switcherDryRun) dropTargetVReplicationStreams(ctx context.Context) err func (dr *switcherDryRun) dropSourceReverseVReplicationStreams(ctx context.Context) error { logs := make([]string, 0) - for _, t := range dr.ts.Sources() { + sources := maps.Values(dr.ts.Sources()) + // Sort the slice for deterministic output. + sort.Slice(sources, func(i, j int) bool { + return sources[i].GetPrimary().Alias.Uid < sources[j].GetPrimary().Alias.Uid + }) + for _, t := range sources { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;workflow:%s;dbname:%s;tablet:%d", t.GetShard().Keyspace(), t.GetShard().ShardName(), ReverseWorkflowName(dr.ts.WorkflowName()), t.GetPrimary().DbName(), t.GetPrimary().Alias.Uid)) } @@ -328,7 +389,12 @@ func (dr *switcherDryRun) dropSourceReverseVReplicationStreams(ctx context.Conte func (dr *switcherDryRun) freezeTargetVReplication(ctx context.Context) error { logs := make([]string, 0) - for _, target := range dr.ts.Targets() { + // Sort the keys and slices for deterministic output. + targets := maps.Values(dr.ts.Targets()) + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, target := range targets { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;tablet:%d;workflow:%s;dbname:%s", target.GetPrimary().Keyspace, target.GetPrimary().Shard, target.GetPrimary().Alias.Uid, dr.ts.WorkflowName(), target.GetPrimary().DbName())) } @@ -340,7 +406,12 @@ func (dr *switcherDryRun) freezeTargetVReplication(ctx context.Context) error { func (dr *switcherDryRun) dropSourceDeniedTables(ctx context.Context) error { logs := make([]string, 0) - for _, si := range dr.ts.SourceShards() { + // Sort the slice for deterministic output. + sourceShards := dr.ts.SourceShards() + sort.Slice(sourceShards, func(i, j int) bool { + return sourceShards[i].PrimaryAlias.Uid < sourceShards[j].PrimaryAlias.Uid + }) + for _, si := range sourceShards { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;tablet:%d", si.Keyspace(), si.ShardName(), si.PrimaryAlias.Uid)) } if len(logs) > 0 { @@ -351,7 +422,12 @@ func (dr *switcherDryRun) dropSourceDeniedTables(ctx context.Context) error { func (dr *switcherDryRun) dropTargetDeniedTables(ctx context.Context) error { logs := make([]string, 0) - for _, si := range dr.ts.TargetShards() { + // Sort the slice for deterministic output. + targetShards := dr.ts.TargetShards() + sort.Slice(targetShards, func(i, j int) bool { + return targetShards[i].PrimaryAlias.Uid < targetShards[j].PrimaryAlias.Uid + }) + for _, si := range targetShards { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;tablet:%d", si.Keyspace(), si.ShardName(), si.PrimaryAlias.Uid)) } if len(logs) > 0 { @@ -366,7 +442,13 @@ func (dr *switcherDryRun) logs() *[]string { func (dr *switcherDryRun) removeTargetTables(ctx context.Context) error { logs := make([]string, 0) - for _, target := range dr.ts.Targets() { + sort.Strings(dr.ts.Tables()) // For deterministic output + // Sort the keys and slices for deterministic output. + targets := maps.Values(dr.ts.Targets()) + sort.Slice(targets, func(i, j int) bool { + return targets[i].GetPrimary().Alias.Uid < targets[j].GetPrimary().Alias.Uid + }) + for _, target := range targets { for _, tableName := range dr.ts.Tables() { logs = append(logs, fmt.Sprintf("keyspace:%s;shard:%s;dbname:%s;tablet:%d;table:%s", target.GetPrimary().Keyspace, target.GetPrimary().Shard, target.GetPrimary().DbName(), target.GetPrimary().Alias.Uid, tableName)) @@ -382,7 +464,13 @@ func (dr *switcherDryRun) removeTargetTables(ctx context.Context) error { func (dr *switcherDryRun) dropTargetShards(ctx context.Context) error { logs := make([]string, 0) tabletsList := make(map[string][]string) - for _, si := range dr.ts.TargetShards() { + sort.Strings(dr.ts.Tables()) // For deterministic output + // Sort the slice for deterministic output. + targetShards := dr.ts.TargetShards() + sort.Slice(targetShards, func(i, j int) bool { + return targetShards[i].PrimaryAlias.Uid < targetShards[j].PrimaryAlias.Uid + }) + for _, si := range targetShards { tabletAliases, err := dr.ts.TopoServer().FindAllTabletAliasesInShard(ctx, si.Keyspace(), si.ShardName()) if err != nil { return err @@ -391,7 +479,7 @@ func (dr *switcherDryRun) dropTargetShards(ctx context.Context) error { for _, t := range tabletAliases { tabletsList[si.ShardName()] = append(tabletsList[si.ShardName()], fmt.Sprintf("%d", t.Uid)) } - sort.Strings(tabletsList[si.ShardName()]) + sort.Strings(tabletsList[si.ShardName()]) // For deterministic output logs = append(logs, fmt.Sprintf("cell:%s;keyspace:%s;shards:[%s]", si.Shard.PrimaryAlias.Cell, si.Keyspace(), si.ShardName()), strings.Join(tabletsList[si.ShardName()], ",")) } @@ -416,6 +504,7 @@ func (dr *switcherDryRun) resetSequences(ctx context.Context) error { } func (dr *switcherDryRun) initializeTargetSequences(ctx context.Context, sequencesByBackingTable map[string]*sequenceMetadata) error { + // Sort keys for deterministic output. sortedBackingTableNames := maps.Keys(sequencesByBackingTable) slices.Sort(sortedBackingTableNames) dr.drLog.Log(fmt.Sprintf("The following sequence backing tables used by tables being moved will be initialized: %s", From 392c8f6e9d714e1acf802be35bf6d55fb979727f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 7 Jun 2024 11:51:56 -0400 Subject: [PATCH 25/34] Add Traffic Switching DryRun unit test Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server_test.go | 246 ++++++++++++++++++++++------ 1 file changed, 200 insertions(+), 46 deletions(-) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 618bdac292c..96811da0e85 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -479,13 +479,9 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { testcases := []struct { name string sourceKeyspace, targetKeyspace *testKeyspace - preFunc func(t *testing.T, env *testEnv) req *vtctldatapb.WorkflowSwitchTrafficRequest - expectedSourceQueries []*queryResult - expectedTargetQueries []*queryResult want *vtctldatapb.WorkflowSwitchTrafficResponse wantErr bool - postFunc func(t *testing.T, env *testEnv) }{ { name: "basic forward", @@ -540,15 +536,6 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { env := newTestEnv(t, ctx, defaultCellName, tc.sourceKeyspace, tc.targetKeyspace) defer env.close() env.tmc.schema = schema - if tc.preFunc != nil { - tc.preFunc(t, env) - } - if tc.expectedSourceQueries != nil { - require.NotNil(t, env.tablets[tc.sourceKeyspace.KeyspaceName]) - for _, eq := range tc.expectedSourceQueries { - env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, eq) - } - } if tc.req.Direction == int32(DirectionForward) { env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, copyTableQR) env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, cutoverQR) @@ -601,51 +588,218 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, createJournalQR) env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, freezeReverseWFQR) } - if tc.expectedTargetQueries != nil { - require.NotNil(t, env.tablets[tc.targetKeyspace.KeyspaceName]) - for _, eq := range tc.expectedTargetQueries { - env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, eq) - } - } got, err := env.ws.WorkflowSwitchTraffic(ctx, tc.req) if (err != nil) != tc.wantErr { require.Fail(t, "unexpected error value", "Server.WorkflowSwitchTraffic() error = %v, wantErr %v", err, tc.wantErr) return } require.Equal(t, tc.want.String(), got.String(), "Server.WorkflowSwitchTraffic() = %v, want %v", got, tc.want) - if tc.postFunc != nil { - tc.postFunc(t, env) - } else { // Default post checks - // Confirm that we have the expected routing rules. - rr, err := env.ts.GetRoutingRules(ctx) - require.NoError(t, err) - to := fmt.Sprintf("%s.%s", tc.targetKeyspace.KeyspaceName, tableName) - if tc.req.Direction == int32(DirectionBackward) { - to = fmt.Sprintf("%s.%s", tc.sourceKeyspace.KeyspaceName, tableName) + + // Confirm that we have the expected routing rules. + rr, err := env.ts.GetRoutingRules(ctx) + require.NoError(t, err) + to := fmt.Sprintf("%s.%s", tc.targetKeyspace.KeyspaceName, tableName) + if tc.req.Direction == int32(DirectionBackward) { + to = fmt.Sprintf("%s.%s", tc.sourceKeyspace.KeyspaceName, tableName) + } + for _, rr := range rr.Rules { + for _, tt := range rr.ToTables { + require.Equal(t, to, tt) } - for _, rr := range rr.Rules { - for _, tt := range rr.ToTables { - require.Equal(t, to, tt) + } + // Confirm that we have the expected denied tables entires. + for _, keyspace := range []*testKeyspace{tc.sourceKeyspace, tc.targetKeyspace} { + for _, shardName := range keyspace.ShardNames { + si, err := env.ts.GetShard(ctx, keyspace.KeyspaceName, shardName) + require.NoError(t, err) + switch { + case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionForward): + require.True(t, hasDeniedTableEntry(si)) + case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionBackward): + require.False(t, hasDeniedTableEntry(si)) + case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionForward): + require.False(t, hasDeniedTableEntry(si)) + case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionBackward): + require.True(t, hasDeniedTableEntry(si)) } } - // Confirm that we have the expected denied tables entires. - for _, keyspace := range []*testKeyspace{tc.sourceKeyspace, tc.targetKeyspace} { - for _, shardName := range keyspace.ShardNames { - si, err := env.ts.GetShard(ctx, keyspace.KeyspaceName, shardName) - require.NoError(t, err) - switch { - case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionForward): - require.True(t, hasDeniedTableEntry(si)) - case keyspace == tc.sourceKeyspace && tc.req.Direction == int32(DirectionBackward): - require.False(t, hasDeniedTableEntry(si)) - case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionForward): - require.False(t, hasDeniedTableEntry(si)) - case keyspace == tc.targetKeyspace && tc.req.Direction == int32(DirectionBackward): - require.True(t, hasDeniedTableEntry(si)) - } + } + }) + } +} + +func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + sourceKeyspaceName := "sourceks" + targetKeyspaceName := "targetks" + vrID := 1 + tabletTypes := []topodatapb.TabletType{ + topodatapb.TabletType_PRIMARY, + topodatapb.TabletType_REPLICA, + topodatapb.TabletType_RDONLY, + } + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + "t1": { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + copyTableQR := &queryResult{ + query: fmt.Sprintf("select vrepl_id, table_name, lastpk from _vt.copy_state where vrepl_id in (%d) and id in (select max(id) from _vt.copy_state where vrepl_id in (%d) group by vrepl_id, table_name)", + vrID, vrID), + result: &querypb.QueryResult{}, + } + journalQR := &queryResult{ + query: "/select val from _vt.resharding_journal.*", + result: &querypb.QueryResult{}, + } + lockTableQR := &queryResult{ + query: fmt.Sprintf("LOCK TABLES `%s` READ", tableName), + result: &querypb.QueryResult{}, + } + + testcases := []struct { + name string + sourceKeyspace, targetKeyspace *testKeyspace + req *vtctldatapb.WorkflowSwitchTrafficRequest + want []string + }{ + { + name: "basic forward", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, + req: &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKeyspaceName, + Workflow: workflowName, + Direction: int32(DirectionForward), + TabletTypes: tabletTypes, + DryRun: true, + }, + want: []string{ + fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), + fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tableName, targetKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), + fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), + fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), + fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:0;position:%s]", sourceKeyspaceName, tableName, sourceKeyspaceName, position), + "Wait for vreplication on stopped streams to catchup for up to 30s", + fmt.Sprintf("Create reverse vreplication workflow %s", ReverseWorkflowName(workflowName)), + "Create journal entries on source databases", + fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", targetKeyspaceName, tableName), + fmt.Sprintf("Switch routing from keyspace %s to keyspace %s", sourceKeyspaceName, targetKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d,tablet:%d]", startingTargetTabletUID, startingTargetTabletUID+tabletUIDStep), + fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:-80;tablet:%d;workflow:%s;dbname:vt_%s,keyspace:%s;shard:80-;tablet:%d;workflow:%s;dbname:vt_%s]", + targetKeyspaceName, startingTargetTabletUID, workflowName, targetKeyspaceName, targetKeyspaceName, startingTargetTabletUID+tabletUIDStep, workflowName, targetKeyspaceName), + fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), + fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), + }, + }, + { + name: "basic backward", + sourceKeyspace: &testKeyspace{ + KeyspaceName: sourceKeyspaceName, + ShardNames: []string{"0"}, + }, + targetKeyspace: &testKeyspace{ + KeyspaceName: targetKeyspaceName, + ShardNames: []string{"-80", "80-"}, + }, + req: &vtctldatapb.WorkflowSwitchTrafficRequest{ + Keyspace: targetKeyspaceName, + Workflow: workflowName, + Direction: int32(DirectionBackward), + TabletTypes: tabletTypes, + DryRun: true, + }, + want: []string{ + fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), + fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tableName, targetKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), + fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), + fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), + fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", + targetKeyspaceName, tableName, targetKeyspaceName, position, targetKeyspaceName, position), + "Wait for vreplication on stopped streams to catchup for up to 30s", + fmt.Sprintf("Create reverse vreplication workflow %s", workflowName), + "Create journal entries on source databases", + fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", sourceKeyspaceName, tableName), + fmt.Sprintf("Switch routing from keyspace %s to keyspace %s", targetKeyspaceName, sourceKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d]", startingSourceTabletUID), + fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:0;tablet:%d;workflow:%s;dbname:vt_%s]", + sourceKeyspaceName, startingSourceTabletUID, ReverseWorkflowName(workflowName), sourceKeyspaceName), + fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), + fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + require.NotNil(t, tc.sourceKeyspace) + require.NotNil(t, tc.targetKeyspace) + require.NotNil(t, tc.req) + env := newTestEnv(t, ctx, defaultCellName, tc.sourceKeyspace, tc.targetKeyspace) + defer env.close() + env.tmc.schema = schema + if tc.req.Direction == int32(DirectionForward) { + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, copyTableQR) + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, journalQR) + } + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, lockTableQR) + } + } else { + env.tmc.reverse.Store(true) + // Setup the routing rules as they would be after having previously done SwitchTraffic. + ks := env.targetKeyspace.KeyspaceName + toTarget := []string{ks + "." + tableName} + rules := make(map[string][]string) + for _, tabletType := range tabletTypes { + tt := strings.ToLower(tabletType.String()) + if tabletType == topodatapb.TabletType_PRIMARY { + rules[tableName] = toTarget + rules[ks+"."+tableName] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget + } else { + rules[tableName+"@"+tt] = toTarget + rules[ks+"."+tableName+"@"+tt] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget } } + err := topotools.SaveRoutingRules(ctx, env.ts, rules) + require.NoError(t, err) + err = env.ts.RebuildSrvVSchema(ctx, nil) + require.NoError(t, err) + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, copyTableQR) + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, journalQR) + } + for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream + env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, lockTableQR) + } } + got, err := env.ws.WorkflowSwitchTraffic(ctx, tc.req) + require.NoError(t, err) + + require.EqualValues(t, tc.want, got.DryRunResults, "Server.WorkflowSwitchTraffic(DryRun:true) = %v, want %v", got.DryRunResults, tc.want) }) } } From 0550ea066d75e62ee03fbfbd2a1c4c605680bd98 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 7 Jun 2024 12:55:47 -0400 Subject: [PATCH 26/34] Improve dry run unit test Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 96811da0e85..79c49f40431 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -676,7 +676,7 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { name: "basic forward", sourceKeyspace: &testKeyspace{ KeyspaceName: sourceKeyspaceName, - ShardNames: []string{"0"}, + ShardNames: []string{"-80", "80-"}, }, targetKeyspace: &testKeyspace{ KeyspaceName: targetKeyspaceName, @@ -696,7 +696,8 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), - fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:0;position:%s]", sourceKeyspaceName, tableName, sourceKeyspaceName, position), + fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", + sourceKeyspaceName, tableName, sourceKeyspaceName, position, sourceKeyspaceName, position), "Wait for vreplication on stopped streams to catchup for up to 30s", fmt.Sprintf("Create reverse vreplication workflow %s", ReverseWorkflowName(workflowName)), "Create journal entries on source databases", @@ -714,7 +715,7 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { name: "basic backward", sourceKeyspace: &testKeyspace{ KeyspaceName: sourceKeyspaceName, - ShardNames: []string{"0"}, + ShardNames: []string{"-80", "80-"}, }, targetKeyspace: &testKeyspace{ KeyspaceName: targetKeyspaceName, @@ -742,9 +743,9 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", sourceKeyspaceName, tableName), fmt.Sprintf("Switch routing from keyspace %s to keyspace %s", targetKeyspaceName, sourceKeyspaceName), fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), - fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d]", startingSourceTabletUID), - fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:0;tablet:%d;workflow:%s;dbname:vt_%s]", - sourceKeyspaceName, startingSourceTabletUID, ReverseWorkflowName(workflowName), sourceKeyspaceName), + fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d,tablet:%d]", startingSourceTabletUID, startingSourceTabletUID+tabletUIDStep), + fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:-80;tablet:%d;workflow:%s;dbname:vt_%s,keyspace:%s;shard:80-;tablet:%d;workflow:%s;dbname:vt_%s]", + sourceKeyspaceName, startingSourceTabletUID, ReverseWorkflowName(workflowName), sourceKeyspaceName, sourceKeyspaceName, startingSourceTabletUID+tabletUIDStep, ReverseWorkflowName(workflowName), sourceKeyspaceName), fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), }, From f86792ce9c3625e7e649da0a2ec333c78a5ff623 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 7 Jun 2024 15:12:46 -0400 Subject: [PATCH 27/34] Dry run unit test improvements part II Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/server_test.go | 67 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index 79c49f40431..cee8c6ac247 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "slices" + "sort" "strings" "testing" "time" @@ -411,7 +412,7 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { topodatapb.TabletType_RDONLY, } schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ - "t1": { + tableName: { TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: tableName, @@ -633,7 +634,11 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { defer cancel() workflowName := "wf1" - tableName := "t1" + table1Name := "t1" + table2Name := "a1" + tables := []string{table1Name, table2Name} + sort.Strings(tables) + tablesStr := strings.Join(tables, ",") sourceKeyspaceName := "sourceks" targetKeyspaceName := "targetks" vrID := 1 @@ -643,11 +648,19 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { topodatapb.TabletType_RDONLY, } schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ - "t1": { + table1Name: { TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { - Name: tableName, - Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + Name: table1Name, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", table1Name), + }, + }, + }, + table2Name: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: table2Name, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", table2Name), }, }, }, @@ -662,7 +675,7 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { result: &querypb.QueryResult{}, } lockTableQR := &queryResult{ - query: fmt.Sprintf("LOCK TABLES `%s` READ", tableName), + query: fmt.Sprintf("LOCK TABLES `%s` READ,`%s` READ", table2Name, table1Name), result: &querypb.QueryResult{}, } @@ -691,19 +704,19 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { }, want: []string{ fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), - fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tableName, targetKeyspaceName), - fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Unlock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", - sourceKeyspaceName, tableName, sourceKeyspaceName, position, sourceKeyspaceName, position), + sourceKeyspaceName, tablesStr, sourceKeyspaceName, position, sourceKeyspaceName, position), "Wait for vreplication on stopped streams to catchup for up to 30s", fmt.Sprintf("Create reverse vreplication workflow %s", ReverseWorkflowName(workflowName)), "Create journal entries on source databases", - fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", targetKeyspaceName, tableName), + fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", targetKeyspaceName, tablesStr), fmt.Sprintf("Switch routing from keyspace %s to keyspace %s", sourceKeyspaceName, targetKeyspaceName), - fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d,tablet:%d]", startingTargetTabletUID, startingTargetTabletUID+tabletUIDStep), fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:-80;tablet:%d;workflow:%s;dbname:vt_%s,keyspace:%s;shard:80-;tablet:%d;workflow:%s;dbname:vt_%s]", targetKeyspaceName, startingTargetTabletUID, workflowName, targetKeyspaceName, targetKeyspaceName, startingTargetTabletUID+tabletUIDStep, workflowName, targetKeyspaceName), @@ -730,19 +743,19 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { }, want: []string{ fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), - fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tableName, targetKeyspaceName), - fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Switch reads for tables [%s] to keyspace %s for tablet types [REPLICA,RDONLY]", tablesStr, targetKeyspaceName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Unlock keyspace %s", targetKeyspaceName), fmt.Sprintf("Lock keyspace %s", targetKeyspaceName), fmt.Sprintf("Lock keyspace %s", sourceKeyspaceName), fmt.Sprintf("Stop writes on keyspace %s for tables [%s]: [keyspace:%s;shard:-80;position:%s,keyspace:%s;shard:80-;position:%s]", - targetKeyspaceName, tableName, targetKeyspaceName, position, targetKeyspaceName, position), + targetKeyspaceName, tablesStr, targetKeyspaceName, position, targetKeyspaceName, position), "Wait for vreplication on stopped streams to catchup for up to 30s", fmt.Sprintf("Create reverse vreplication workflow %s", workflowName), "Create journal entries on source databases", - fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", sourceKeyspaceName, tableName), + fmt.Sprintf("Enable writes on keyspace %s for tables [%s]", sourceKeyspaceName, tablesStr), fmt.Sprintf("Switch routing from keyspace %s to keyspace %s", targetKeyspaceName, sourceKeyspaceName), - fmt.Sprintf("Routing rules for tables [%s] will be updated", tableName), + fmt.Sprintf("Routing rules for tables [%s] will be updated", tablesStr), fmt.Sprintf("Switch writes completed, freeze and delete vreplication streams on: [tablet:%d,tablet:%d]", startingSourceTabletUID, startingSourceTabletUID+tabletUIDStep), fmt.Sprintf("Mark vreplication streams frozen on: [keyspace:%s;shard:-80;tablet:%d;workflow:%s;dbname:vt_%s,keyspace:%s;shard:80-;tablet:%d;workflow:%s;dbname:vt_%s]", sourceKeyspaceName, startingSourceTabletUID, ReverseWorkflowName(workflowName), sourceKeyspaceName, sourceKeyspaceName, startingSourceTabletUID+tabletUIDStep, ReverseWorkflowName(workflowName), sourceKeyspaceName), @@ -771,18 +784,20 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { env.tmc.reverse.Store(true) // Setup the routing rules as they would be after having previously done SwitchTraffic. ks := env.targetKeyspace.KeyspaceName - toTarget := []string{ks + "." + tableName} + toTarget := []string{ks + "." + table1Name} rules := make(map[string][]string) for _, tabletType := range tabletTypes { - tt := strings.ToLower(tabletType.String()) - if tabletType == topodatapb.TabletType_PRIMARY { - rules[tableName] = toTarget - rules[ks+"."+tableName] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget - } else { - rules[tableName+"@"+tt] = toTarget - rules[ks+"."+tableName+"@"+tt] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget + for _, tableName := range tables { + tt := strings.ToLower(tabletType.String()) + if tabletType == topodatapb.TabletType_PRIMARY { + rules[tableName] = toTarget + rules[ks+"."+tableName] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget + } else { + rules[tableName+"@"+tt] = toTarget + rules[ks+"."+tableName+"@"+tt] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget + } } } err := topotools.SaveRoutingRules(ctx, env.ts, rules) From 46e8b1f8663a49d9517c31363ecdd5beac0ff09f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 8 Jun 2024 00:37:41 -0400 Subject: [PATCH 28/34] Be consistent with test framework mutex Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index e183d3b0f6b..424e161fe9c 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -179,6 +179,9 @@ func newTestTMClient(env *testEnv) *testTMClient { } func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) (*tabletmanagerdatapb.CreateVReplicationWorkflowResponse, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + if expect := tmc.createVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { if !proto.Equal(expect, request) { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected CreateVReplicationWorkflow request: got %+v, want %+v", request, expect) @@ -189,6 +192,9 @@ func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet } func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.ReadVReplicationWorkflowRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowResponse, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + if expect := tmc.readVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { if !proto.Equal(expect, request) { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ReadVReplicationWorkflow request: got %+v, want %+v", request, expect) @@ -233,8 +239,6 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t } func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (response *tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, err error) { - tmc.mu.Lock() - defer tmc.mu.Unlock() return &tabletmanagerdatapb.DeleteVReplicationWorkflowResponse{ Result: &querypb.QueryResult{ RowsAffected: 1, @@ -243,6 +247,9 @@ func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet } func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.GetSchemaRequest) (*tabletmanagerdatapb.SchemaDefinition, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + schemaDefn := &tabletmanagerdatapb.SchemaDefinition{} for _, table := range request.Tables { if table == "/.*/" { @@ -291,22 +298,6 @@ func (tmc *testTMClient) expectCreateVReplicationWorkflowRequest(tabletID uint32 tmc.createVReplicationWorkflowRequests[tabletID] = req } -func (tmc *testTMClient) verifyQueries(t *testing.T) { - t.Helper() - tmc.mu.Lock() - defer tmc.mu.Unlock() - - for tabletID, qrs := range tmc.vrQueries { - if len(qrs) != 0 { - var list []string - for _, qr := range qrs { - list = append(list, qr.query) - } - require.Failf(t, "missing query", "tablet %v: found queries that were expected but never got executed by the test: %v", tabletID, list) - } - } -} - func (tmc *testTMClient) VReplicationExec(ctx context.Context, tablet *topodatapb.Tablet, query string) (*querypb.QueryResult, error) { tmc.mu.Lock() defer tmc.mu.Unlock() @@ -372,6 +363,9 @@ func (tmc *testTMClient) HasVReplicationWorkflows(ctx context.Context, tablet *t } func (tmc *testTMClient) ReadVReplicationWorkflows(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.ReadVReplicationWorkflowsRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowsResponse, error) { + tmc.mu.Lock() + defer tmc.mu.Unlock() + workflowType := binlogdatapb.VReplicationWorkflowType_MoveTables if len(req.IncludeWorkflows) > 0 { for _, wf := range req.IncludeWorkflows { From ae34e945f59e602dd075b7ee2711787274f936fe Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 8 Jun 2024 00:41:55 -0400 Subject: [PATCH 29/34] Use std parameter name Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index 424e161fe9c..f05e1e201fa 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -178,34 +178,34 @@ func newTestTMClient(env *testEnv) *testTMClient { } } -func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) (*tabletmanagerdatapb.CreateVReplicationWorkflowResponse, error) { +func (tmc *testTMClient) CreateVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.CreateVReplicationWorkflowRequest) (*tabletmanagerdatapb.CreateVReplicationWorkflowResponse, error) { tmc.mu.Lock() defer tmc.mu.Unlock() if expect := tmc.createVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { - if !proto.Equal(expect, request) { - return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected CreateVReplicationWorkflow request: got %+v, want %+v", request, expect) + if !proto.Equal(expect, req) { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected CreateVReplicationWorkflow request: got %+v, want %+v", req, expect) } } res := sqltypes.MakeTestResult(sqltypes.MakeTestFields("rowsaffected", "int64"), "1") return &tabletmanagerdatapb.CreateVReplicationWorkflowResponse{Result: sqltypes.ResultToProto3(res)}, nil } -func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.ReadVReplicationWorkflowRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowResponse, error) { +func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.ReadVReplicationWorkflowRequest) (*tabletmanagerdatapb.ReadVReplicationWorkflowResponse, error) { tmc.mu.Lock() defer tmc.mu.Unlock() if expect := tmc.readVReplicationWorkflowRequests[tablet.Alias.Uid]; expect != nil { - if !proto.Equal(expect, request) { - return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ReadVReplicationWorkflow request: got %+v, want %+v", request, expect) + if !proto.Equal(expect, req) { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ReadVReplicationWorkflow request: got %+v, want %+v", req, expect) } } workflowType := binlogdatapb.VReplicationWorkflowType_MoveTables - if strings.Contains(request.Workflow, "lookup") { + if strings.Contains(req.Workflow, "lookup") { workflowType = binlogdatapb.VReplicationWorkflowType_CreateLookupIndex } res := &tabletmanagerdatapb.ReadVReplicationWorkflowResponse{ - Workflow: request.Workflow, + Workflow: req.Workflow, WorkflowType: workflowType, Streams: make([]*tabletmanagerdatapb.ReadVReplicationWorkflowResponse_Stream, 0, 2), } @@ -238,7 +238,7 @@ func (tmc *testTMClient) ReadVReplicationWorkflow(ctx context.Context, tablet *t return res, nil } -func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (response *tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, err error) { +func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.DeleteVReplicationWorkflowRequest) (response *tabletmanagerdatapb.DeleteVReplicationWorkflowResponse, err error) { return &tabletmanagerdatapb.DeleteVReplicationWorkflowResponse{ Result: &querypb.QueryResult{ RowsAffected: 1, @@ -246,12 +246,12 @@ func (tmc *testTMClient) DeleteVReplicationWorkflow(ctx context.Context, tablet }, nil } -func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.GetSchemaRequest) (*tabletmanagerdatapb.SchemaDefinition, error) { +func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, req *tabletmanagerdatapb.GetSchemaRequest) (*tabletmanagerdatapb.SchemaDefinition, error) { tmc.mu.Lock() defer tmc.mu.Unlock() schemaDefn := &tabletmanagerdatapb.SchemaDefinition{} - for _, table := range request.Tables { + for _, table := range req.Tables { if table == "/.*/" { // Special case of all tables in keyspace. for key, tableDefn := range tmc.schema { From b308ed718a752b68b278ee67b9d047fa200e9024 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 8 Jun 2024 00:44:46 -0400 Subject: [PATCH 30/34] Remove extraneous changes Signed-off-by: Matt Lord --- go/vt/vttablet/tabletmanager/rpc_vreplication_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go index 691f5a00d63..789319a2a53 100644 --- a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go +++ b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go @@ -477,6 +477,7 @@ func TestMoveTables(t *testing.T) { AutoStart: true, }) require.NoError(t, err) + for _, ftc := range targetShards { ftc.vrdbClient.ExpectRequest(fmt.Sprintf(readWorkflowsLimited, tenv.dbName, wf), sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -503,7 +504,6 @@ func TestMoveTables(t *testing.T) { InitializeTargetSequences: true, Direction: int32(workflow.DirectionForward), }) - require.NoError(t, err) for _, ftc := range targetShards { From 93613ac09dac962e19d61a6113bec26dead560d6 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 8 Jun 2024 11:56:46 -0400 Subject: [PATCH 31/34] Correct log message Signed-off-by: Matt Lord --- go/vt/topo/shard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/topo/shard.go b/go/vt/topo/shard.go index f3d88099230..77983f20d2d 100644 --- a/go/vt/topo/shard.go +++ b/go/vt/topo/shard.go @@ -477,7 +477,7 @@ func (si *ShardInfo) updatePrimaryTabletControl(tc *topodatapb.Shard_TabletContr if len(newTables) != len(tables) { // Some of the tables already existed in the DeniedTables list so we don't // need to add them. - log.Warningf("%s:%s", dlTablesAlreadyPresent, strings.Join(newTables, ",")) + log.Warningf("%s:%s", dlTablesAlreadyPresent, strings.Join(tables, ",")) // We do need to merge the lists, however. tables = append(tables, newTables...) tc.DeniedTables = append(tc.DeniedTables, tables...) From f66d6c2357324ba32ae006bf0c2d62f316da0cba Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 9 Jun 2024 12:51:44 -0400 Subject: [PATCH 32/34] Add routing rules helper to test framework Signed-off-by: Matt Lord --- go/vt/vtctl/workflow/framework_test.go | 27 ++++++++++++++++ go/vt/vtctl/workflow/server_test.go | 43 ++------------------------ 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index f05e1e201fa..73b34015338 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -36,6 +36,7 @@ import ( "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -150,6 +151,32 @@ func (env *testEnv) addTablet(t *testing.T, ctx context.Context, id int, keyspac return tablet } +// addTableRoutingRules adds routing rules from the test env's source keyspace to +// its target keyspace for the given tablet types and tables. +func (env *testEnv) addTableRoutingRules(t *testing.T, ctx context.Context, tabletTypes []topodatapb.TabletType, tables []string) { + ks := env.targetKeyspace.KeyspaceName + rules := make(map[string][]string, len(tables)*(len(tabletTypes)*3)) + for _, tabletType := range tabletTypes { + for _, tableName := range tables { + toTarget := []string{ks + "." + tableName} + tt := strings.ToLower(tabletType.String()) + if tabletType == topodatapb.TabletType_PRIMARY { + rules[tableName] = toTarget + rules[ks+"."+tableName] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget + } else { + rules[tableName+"@"+tt] = toTarget + rules[ks+"."+tableName+"@"+tt] = toTarget + rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget + } + } + } + err := topotools.SaveRoutingRules(ctx, env.ts, rules) + require.NoError(t, err) + err = env.ts.RebuildSrvVSchema(ctx, nil) + require.NoError(t, err) +} + func (env *testEnv) deleteTablet(tablet *topodatapb.Tablet) { _ = env.ts.DeleteTablet(context.Background(), tablet.Alias) delete(env.tablets[tablet.Keyspace], int(tablet.Alias.Uid)) diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index cee8c6ac247..fb432403155 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -34,7 +34,6 @@ import ( "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -555,25 +554,7 @@ func TestMoveTablesTrafficSwitching(t *testing.T) { } else { env.tmc.reverse.Store(true) // Setup the routing rules as they would be after having previously done SwitchTraffic. - ks := env.targetKeyspace.KeyspaceName - toTarget := []string{ks + "." + tableName} - rules := make(map[string][]string) - for _, tabletType := range tabletTypes { - tt := strings.ToLower(tabletType.String()) - if tabletType == topodatapb.TabletType_PRIMARY { - rules[tableName] = toTarget - rules[ks+"."+tableName] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget - } else { - rules[tableName+"@"+tt] = toTarget - rules[ks+"."+tableName+"@"+tt] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget - } - } - err := topotools.SaveRoutingRules(ctx, env.ts, rules) - require.NoError(t, err) - err = env.ts.RebuildSrvVSchema(ctx, nil) - require.NoError(t, err) + env.addTableRoutingRules(t, ctx, tabletTypes, []string{tableName}) env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, copyTableQR) for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, cutoverQR) @@ -783,27 +764,7 @@ func TestMoveTablesTrafficSwitchingDryRun(t *testing.T) { } else { env.tmc.reverse.Store(true) // Setup the routing rules as they would be after having previously done SwitchTraffic. - ks := env.targetKeyspace.KeyspaceName - toTarget := []string{ks + "." + table1Name} - rules := make(map[string][]string) - for _, tabletType := range tabletTypes { - for _, tableName := range tables { - tt := strings.ToLower(tabletType.String()) - if tabletType == topodatapb.TabletType_PRIMARY { - rules[tableName] = toTarget - rules[ks+"."+tableName] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName] = toTarget - } else { - rules[tableName+"@"+tt] = toTarget - rules[ks+"."+tableName+"@"+tt] = toTarget - rules[env.sourceKeyspace.KeyspaceName+"."+tableName+"@"+tt] = toTarget - } - } - } - err := topotools.SaveRoutingRules(ctx, env.ts, rules) - require.NoError(t, err) - err = env.ts.RebuildSrvVSchema(ctx, nil) - require.NoError(t, err) + env.addTableRoutingRules(t, ctx, tabletTypes, tables) env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.sourceKeyspace.KeyspaceName, copyTableQR) for i := 0; i < len(tc.targetKeyspace.ShardNames); i++ { // Per stream env.tmc.expectVRQueryResultOnKeyspaceTablets(tc.targetKeyspace.KeyspaceName, journalQR) From 4fc0037a7bcb68e03c07f814c5f2f4e0715b2820 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 10 Jun 2024 10:54:00 -0400 Subject: [PATCH 33/34] Trim down PR These changes did not end up being necessary. Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/fk_test.go | 4 +- go/test/endtoend/vreplication/helper_test.go | 23 +++++------ .../vreplication/initial_data_test.go | 30 +++++++------- .../endtoend/vreplication/materialize_test.go | 2 +- go/test/endtoend/vreplication/migrate_test.go | 22 +++++------ .../partial_movetables_seq_test.go | 6 +-- .../endtoend/vreplication/reference_test.go | 2 +- .../resharding_workflows_v2_test.go | 22 +++++------ .../endtoend/vreplication/time_zone_test.go | 4 +- go/test/endtoend/vreplication/vdiff2_test.go | 8 ++-- .../vreplication/vdiff_helper_test.go | 4 +- .../vreplication/vdiff_online_ddl_test.go | 2 +- .../vreplication/vreplication_test.go | 39 ++++++++++--------- go/test/endtoend/vreplication/vstream_test.go | 8 ++-- 14 files changed, 86 insertions(+), 90 deletions(-) diff --git a/go/test/endtoend/vreplication/fk_test.go b/go/test/endtoend/vreplication/fk_test.go index 69f8b8a63be..09692930c5c 100644 --- a/go/test/endtoend/vreplication/fk_test.go +++ b/go/test/endtoend/vreplication/fk_test.go @@ -222,7 +222,7 @@ func (ls *fkLoadSimulator) simulateLoad() { func (ls *fkLoadSimulator) getNumRowsParent(vtgateConn *mysql.Conn) int { t := ls.t - qr := execQueryWithDatabase(t, vtgateConn, "fksource", "SELECT COUNT(*) FROM parent") + qr := execVtgateQuery(t, vtgateConn, "fksource", "SELECT COUNT(*) FROM parent") require.NotNil(t, qr) numRows, err := strconv.Atoi(qr.Rows[0][0].ToString()) require.NoError(t, err) @@ -296,7 +296,7 @@ func (ls *fkLoadSimulator) exec(query string) *sqltypes.Result { t := ls.t vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execQueryWithDatabase(t, vtgateConn, "fksource", query) + qr := execVtgateQuery(t, vtgateConn, "fksource", query) require.NotNil(t, qr) return qr } diff --git a/go/test/endtoend/vreplication/helper_test.go b/go/test/endtoend/vreplication/helper_test.go index 8126b41fa76..eca4c312ae7 100644 --- a/go/test/endtoend/vreplication/helper_test.go +++ b/go/test/endtoend/vreplication/helper_test.go @@ -72,7 +72,7 @@ func execMultipleQueries(t *testing.T, conn *mysql.Conn, database string, lines if strings.HasPrefix(query, "--") { continue } - execQueryWithDatabase(t, conn, database, string(query)) + execVtgateQuery(t, conn, database, string(query)) } } @@ -134,7 +134,7 @@ func getConnection(t *testing.T, hostname string, port int) *mysql.Conn { return conn } -func execQueryWithDatabase(t *testing.T, conn *mysql.Conn, database string, query string) *sqltypes.Result { +func execVtgateQuery(t *testing.T, conn *mysql.Conn, database string, query string) *sqltypes.Result { if strings.TrimSpace(query) == "" { return nil } @@ -158,7 +158,7 @@ func waitForQueryResult(t *testing.T, conn *mysql.Conn, database string, query s timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - qr := execQueryWithDatabase(t, conn, database, query) + qr := execVtgateQuery(t, conn, database, query) require.NotNil(t, qr) if want == fmt.Sprintf("%v", qr.Rows) { return @@ -232,7 +232,7 @@ func waitForNoWorkflowLag(t *testing.T, vc *VitessCluster, keyspace, worfklow st // verifyNoInternalTables can e.g. be used to confirm that no internal tables were // copied from a source to a target during a MoveTables or Reshard operation. func verifyNoInternalTables(t *testing.T, conn *mysql.Conn, keyspaceShard string) { - qr := execQueryWithDatabase(t, conn, keyspaceShard, "show tables") + qr := execVtgateQuery(t, conn, keyspaceShard, "show tables") require.NotNil(t, qr) require.NotNil(t, qr.Rows) for _, row := range qr.Rows { @@ -247,7 +247,7 @@ func waitForRowCount(t *testing.T, conn *mysql.Conn, database string, table stri timer := time.NewTimer(defaultTimeout) defer timer.Stop() for { - qr := execQueryWithDatabase(t, conn, database, query) + qr := execVtgateQuery(t, conn, database, query) require.NotNil(t, qr) if wantRes == fmt.Sprintf("%v", qr.Rows) { return @@ -319,7 +319,7 @@ func executeOnTablet(t *testing.T, conn *mysql.Conn, tablet *cluster.VttabletPro count0, body0 := getQueryCount(t, queryStatsURL, matchQuery) - qr := execQueryWithDatabase(t, conn, ksName, query) + qr := execVtgateQuery(t, conn, ksName, query) require.NotNil(t, qr) count1, body1 := getQueryCount(t, queryStatsURL, matchQuery) @@ -595,15 +595,10 @@ func expectNumberOfStreams(t *testing.T, vtgateConn *mysql.Conn, name string, wo // confirmAllStreamsRunning confirms that all of the workflow's streams are // in the running state. -func confirmAllStreamsRunning(t *testing.T, keyspace, shard string) { +func confirmAllStreamsRunning(t *testing.T, vtgateConn *mysql.Conn, database string) { query := sqlparser.BuildParsedQuery("select count(*) from %s.vreplication where state != '%s'", sidecarDBIdentifier, binlogdatapb.VReplicationWorkflowState_Running.String()).Query - tablet := vc.getPrimaryTablet(t, keyspace, shard) - // Query the tablet's mysqld directly as the target may have denied table entries. - dbc, err := tablet.TabletConn(keyspace, true) - require.NoError(t, err) - defer dbc.Close() - waitForQueryResult(t, dbc, sidecarDBName, query, `[[INT64(0)]]`) + waitForQueryResult(t, vtgateConn, database, query, `[[INT64(0)]]`) } func printShardPositions(vc *VitessCluster, ksShards []string) { @@ -1005,7 +1000,7 @@ func vexplain(t *testing.T, database, query string) *VExplainPlan { vtgateConn := vc.GetVTGateConn(t) defer vtgateConn.Close() - qr := execQueryWithDatabase(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) + qr := execVtgateQuery(t, vtgateConn, database, fmt.Sprintf("vexplain %s", query)) require.NotNil(t, qr) require.Equal(t, 1, len(qr.Rows)) json := qr.Rows[0][0].ToString() diff --git a/go/test/endtoend/vreplication/initial_data_test.go b/go/test/endtoend/vreplication/initial_data_test.go index 22cd9c50c33..ea34ef7fddf 100644 --- a/go/test/endtoend/vreplication/initial_data_test.go +++ b/go/test/endtoend/vreplication/initial_data_test.go @@ -32,9 +32,9 @@ func insertInitialData(t *testing.T) { log.Infof("Inserting initial data") lines, _ := os.ReadFile("unsharded_init_data.sql") execMultipleQueries(t, vtgateConn, "product:0", string(lines)) - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into customer_seq(id, next_id, cache) values(0, 100, 100);") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into order_seq(id, next_id, cache) values(0, 100, 100);") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into customer_seq2(id, next_id, cache) values(0, 100, 100);") + execVtgateQuery(t, vtgateConn, "product:0", "insert into customer_seq(id, next_id, cache) values(0, 100, 100);") + execVtgateQuery(t, vtgateConn, "product:0", "insert into order_seq(id, next_id, cache) values(0, 100, 100);") + execVtgateQuery(t, vtgateConn, "product:0", "insert into customer_seq2(id, next_id, cache) values(0, 100, 100);") log.Infof("Done inserting initial data") waitForRowCount(t, vtgateConn, "product:0", "product", 2) @@ -52,12 +52,12 @@ func insertJSONValues(t *testing.T) { // insert null value combinations vtgateConn, closeConn := getVTGateConn() defer closeConn() - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(1, \"{}\")") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j3) values(2, \"{}\", \"{}\")") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j2, j3) values(3, \"{}\", \"{}\")") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j2, j3) values(4, NULL, 'null', '\"null\"')") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(5, JSON_QUOTE('null'))") - execQueryWithDatabase(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(6, '{}')") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(1, \"{}\")") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j3) values(2, \"{}\", \"{}\")") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j2, j3) values(3, \"{}\", \"{}\")") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j1, j2, j3) values(4, NULL, 'null', '\"null\"')") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(5, JSON_QUOTE('null'))") + execVtgateQuery(t, vtgateConn, "product:0", "insert into json_tbl(id, j3) values(6, '{}')") id := 8 // 6 inserted above and one after copy phase is done @@ -68,7 +68,7 @@ func insertJSONValues(t *testing.T) { j1 := rand.IntN(numJsonValues) j2 := rand.IntN(numJsonValues) query := fmt.Sprintf(q, id, jsonValues[j1], jsonValues[j2]) - execQueryWithDatabase(t, vtgateConn, "product:0", query) + execVtgateQuery(t, vtgateConn, "product:0", query) } } @@ -97,28 +97,28 @@ func insertMoreCustomers(t *testing.T, numCustomers int) { } cid++ } - execQueryWithDatabase(t, vtgateConn, "customer", sql) + execVtgateQuery(t, vtgateConn, "customer", sql) } func insertMoreProducts(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(3, 'cpu'),(4, 'camera'),(5, 'mouse');" - execQueryWithDatabase(t, vtgateConn, "product", sql) + execVtgateQuery(t, vtgateConn, "product", sql) } func insertMoreProductsForSourceThrottler(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(103, 'new-cpu'),(104, 'new-camera'),(105, 'new-mouse');" - execQueryWithDatabase(t, vtgateConn, "product", sql) + execVtgateQuery(t, vtgateConn, "product", sql) } func insertMoreProductsForTargetThrottler(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() sql := "insert into product(pid, description) values(203, 'new-cpu'),(204, 'new-camera'),(205, 'new-mouse');" - execQueryWithDatabase(t, vtgateConn, "product", sql) + execVtgateQuery(t, vtgateConn, "product", sql) } var blobTableQueries = []string{ @@ -137,6 +137,6 @@ func insertIntoBlobTable(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() for _, query := range blobTableQueries { - execQueryWithDatabase(t, vtgateConn, "product:0", query) + execVtgateQuery(t, vtgateConn, "product:0", query) } } diff --git a/go/test/endtoend/vreplication/materialize_test.go b/go/test/endtoend/vreplication/materialize_test.go index 3297d83acd7..486692a58ba 100644 --- a/go/test/endtoend/vreplication/materialize_test.go +++ b/go/test/endtoend/vreplication/materialize_test.go @@ -206,7 +206,7 @@ func testMaterialize(t *testing.T, useVtctldClient bool) { waitForQueryResult(t, vtgateConn, targetKs, "select id, val, ts, day, month, x from mat2", want) // insert data to test the replication phase - execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into mat(id, val, ts) values (3, 'ghi', '2021-12-11 16:17:36')") + execVtgateQuery(t, vtgateConn, sourceKs, "insert into mat(id, val, ts) values (3, 'ghi', '2021-12-11 16:17:36')") // validate data after the replication phase waitForQueryResult(t, vtgateConn, targetKs, "select count(*) from mat2", "[[INT64(3)]]") diff --git a/go/test/endtoend/vreplication/migrate_test.go b/go/test/endtoend/vreplication/migrate_test.go index f2d6b62024c..1f365c47600 100644 --- a/go/test/endtoend/vreplication/migrate_test.go +++ b/go/test/endtoend/vreplication/migrate_test.go @@ -31,11 +31,11 @@ import ( func insertInitialDataIntoExternalCluster(t *testing.T, conn *mysql.Conn) { t.Run("insertInitialData", func(t *testing.T) { fmt.Printf("Inserting initial data\n") - execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(1, 1, 'review1');") - execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(2, 1, 'review2');") - execQueryWithDatabase(t, conn, "rating:0", "insert into review(rid, pid, review) values(3, 2, 'review3');") - execQueryWithDatabase(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(1, 1, 4);") - execQueryWithDatabase(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(2, 2, 5);") + execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(1, 1, 'review1');") + execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(2, 1, 'review2');") + execVtgateQuery(t, conn, "rating:0", "insert into review(rid, pid, review) values(3, 2, 'review3');") + execVtgateQuery(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(1, 1, 4);") + execVtgateQuery(t, conn, "rating:0", "insert into rating(gid, pid, rating) values(2, 2, 5);") }) } @@ -109,8 +109,8 @@ func TestVtctlMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 1) waitForRowCount(t, vtgateConn, "product:0", "rating", 2) waitForRowCount(t, vtgateConn, "product:0", "review", 3) - execQueryWithDatabase(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") - execQueryWithDatabase(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") + execVtgateQuery(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") + execVtgateQuery(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") waitForRowCount(t, vtgateConn, "product:0", "rating", 3) waitForRowCount(t, vtgateConn, "product:0", "review", 4) vdiffSideBySide(t, ksWorkflow, "extcell1") @@ -122,7 +122,7 @@ func TestVtctlMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 0) }) t.Run("cancel migrate workflow", func(t *testing.T) { - execQueryWithDatabase(t, vtgateConn, "product", "drop table review,rating") + execVtgateQuery(t, vtgateConn, "product", "drop table review,rating") if output, err = vc.VtctlClient.ExecuteCommandWithOutput("Migrate", "--", "--all", "--auto_start=false", "--cells=extcell1", "--source=ext1.rating", "create", ksWorkflow); err != nil { @@ -234,8 +234,8 @@ func TestVtctldMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 1) waitForRowCount(t, vtgateConn, "product:0", "rating", 2) waitForRowCount(t, vtgateConn, "product:0", "review", 3) - execQueryWithDatabase(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") - execQueryWithDatabase(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") + execVtgateQuery(t, extVtgateConn, "rating", "insert into review(rid, pid, review) values(4, 1, 'review4');") + execVtgateQuery(t, extVtgateConn, "rating", "insert into rating(gid, pid, rating) values(3, 1, 3);") waitForRowCount(t, vtgateConn, "product:0", "rating", 3) waitForRowCount(t, vtgateConn, "product:0", "review", 4) vdiffSideBySide(t, ksWorkflow, "extcell1") @@ -261,7 +261,7 @@ func TestVtctldMigrate(t *testing.T) { expectNumberOfStreams(t, vtgateConn, "migrate", "e1", "product:0", 0) }) t.Run("cancel migrate workflow", func(t *testing.T) { - execQueryWithDatabase(t, vtgateConn, "product", "drop table review,rating") + execVtgateQuery(t, vtgateConn, "product", "drop table review,rating") output, err = vc.VtctldClient.ExecuteCommandWithOutput("Migrate", "--target-keyspace", "product", "--workflow", "e1", "Create", "--source-keyspace", "rating", "--mount-name", "ext1", "--all-tables", "--auto-start=false", "--cells=extcell1") diff --git a/go/test/endtoend/vreplication/partial_movetables_seq_test.go b/go/test/endtoend/vreplication/partial_movetables_seq_test.go index d57e18f4bf1..eec304e0a4d 100644 --- a/go/test/endtoend/vreplication/partial_movetables_seq_test.go +++ b/go/test/endtoend/vreplication/partial_movetables_seq_test.go @@ -532,7 +532,7 @@ var lastCustomerId int64 func getCustomerCount(t *testing.T, msg string) int64 { vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execQueryWithDatabase(t, vtgateConn, "", "select count(*) from customer") + qr := execVtgateQuery(t, vtgateConn, "", "select count(*) from customer") require.NotNil(t, qr) count, err := qr.Rows[0][0].ToInt64() require.NoError(t, err) @@ -542,7 +542,7 @@ func getCustomerCount(t *testing.T, msg string) int64 { func confirmLastCustomerIdHasIncreased(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() - qr := execQueryWithDatabase(t, vtgateConn, "", "select cid from customer order by cid desc limit 1") + qr := execVtgateQuery(t, vtgateConn, "", "select cid from customer order by cid desc limit 1") require.NotNil(t, qr) currentCustomerId, err := qr.Rows[0][0].ToInt64() require.NoError(t, err) @@ -554,7 +554,7 @@ func insertCustomers(t *testing.T) { vtgateConn, closeConn := getVTGateConn() defer closeConn() for i := int64(1); i < newCustomerCount+1; i++ { - execQueryWithDatabase(t, vtgateConn, "customer@primary", fmt.Sprintf("insert into customer(name) values ('name-%d')", currentCustomerCount+i)) + execVtgateQuery(t, vtgateConn, "customer@primary", fmt.Sprintf("insert into customer(name) values ('name-%d')", currentCustomerCount+i)) } customerCount = getCustomerCount(t, "") require.Equal(t, currentCustomerCount+newCustomerCount, customerCount) diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index 969e0bd2c88..8ff77de8708 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -146,7 +146,7 @@ func TestReferenceTableMaterializationAndRouting(t *testing.T) { execRefQuery(t, "update sks.mfg2 set name = concat(name, '-updated') where id = 4") waitForRowCount(t, vtgateConn, uks, "mfg", 8) - qr := execQueryWithDatabase(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") + qr := execVtgateQuery(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") require.NotNil(t, qr) require.Equal(t, "4", qr.Rows[0][0].ToString()) diff --git a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go index 862ab31cb4d..82c859acb40 100644 --- a/go/test/endtoend/vreplication/resharding_workflows_v2_test.go +++ b/go/test/endtoend/vreplication/resharding_workflows_v2_test.go @@ -359,7 +359,7 @@ func validateWritesRouteToSource(t *testing.T) { insertQuery := "insert into customer(name, cid) values('tempCustomer2', 200)" matchInsertQuery := "insert into customer(`name`, cid) values" assertQueryExecutesOnTablet(t, vtgateConn, sourceTab, "customer", insertQuery, matchInsertQuery) - execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where cid = 200") + execVtgateQuery(t, vtgateConn, "customer", "delete from customer where cid = 200") } func validateWritesRouteToTarget(t *testing.T) { @@ -370,7 +370,7 @@ func validateWritesRouteToTarget(t *testing.T) { assertQueryExecutesOnTablet(t, vtgateConn, targetTab2, "customer", insertQuery, matchInsertQuery) insertQuery = "insert into customer(name, cid) values('tempCustomer3', 102)" assertQueryExecutesOnTablet(t, vtgateConn, targetTab1, "customer", insertQuery, matchInsertQuery) - execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where cid in (101, 102)") + execVtgateQuery(t, vtgateConn, "customer", "delete from customer where cid in (101, 102)") } func revert(t *testing.T, workflowType string) { @@ -470,7 +470,7 @@ func testVSchemaForSequenceAfterMoveTables(t *testing.T) { // ensure sequence is available to vtgate num := 5 for i := 0; i < num; i++ { - execQueryWithDatabase(t, vtgateConn, "customer", "insert into customer2(name) values('a')") + execVtgateQuery(t, vtgateConn, "customer", "insert into customer2(name) values('a')") } waitForRowCount(t, vtgateConn, "customer", "customer2", 3+num) want := fmt.Sprintf("[[INT32(%d)]]", 100+num-1) @@ -502,10 +502,10 @@ func testVSchemaForSequenceAfterMoveTables(t *testing.T) { // ensure sequence is available to vtgate for i := 0; i < num; i++ { - execQueryWithDatabase(t, vtgateConn, "product", "insert into customer2(name) values('a')") + execVtgateQuery(t, vtgateConn, "product", "insert into customer2(name) values('a')") } waitForRowCount(t, vtgateConn, "product", "customer2", 3+num+num) - res := execQueryWithDatabase(t, vtgateConn, "product", "select max(cid) from customer2") + res := execVtgateQuery(t, vtgateConn, "product", "select max(cid) from customer2") cid, err := res.Rows[0][0].ToInt() require.NoError(t, err) require.GreaterOrEqual(t, cid, 100+num+num-1) @@ -527,10 +527,10 @@ func testReplicatingWithPKEnumCols(t *testing.T) { // typ is an enum, with soho having a stored and binlogged value of 2 deleteQuery := "delete from customer where cid = 2 and typ = 'soho'" insertQuery := "insert into customer(cid, name, typ, sport, meta) values(2, 'Paül','soho','cricket',convert(x'7b7d' using utf8mb4))" - execQueryWithDatabase(t, vtgateConn, sourceKs, deleteQuery) + execVtgateQuery(t, vtgateConn, sourceKs, deleteQuery) waitForNoWorkflowLag(t, vc, targetKs, workflowName) vdiffSideBySide(t, ksWorkflow, "") - execQueryWithDatabase(t, vtgateConn, sourceKs, insertQuery) + execVtgateQuery(t, vtgateConn, sourceKs, insertQuery) waitForNoWorkflowLag(t, vc, targetKs, workflowName) vdiffSideBySide(t, ksWorkflow, "") } @@ -559,7 +559,7 @@ func testReshardV2Workflow(t *testing.T) { return default: // Use a random customer type for each record. - _ = execQueryWithDatabase(t, dataGenConn, "customer", fmt.Sprintf("insert into customer (cid, name, typ) values (%d, 'tempCustomer%d', %s)", + _ = execVtgateQuery(t, dataGenConn, "customer", fmt.Sprintf("insert into customer (cid, name, typ) values (%d, 'tempCustomer%d', %s)", id, id, customerTypes[rand.IntN(len(customerTypes))])) } time.Sleep(1 * time.Millisecond) @@ -590,17 +590,17 @@ func testReshardV2Workflow(t *testing.T) { // Confirm that we lost no customer related writes during the Reshard. dataGenCancel() dataGenWg.Wait() - cres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from customer") + cres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from customer") require.Len(t, cres.Rows, 1) waitForNoWorkflowLag(t, vc, "customer", "customer_name") - cnres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from customer_name") + cnres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from customer_name") require.Len(t, cnres.Rows, 1) require.EqualValues(t, cres.Rows, cnres.Rows) if debugMode { // We expect the row count to differ in enterprise_customer because it is // using a `where typ='enterprise'` filter. So the count is only for debug // info. - ecres := execQueryWithDatabase(t, dataGenConn, "customer", "select count(*) from enterprise_customer") + ecres := execVtgateQuery(t, dataGenConn, "customer", "select count(*) from enterprise_customer") t.Logf("Done inserting customer data. Record counts in customer: %s, customer_name: %s, enterprise_customer: %s", cres.Rows[0][0].ToString(), cnres.Rows[0][0].ToString(), ecres.Rows[0][0].ToString()) } diff --git a/go/test/endtoend/vreplication/time_zone_test.go b/go/test/endtoend/vreplication/time_zone_test.go index cb271e04c8a..2c0a9a4f5a5 100644 --- a/go/test/endtoend/vreplication/time_zone_test.go +++ b/go/test/endtoend/vreplication/time_zone_test.go @@ -190,7 +190,7 @@ func TestMoveTablesTZ(t *testing.T) { } // inserts to test date conversions in reverse replication - execQueryWithDatabase(t, vtgateConn, "customer", "insert into datze(id, dt2) values (13, '2022-01-01 18:20:30')") - execQueryWithDatabase(t, vtgateConn, "customer", "insert into datze(id, dt2) values (14, '2022-04-01 12:06:07')") + execVtgateQuery(t, vtgateConn, "customer", "insert into datze(id, dt2) values (13, '2022-01-01 18:20:30')") + execVtgateQuery(t, vtgateConn, "customer", "insert into datze(id, dt2) values (14, '2022-04-01 12:06:07')") vdiffSideBySide(t, ksReverseWorkflow, "") } diff --git a/go/test/endtoend/vreplication/vdiff2_test.go b/go/test/endtoend/vreplication/vdiff2_test.go index 7fb37430768..08f5bb8926d 100644 --- a/go/test/endtoend/vreplication/vdiff2_test.go +++ b/go/test/endtoend/vreplication/vdiff2_test.go @@ -164,13 +164,13 @@ func TestVDiff2(t *testing.T) { // Insert null and empty enum values for testing vdiff comparisons for those values. // If we add this to the initial data list, the counts in several other tests will need to change query := `insert into customer(cid, name, typ, sport) values(1001, null, 'soho','')` - execQueryWithDatabase(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) + execVtgateQuery(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) generateMoreCustomers(t, sourceKs, 1000) // Create rows in the nopk table using the customer names and random ages between 20 and 100. query = "insert into nopk(name, age) select name, floor(rand()*80)+20 from customer" - execQueryWithDatabase(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) + execVtgateQuery(t, vtgateConn, fmt.Sprintf("%s:%s", sourceKs, sourceShards[0]), query) // The primary tablet is only added in the first cell. // We ONLY add primary tablets in this test. @@ -502,7 +502,7 @@ func testResume(t *testing.T, tc *testCase, cells string) { expectedNewRows := int64(0) if tc.resumeInsert != "" { - res := execQueryWithDatabase(t, vtgateConn, tc.sourceKs, tc.resumeInsert) + res := execVtgateQuery(t, vtgateConn, tc.sourceKs, tc.resumeInsert) expectedNewRows = int64(res.RowsAffected) } expectedRows := rowsCompared + expectedNewRows @@ -549,7 +549,7 @@ func testAutoRetryError(t *testing.T, tc *testCase, cells string) { // compared is cumulative. expectedNewRows := int64(0) if tc.retryInsert != "" { - res := execQueryWithDatabase(t, vtgateConn, tc.sourceKs, tc.retryInsert) + res := execVtgateQuery(t, vtgateConn, tc.sourceKs, tc.retryInsert) expectedNewRows = int64(res.RowsAffected) } expectedRows := rowsCompared + expectedNewRows diff --git a/go/test/endtoend/vreplication/vdiff_helper_test.go b/go/test/endtoend/vreplication/vdiff_helper_test.go index a2a967df3f1..53e19e56731 100644 --- a/go/test/endtoend/vreplication/vdiff_helper_test.go +++ b/go/test/endtoend/vreplication/vdiff_helper_test.go @@ -354,7 +354,7 @@ func generateMoreCustomers(t *testing.T, keyspace string, numCustomers int64) { vtgateConn, closeConn := getVTGateConn() defer closeConn() log.Infof("Generating more test data with an additional %d customers", numCustomers) - res := execQueryWithDatabase(t, vtgateConn, keyspace, "select max(cid) from customer") + res := execVtgateQuery(t, vtgateConn, keyspace, "select max(cid) from customer") startingID, _ := res.Rows[0][0].ToInt64() insert := strings.Builder{} insert.WriteString("insert into customer(cid, name, typ) values ") @@ -366,5 +366,5 @@ func generateMoreCustomers(t *testing.T, keyspace string, numCustomers int64) { insert.WriteString(", ") } } - execQueryWithDatabase(t, vtgateConn, keyspace, insert.String()) + execVtgateQuery(t, vtgateConn, keyspace, insert.String()) } diff --git a/go/test/endtoend/vreplication/vdiff_online_ddl_test.go b/go/test/endtoend/vreplication/vdiff_online_ddl_test.go index 1650ea50e0f..92977111294 100644 --- a/go/test/endtoend/vreplication/vdiff_online_ddl_test.go +++ b/go/test/endtoend/vreplication/vdiff_online_ddl_test.go @@ -132,7 +132,7 @@ func waitForAdditionalRows(t *testing.T, keyspace, table string, count int) { } func getNumRows(t *testing.T, vtgateConn *mysql.Conn, keyspace, table string) int { - qr := execQueryWithDatabase(t, vtgateConn, keyspace, fmt.Sprintf("SELECT COUNT(*) FROM %s", table)) + qr := execVtgateQuery(t, vtgateConn, keyspace, fmt.Sprintf("SELECT COUNT(*) FROM %s", table)) require.NotNil(t, qr) numRows, err := strconv.Atoi(qr.Rows[0][0].ToString()) require.NoError(t, err) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 726f91cfbd0..1f298992d90 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -340,7 +340,7 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string // the Lead and Lead-1 tables tested a specific case with binary sharding keys. Drop it now so that we don't // have to update the rest of the tests - execQueryWithDatabase(t, vtgateConn, "customer", "drop table `Lead`,`Lead-1`") + execVtgateQuery(t, vtgateConn, "customer", "drop table `Lead`,`Lead-1`") validateRollupReplicates(t) shardOrders(t) shardMerchant(t) @@ -360,13 +360,14 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string insertMoreCustomers(t, 16) reshardCustomer2to4Split(t, nil, "") - confirmAllStreamsRunning(t, "customer", "-40") + confirmAllStreamsRunning(t, vtgateConn, "customer:-40") expectNumberOfStreams(t, vtgateConn, "Customer2to4", "sales", "product:0", 4) reshardCustomer3to2SplitMerge(t) - confirmAllStreamsRunning(t, "customer", "-60") + confirmAllStreamsRunning(t, vtgateConn, "customer:-60") expectNumberOfStreams(t, vtgateConn, "Customer3to2", "sales", "product:0", 3) reshardCustomer3to1Merge(t) - confirmAllStreamsRunning(t, "customer", "0") + confirmAllStreamsRunning(t, vtgateConn, "customer:0") + expectNumberOfStreams(t, vtgateConn, "Customer3to1", "sales", "product:0", 1) t.Run("Verify CopyState Is Optimized Afterwards", func(t *testing.T) { @@ -717,10 +718,10 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl defer vtgateConn.Close() // Confirm that the 0 scale decimal field, dec80, is replicated correctly dec80Replicated := false - execQueryWithDatabase(t, vtgateConn, sourceKs, "update customer set dec80 = 0") - execQueryWithDatabase(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") - execQueryWithDatabase(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") - execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") + execVtgateQuery(t, vtgateConn, sourceKs, "update customer set dec80 = 0") + execVtgateQuery(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") + execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") waitForNoWorkflowLag(t, vc, targetKs, workflow) for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { // Query the tablet's mysqld directly as the target may have denied table entries. @@ -737,8 +738,8 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // Insert multiple rows in the loadtest table and immediately delete them to confirm that bulk delete // works the same way with the vplayer optimization enabled and disabled. Currently this optimization // is disabled by default, but enabled in TestCellAliasVreplicationWorkflow. - execQueryWithDatabase(t, vtgateConn, sourceKs, "insert into loadtest(id, name) values(10001, 'tempCustomer'), (10002, 'tempCustomer2'), (10003, 'tempCustomer3'), (10004, 'tempCustomer4')") - execQueryWithDatabase(t, vtgateConn, sourceKs, "delete from loadtest where id > 10000") + execVtgateQuery(t, vtgateConn, sourceKs, "insert into loadtest(id, name) values(10001, 'tempCustomer'), (10002, 'tempCustomer2'), (10003, 'tempCustomer3'), (10004, 'tempCustomer4')") + execVtgateQuery(t, vtgateConn, sourceKs, "delete from loadtest where id > 10000") // Confirm that all partial query metrics get updated when we are testing the noblob mode. t.Run("validate partial query counts", func(t *testing.T) { @@ -782,7 +783,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl if err != nil { require.FailNow(t, output) } - execQueryWithDatabase(t, vtgateConn, "product", fmt.Sprintf("update `%s` set name='xyz'", tbl)) + execVtgateQuery(t, vtgateConn, "product", fmt.Sprintf("update `%s` set name='xyz'", tbl)) } } vdiffSideBySide(t, ksWorkflow, "") @@ -803,7 +804,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // The original unsharded customer data included an insert with the // vindex column (cid) of 999999, so the backing sequence table should // now have a next_id of 1000000 after SwitchTraffic. - res := execQueryWithDatabase(t, vtgateConn, sourceKs, "select next_id from customer_seq where id = 0") + res := execVtgateQuery(t, vtgateConn, sourceKs, "select next_id from customer_seq where id = 0") require.Equal(t, "1000000", res.Rows[0][0].ToString()) if withOpenTx && commit != nil { @@ -814,7 +815,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl vdiffSideBySide(t, "product.p2c_reverse", "") if withOpenTx { - execQueryWithDatabase(t, vtgateConn, "", deleteOpenTxQuery) + execVtgateQuery(t, vtgateConn, "", deleteOpenTxQuery) } ksShards := []string{"product/0", "customer/-80", "customer/80-"} @@ -829,7 +830,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl insertQuery2 = "insert into customer(name, cid) values('tempCustomer4', 102)" // ID 102, hence due to reverse_bits in shard -80 assertQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery2, matchInsertQuery2) - execQueryWithDatabase(t, vtgateConn, "customer", "update customer set meta = convert(x'7b7d' using utf8mb4) where cid = 1") + execVtgateQuery(t, vtgateConn, "customer", "update customer set meta = convert(x'7b7d' using utf8mb4) where cid = 1") if testReverse { // Reverse Replicate switchReads(t, workflowType, cellNames, ksWorkflow, true) @@ -888,13 +889,13 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl insertQuery2 = "insert into customer(name, cid) values('tempCustomer9', 105)" // ID 104, hence due to reverse_bits in shard 80- assertQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery2, matchInsertQuery2) - execQueryWithDatabase(t, vtgateConn, "customer", "delete from customer where name like 'tempCustomer%'") + execVtgateQuery(t, vtgateConn, "customer", "delete from customer where name like 'tempCustomer%'") waitForRowCountInTablet(t, customerTab1, "customer", "customer", 1) waitForRowCountInTablet(t, customerTab2, "customer", "customer", 2) waitForRowCount(t, vtgateConn, "customer", "customer.customer", 3) query = "insert into customer (name, cid) values('george', 5)" - execQueryWithDatabase(t, vtgateConn, "customer", query) + execVtgateQuery(t, vtgateConn, "customer", query) waitForRowCountInTablet(t, customerTab1, "customer", "customer", 1) waitForRowCountInTablet(t, customerTab2, "customer", "customer", 3) waitForRowCount(t, vtgateConn, "customer", "customer.customer", 4) @@ -923,7 +924,7 @@ func reshardCustomer2to4Split(t *testing.T, cells []*Cell, sourceCellOrAlias str 600, counts, nil, nil, cells, sourceCellOrAlias, 1) waitForRowCount(t, vtgateConn, ksName, "customer", 20) query := "insert into customer (name) values('yoko')" - execQueryWithDatabase(t, vtgateConn, ksName, query) + execVtgateQuery(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "customer", 21) }) } @@ -938,7 +939,7 @@ func reshardMerchant2to3SplitMerge(t *testing.T) { 1600, counts, dryRunResultsSwitchReadM2m3, dryRunResultsSwitchWritesM2m3, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 2) query := "insert into merchant (mname, category) values('amazon', 'electronics')" - execQueryWithDatabase(t, vtgateConn, ksName, query) + execVtgateQuery(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "merchant", 3) var output string @@ -987,7 +988,7 @@ func reshardMerchant3to1Merge(t *testing.T) { 2000, counts, nil, nil, nil, "", 1) waitForRowCount(t, vtgateConn, ksName, "merchant", 3) query := "insert into merchant (mname, category) values('flipkart', 'electronics')" - execQueryWithDatabase(t, vtgateConn, ksName, query) + execVtgateQuery(t, vtgateConn, ksName, query) waitForRowCount(t, vtgateConn, ksName, "merchant", 4) }) } diff --git a/go/test/endtoend/vreplication/vstream_test.go b/go/test/endtoend/vreplication/vstream_test.go index b92dbeb9937..e13c3e24e80 100644 --- a/go/test/endtoend/vreplication/vstream_test.go +++ b/go/test/endtoend/vreplication/vstream_test.go @@ -95,7 +95,7 @@ func testVStreamWithFailover(t *testing.T, failover bool) { } insertMu.Lock() id++ - execQueryWithDatabase(t, vtgateConn, "product", fmt.Sprintf("insert into customer (cid, name) values (%d, 'customer%d')", id+100, id)) + execVtgateQuery(t, vtgateConn, "product", fmt.Sprintf("insert into customer (cid, name) values (%d, 'customer%d')", id+100, id)) insertMu.Unlock() } }() @@ -164,7 +164,7 @@ func testVStreamWithFailover(t *testing.T, failover bool) { } } - qr := execQueryWithDatabase(t, vtgateConn, "product", "select count(*) from customer") + qr := execVtgateQuery(t, vtgateConn, "product", "select count(*) from customer") require.NotNil(t, qr) // total number of row events found by the VStream API should match the rows inserted insertedRows, err := qr.Rows[0][0].ToCastInt64() @@ -507,7 +507,7 @@ func testVStreamCopyMultiKeyspaceReshard(t *testing.T, baseTabletID int) numEven // because the keyspace remains unsharded and the number of rows in the customer_seq table is always 1. // We believe that checking the number of row events for the unsharded keyspace, which should always be greater than 0 before and after resharding, // is sufficient to confirm that the resharding of one keyspace does not affect another keyspace, while keeping the test straightforward. - customerResult := execQueryWithDatabase(t, vtgateConn, "sharded", "select count(*) from customer") + customerResult := execVtgateQuery(t, vtgateConn, "sharded", "select count(*) from customer") insertedCustomerRows, err := customerResult.Rows[0][0].ToCastInt64() require.NoError(t, err) require.Equal(t, insertedCustomerRows, ne.numDash80Events+ne.num80DashEvents+ne.numDash40Events+ne.num40DashEvents) @@ -698,7 +698,7 @@ func TestMultiVStreamsKeyspaceReshard(t *testing.T) { require.NotZero(t, newShardRowEvents) // The number of row events streamed by the VStream API should match the number of rows inserted. - customerResult := execQueryWithDatabase(t, vtgateConn, ks, "select count(*) from customer") + customerResult := execVtgateQuery(t, vtgateConn, ks, "select count(*) from customer") customerCount, err := customerResult.Rows[0][0].ToInt64() require.NoError(t, err) require.Equal(t, customerCount, int64(oldShardRowEvents+newShardRowEvents)) From 904ba09a5583ebc1f54d8bc444d2f4e218e97d83 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 10 Jun 2024 11:26:50 -0400 Subject: [PATCH 34/34] Correct comment Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/vreplication_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 1f298992d90..db58f2880c2 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -724,7 +724,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") waitForNoWorkflowLag(t, vc, targetKs, workflow) for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { - // Query the tablet's mysqld directly as the target may have denied table entries. + // Query the tablet's mysqld directly as the targets will have denied table entries. dbc, err := tablet.TabletConn(targetKs, true) require.NoError(t, err) defer dbc.Close()