Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add tablet-tags/--init_tags stats #16695

Merged
merged 8 commits into from
Sep 25, 2024
Merged

Add tablet-tags/--init_tags stats #16695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d298211
Add tablet-tags/`--init_tags` stats
timvaillancourt Sep 2, 2024
6e48b82
add test
timvaillancourt Sep 24, 2024
94cea7c
cleanup
timvaillancourt Sep 24, 2024
900df5f
Merge remote-tracking branch 'origin/main' into vttablet-tablet-tags-…
timvaillancourt Sep 25, 2024
5e7dcf1
fix new file
timvaillancourt Sep 25, 2024
25de1bd
fix newline
timvaillancourt Sep 25, 2024
910b1ad
reset stats
timvaillancourt Sep 25, 2024
291aeb8
move stats reset
timvaillancourt Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions go/vt/vttablet/tabletmanager/rpc_transaction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func TestTabletManager_UnresolvedTransactions(t *testing.T) {
BatchCtx: ctx,
}
close(tm._waitForGrantsComplete)
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80"))
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80", nil))

_, err := tm.GetUnresolvedTransactions(ctx, 0)
require.NoError(t, err)
Expand All @@ -56,7 +56,7 @@ func TestTabletManager_ReadTransaction(t *testing.T) {
BatchCtx: ctx,
}
close(tm._waitForGrantsComplete)
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80"))
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80", nil))

_, err := tm.ReadTransaction(ctx, &tabletmanagerdatapb.ReadTransactionRequest{
Dtid: "dtid01",
Expand All @@ -76,7 +76,7 @@ func TestTabletManager_ConcludeTransaction(t *testing.T) {
BatchCtx: ctx,
}
close(tm._waitForGrantsComplete)
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80"))
tm.tmState = newTMState(tm, newTestTablet(t, 100, "ks", "-80", nil))

err := tm.ConcludeTransaction(ctx, &tabletmanagerdatapb.ConcludeTransactionRequest{
Dtid: "dtid01",
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vttablet/tabletmanager/shard_sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func TestShardSync(t *testing.T) {
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
statsTabletTypeCount.ResetAll()
tm := newTestTM(t, ts, 100, keyspace, shard)
tm := newTestTM(t, ts, 100, keyspace, shard, nil)
defer tm.Stop()

// update the primary info in the shard record and set it to nil
Expand Down
7 changes: 7 additions & 0 deletions go/vt/vttablet/tabletmanager/tm_init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ var (
// statsIsInSrvKeyspace is set to 1 (true), 0 (false) whether the tablet is in the serving keyspace
statsIsInSrvKeyspace *stats.Gauge

// statsTabletTags is set to 1 (true) if a tablet tag exists.
statsTabletTags *stats.GaugesWithMultiLabels

statsKeyspace = stats.NewString("TabletKeyspace")
statsShard = stats.NewString("TabletShard")
statsKeyRangeStart = stats.NewString("TabletKeyRangeStart")
Expand All @@ -142,6 +145,7 @@ func init() {
statsTabletTypeCount = stats.NewCountersWithSingleLabel("TabletTypeCount", "Number of times the tablet changed to the labeled type", "type")
statsBackupIsRunning = stats.NewGaugesWithMultiLabels("BackupIsRunning", "Whether a backup is running", []string{"mode"})
statsIsInSrvKeyspace = stats.NewGauge("IsInSrvKeyspace", "Whether the vttablet is in the serving keyspace (1 = true / 0 = false)")
statsTabletTags = stats.NewGaugesWithMultiLabels("TabletTags", "Tablet tags key/values", []string{"key", "value"})
}

// TabletManager is the main class for the tablet manager.
Expand Down Expand Up @@ -889,6 +893,9 @@ func (tm *TabletManager) exportStats() {
statsKeyRangeEnd.Set(hex.EncodeToString(tablet.KeyRange.End))
}
statsAlias.Set(topoproto.TabletAliasString(tablet.Alias))
for k, v := range tablet.Tags {
statsTabletTags.Set([]string{k, v}, 1)
}
}

// withRetry will exponentially back off and retry a function upon
Expand Down
58 changes: 42 additions & 16 deletions go/vt/vttablet/tabletmanager/tm_init_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func TestStartCreateKeyspaceShard(t *testing.T) {
statsTabletTypeCount.ResetAll()
cell := "cell1"
ts := memorytopo.NewServer(ctx, cell)
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

assert.Equal(t, "replica", statsTabletType.Get())
Expand All @@ -193,7 +193,7 @@ func TestStartCreateKeyspaceShard(t *testing.T) {
// keyspace-shard already created.
_, err = ts.GetOrCreateShard(ctx, "ks1", "0")
require.NoError(t, err)
tm = newTestTM(t, ts, 2, "ks1", "0")
tm = newTestTM(t, ts, 2, "ks1", "0", nil)
defer tm.Stop()
_, err = ts.GetShard(ctx, "ks1", "0")
require.NoError(t, err)
Expand All @@ -207,7 +207,7 @@ func TestStartCreateKeyspaceShard(t *testing.T) {
require.NoError(t, err)
err = topotools.RebuildKeyspace(ctx, logutil.NewConsoleLogger(), ts, "ks2", []string{cell}, false)
require.NoError(t, err)
tm = newTestTM(t, ts, 3, "ks2", "0")
tm = newTestTM(t, ts, 3, "ks2", "0", nil)
defer tm.Stop()
_, err = ts.GetShard(ctx, "ks2", "0")
require.NoError(t, err)
Expand All @@ -224,7 +224,7 @@ func TestStartCreateKeyspaceShard(t *testing.T) {
require.NoError(t, err)
err = ts.RebuildSrvVSchema(ctx, []string{cell})
require.NoError(t, err)
tm = newTestTM(t, ts, 4, "ks3", "0")
tm = newTestTM(t, ts, 4, "ks3", "0", nil)
defer tm.Stop()
_, err = ts.GetShard(ctx, "ks3", "0")
require.NoError(t, err)
Expand All @@ -235,15 +235,15 @@ func TestStartCreateKeyspaceShard(t *testing.T) {
assert.Equal(t, wantVSchema, srvVSchema.Keyspaces["ks3"])

// Multi-shard
tm1 := newTestTM(t, ts, 5, "ks4", "-80")
tm1 := newTestTM(t, ts, 5, "ks4", "-80", nil)
defer tm1.Stop()

// Wait a bit and make sure that srvKeyspace is still not created.
time.Sleep(100 * time.Millisecond)
_, err = ts.GetSrvKeyspace(context.Background(), cell, "ks4")
require.True(t, topo.IsErrType(err, topo.NoNode), err)

tm2 := newTestTM(t, ts, 6, "ks4", "80-")
tm2 := newTestTM(t, ts, 6, "ks4", "80-", nil)
defer tm2.Stop()
// Now that we've started the tablet for the other shard, srvKeyspace will succeed.
ensureSrvKeyspace(t, ctx, ts, cell, "ks4")
Expand All @@ -264,7 +264,7 @@ func TestCheckPrimaryShip(t *testing.T) {

// 1. Initialize the tablet as REPLICA.
// This will create the respective topology records.
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
tablet := tm.Tablet()
ensureSrvKeyspace(t, ctx, ts, cell, "ks")
ti, err := ts.GetTablet(ctx, alias)
Expand Down Expand Up @@ -398,7 +398,7 @@ func TestStartCheckMysql(t *testing.T) {
defer cancel()
cell := "cell1"
ts := memorytopo.NewServer(ctx, cell)
tablet := newTestTablet(t, 1, "ks", "0")
tablet := newTestTablet(t, 1, "ks", "0", nil)
cp := mysql.ConnParams{
Host: "foo",
Port: 1,
Expand Down Expand Up @@ -429,7 +429,7 @@ func TestStartFindMysqlPort(t *testing.T) {
defer cancel()
cell := "cell1"
ts := memorytopo.NewServer(ctx, cell)
tablet := newTestTablet(t, 1, "ks", "0")
tablet := newTestTablet(t, 1, "ks", "0", nil)
fmd := newTestMysqlDaemon(t, -1)
tm := &TabletManager{
BatchCtx: context.Background(),
Expand Down Expand Up @@ -472,7 +472,7 @@ func TestStartFixesReplicationData(t *testing.T) {
defer cancel()
cell := "cell1"
ts := memorytopo.NewServer(ctx, cell, "cell2")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()
tabletAlias := tm.tabletAlias

Expand Down Expand Up @@ -505,14 +505,14 @@ func TestStartDoesNotUpdateReplicationDataForTabletInWrongShard(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1", "cell2")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
tm.Stop()

tabletAliases, err := ts.FindAllTabletAliasesInShard(ctx, "ks", "0")
require.NoError(t, err)
assert.Equal(t, uint32(1), tabletAliases[0].Uid)

tablet := newTestTablet(t, 1, "ks", "-d0")
tablet := newTestTablet(t, 1, "ks", "-d0", nil)
require.NoError(t, err)
err = tm.Start(tablet, nil)
assert.Contains(t, err.Error(), "existing tablet keyspace and shard ks/0 differ")
Expand All @@ -537,7 +537,7 @@ func TestCheckTabletTypeResets(t *testing.T) {

// 1. Initialize the tablet as REPLICA.
// This will create the respective topology records.
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
tablet := tm.Tablet()
ensureSrvKeyspace(t, ctx, ts, cell, "ks")
ti, err := ts.GetTablet(ctx, alias)
Expand Down Expand Up @@ -648,6 +648,27 @@ func TestGetBuildTags(t *testing.T) {
}
}

func TestStartExportStats(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

ts := memorytopo.NewServer(ctx, "cell1")
_ = newTestTM(t, ts, 1, "ks", "0", map[string]string{
"test": t.Name(),
})

assert.Equal(t, "ks", statsKeyspace.Get())
assert.Equal(t, "0", statsShard.Get())
assert.Equal(t, "replica", statsTabletType.Get())
assert.Equal(t, map[string]int64{
"replica": 1,
}, statsTabletTypeCount.Counts())
assert.Equal(t, "cell1-0000000001", statsAlias.Get())
assert.Equal(t, map[string]int64{
"test." + t.Name(): 1,
}, statsTabletTags.Counts())
}

func newTestMysqlDaemon(t *testing.T, port int32) *mysqlctl.FakeMysqlDaemon {
t.Helper()

Expand All @@ -662,10 +683,14 @@ func newTestMysqlDaemon(t *testing.T, port int32) *mysqlctl.FakeMysqlDaemon {
return mysqld
}

func newTestTM(t *testing.T, ts *topo.Server, uid int, keyspace, shard string) *TabletManager {
func newTestTM(t *testing.T, ts *topo.Server, uid int, keyspace, shard string, tags map[string]string) *TabletManager {
// reset stats
statsTabletTags.ResetAll()
statsTabletTypeCount.ResetAll()

t.Helper()
ctx := context.Background()
tablet := newTestTablet(t, uid, keyspace, shard)
tablet := newTestTablet(t, uid, keyspace, shard, tags)
tm := &TabletManager{
BatchCtx: ctx,
TopoServer: ts,
Expand Down Expand Up @@ -701,7 +726,7 @@ func newTestTM(t *testing.T, ts *topo.Server, uid int, keyspace, shard string) *
}
}

func newTestTablet(t *testing.T, uid int, keyspace, shard string) *topodatapb.Tablet {
func newTestTablet(t *testing.T, uid int, keyspace, shard string, tags map[string]string) *topodatapb.Tablet {
shard, keyRange, err := topo.ValidateShardName(shard)
require.NoError(t, err)
return &topodatapb.Tablet{
Expand All @@ -718,6 +743,7 @@ func newTestTablet(t *testing.T, uid int, keyspace, shard string) *topodatapb.Ta
Shard: shard,
KeyRange: keyRange,
Type: topodatapb.TabletType_REPLICA,
Tags: tags,
}
}

Expand Down
24 changes: 12 additions & 12 deletions go/vt/vttablet/tabletmanager/tm_state_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func TestStateOpenClose(t *testing.T) {
defer cancel()

ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)

// Re-Open should be a no-op
tm.tmState.mu.Lock()
Expand All @@ -69,7 +69,7 @@ func TestStateRefreshFromTopo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

err := tm.RefreshState(ctx)
Expand All @@ -80,7 +80,7 @@ func TestStateResharding(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

tm.tmState.mu.Lock()
Expand Down Expand Up @@ -108,7 +108,7 @@ func TestStateDenyList(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

fmd := tm.MysqlDaemon.(*mysqlctl.FakeMysqlDaemon)
Expand Down Expand Up @@ -140,7 +140,7 @@ func TestStateTabletControls(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

ks := &topodatapb.SrvKeyspace{
Expand Down Expand Up @@ -169,7 +169,7 @@ func TestStateIsShardServingisInSrvKeyspace(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

tm.tmState.mu.Lock()
Expand Down Expand Up @@ -341,7 +341,7 @@ func TestStateNonServing(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 1, "ks", "0")
tm := newTestTM(t, ts, 1, "ks", "0", nil)
defer tm.Stop()

tm.tmState.mu.Lock()
Expand All @@ -359,7 +359,7 @@ func TestStateChangeTabletType(t *testing.T) {
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
statsTabletTypeCount.ResetAll()
tm := newTestTM(t, ts, 2, "ks", "0")
tm := newTestTM(t, ts, 2, "ks", "0", nil)
defer tm.Stop()

assert.Equal(t, 1, len(statsTabletTypeCount.Counts()))
Expand Down Expand Up @@ -402,7 +402,7 @@ func TestStateChangeTabletTypeWithFailure(t *testing.T) {
ts := memorytopo.NewServer(ctx, "cell1")
statsTabletTypeCount.ResetAll()
// create TM with replica and put a hook to return error during SetServingType
tm := newTestTM(t, ts, 2, "ks", "0")
tm := newTestTM(t, ts, 2, "ks", "0", nil)
qsc := tm.QueryServiceControl.(*tabletservermock.Controller)
qsc.SetServingTypeError = vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED, "mocking resource exhaustion error ")
defer tm.Stop()
Expand Down Expand Up @@ -485,7 +485,7 @@ func TestChangeTypeErrorWhileWritingToTopo(t *testing.T) {
fakeConn := factory.AddCell("cell1")
ts := faketopo.NewFakeTopoServer(context.TODO(), factory)
statsTabletTypeCount.ResetAll()
tm := newTestTM(t, ts, 2, "ks", "0")
tm := newTestTM(t, ts, 2, "ks", "0", nil)
defer tm.Stop()

// ChangeTabletType calls topotools.ChangeType which in-turn issues
Expand Down Expand Up @@ -533,7 +533,7 @@ func TestPublishStateNew(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 42, "ks", "0")
tm := newTestTM(t, ts, 42, "ks", "0", nil)
ttablet, err := tm.TopoServer.GetTablet(ctx, tm.tabletAlias)
require.NoError(t, err)
utils.MustMatch(t, tm.Tablet(), ttablet.Tablet)
Expand Down Expand Up @@ -580,7 +580,7 @@ func TestPublishDeleted(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := memorytopo.NewServer(ctx, "cell1")
tm := newTestTM(t, ts, 2, "ks", "0")
tm := newTestTM(t, ts, 2, "ks", "0", nil)
defer tm.Stop()

alias := &topodatapb.TabletAlias{
Expand Down
Loading