Tablet throttler: adding more stats (#15224)

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

go/vt/vttablet/tabletmanager/rpc_throttler.go

@@ -19,6 +19,7 @@ package tabletmanager
 import (
  "context"
 
+ "vitess.io/vitess/go/stats"
  tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
  "vitess.io/vitess/go/vt/proto/vtrpc"
  "vitess.io/vitess/go/vt/vterrors"
@@ -28,6 +29,7 @@ import (
 
 // CheckThrottler executes a throttler check
 func (tm *TabletManager) CheckThrottler(ctx context.Context, req *tabletmanagerdatapb.CheckThrottlerRequest) (*tabletmanagerdatapb.CheckThrottlerResponse, error) {
+ go stats.GetOrNewCounter("ThrottlerCheckRequest", "CheckThrottler requests").Add(1)
  if req.AppName == "" {
  req.AppName = throttlerapp.VitessName.String()
  }

go/vt/vttablet/tabletserver/throttle/throttler.go

@@ -59,6 +59,7 @@ import (
 
  "vitess.io/vitess/go/constants/sidecar"
  "vitess.io/vitess/go/protoutil"
+ "vitess.io/vitess/go/stats"
 
  "vitess.io/vitess/go/textutil"
  "vitess.io/vitess/go/timer"
@@ -443,7 +444,7 @@ func (throttler *Throttler) Enable() *sync.WaitGroup {
  throttler.Operate(ctx, wg)
 
  // Make a one-time request for a lease of heartbeats
- go throttler.heartbeatWriter.RequestHeartbeats()
+ throttler.requestHeartbeats()
 
  return wg
 }
@@ -568,6 +569,13 @@ func (throttler *Throttler) Close() {
  log.Infof("Throttler: finished execution of Close")
 }
 
+// requestHeartbeats sends a heartbeat lease request to the heartbeat writer.
+// This action is recorded in stats.
+func (throttler *Throttler) requestHeartbeats() {
+ go throttler.heartbeatWriter.RequestHeartbeats()
+ go stats.GetOrNewCounter("ThrottlerHeartbeatRequests", "heartbeat requests").Add(1)
+}
+
 func (throttler *Throttler) generateSelfMySQLThrottleMetricFunc(ctx context.Context, probe *mysql.Probe) func() *mysql.MySQLThrottleMetric {
  f := func() *mysql.MySQLThrottleMetric {
  return throttler.readSelfThrottleMetric(ctx, probe)
@@ -708,7 +716,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) {
  if transitionedIntoLeader {
  // transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks
  go mysqlRefreshTicker.TickNow()
- go throttler.heartbeatWriter.RequestHeartbeats()
+ throttler.requestHeartbeats()
  }
  }()
  case <-mysqlCollectTicker.C:
@@ -782,7 +790,8 @@ func (throttler *Throttler) generateTabletProbeFunction(ctx context.Context, clu
  if resp.RecentlyChecked {
  // We have just probed a tablet, and it reported back that someone just recently "check"ed it.
  // We therefore renew the heartbeats lease.
- go throttler.heartbeatWriter.RequestHeartbeats()
+ throttler.requestHeartbeats()
+ go stats.GetOrNewCounter("ThrottlerProbeRecentlyChecked", "probe recently checked").Add(1)
  }
  return mySQLThrottleMetric
  }
@@ -1017,7 +1026,7 @@ func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, rati
 func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.AppThrottle) {
  throttler.throttledApps.Delete(appName)
  // the app is likely to check
- go throttler.heartbeatWriter.RequestHeartbeats()
+ throttler.requestHeartbeats()
  return base.NewAppThrottle(appName, time.Now(), 0, false)
 }
 
@@ -1164,7 +1173,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
  return okMetricCheckResult
  }
  if !flags.SkipRequestHeartbeats && !throttlerapp.VitessName.Equals(appName) {
- go throttler.heartbeatWriter.RequestHeartbeats()
+ throttler.requestHeartbeats()
  // This check was made by someone other than the throttler itself, i.e. this came from online-ddl or vreplication or other.
  // We mark the fact that someone just made a check. If this is a REPLICA or RDONLY tables, this will be reported back
  // to the PRIMARY so that it knows it must renew the heartbeat lease.
@@ -1178,6 +1187,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
  // If this tablet is a REPLICA or RDONLY, we want to advertise to the PRIMARY that someone did a recent check,
  // so that the PRIMARY knows it must renew the heartbeat lease.
  checkResult.RecentlyChecked = true
+ go stats.GetOrNewCounter("ThrottlerRecentlyChecked", "recently checked").Add(1)
  }
 
  return checkResult