From 365f936ace72df84619dcda2a7348720dab45810 Mon Sep 17 00:00:00 2001
From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
Date: Thu, 15 Feb 2024 14:50:52 +0200
Subject: [PATCH] Tablet throttler: adding more stats (#15224)

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
---
 go/vt/vttablet/tabletmanager/rpc_throttler.go |  2 ++
 .../tabletserver/throttle/throttler.go        | 20 ++++++++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/go/vt/vttablet/tabletmanager/rpc_throttler.go b/go/vt/vttablet/tabletmanager/rpc_throttler.go
index dfdc0d230fb..c961761c5f2 100644
--- a/go/vt/vttablet/tabletmanager/rpc_throttler.go
+++ b/go/vt/vttablet/tabletmanager/rpc_throttler.go
@@ -19,6 +19,7 @@ package tabletmanager
 import (
 	"context"
 
+	"vitess.io/vitess/go/stats"
 	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
 	"vitess.io/vitess/go/vt/proto/vtrpc"
 	"vitess.io/vitess/go/vt/vterrors"
@@ -28,6 +29,7 @@ import (
 
 // CheckThrottler executes a throttler check
 func (tm *TabletManager) CheckThrottler(ctx context.Context, req *tabletmanagerdatapb.CheckThrottlerRequest) (*tabletmanagerdatapb.CheckThrottlerResponse, error) {
+	go stats.GetOrNewCounter("ThrottlerCheckRequest", "CheckThrottler requests").Add(1)
 	if req.AppName == "" {
 		req.AppName = throttlerapp.VitessName.String()
 	}
diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go
index a451850a23f..b75898a3256 100644
--- a/go/vt/vttablet/tabletserver/throttle/throttler.go
+++ b/go/vt/vttablet/tabletserver/throttle/throttler.go
@@ -59,6 +59,7 @@ import (
 
 	"vitess.io/vitess/go/constants/sidecar"
 	"vitess.io/vitess/go/protoutil"
+	"vitess.io/vitess/go/stats"
 
 	"vitess.io/vitess/go/textutil"
 	"vitess.io/vitess/go/timer"
@@ -443,7 +444,7 @@ func (throttler *Throttler) Enable() *sync.WaitGroup {
 	throttler.Operate(ctx, wg)
 
 	// Make a one-time request for a lease of heartbeats
-	go throttler.heartbeatWriter.RequestHeartbeats()
+	throttler.requestHeartbeats()
 
 	return wg
 }
@@ -568,6 +569,13 @@ func (throttler *Throttler) Close() {
 	log.Infof("Throttler: finished execution of Close")
 }
 
+// requestHeartbeats sends a heartbeat lease request to the heartbeat writer.
+// This action is recorded in stats.
+func (throttler *Throttler) requestHeartbeats() {
+	go throttler.heartbeatWriter.RequestHeartbeats()
+	go stats.GetOrNewCounter("ThrottlerHeartbeatRequests", "heartbeat requests").Add(1)
+}
+
 func (throttler *Throttler) generateSelfMySQLThrottleMetricFunc(ctx context.Context, probe *mysql.Probe) func() *mysql.MySQLThrottleMetric {
 	f := func() *mysql.MySQLThrottleMetric {
 		return throttler.readSelfThrottleMetric(ctx, probe)
@@ -708,7 +716,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) {
 					if transitionedIntoLeader {
 						// transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks
 						go mysqlRefreshTicker.TickNow()
-						go throttler.heartbeatWriter.RequestHeartbeats()
+						throttler.requestHeartbeats()
 					}
 				}()
 			case <-mysqlCollectTicker.C:
@@ -782,7 +790,8 @@ func (throttler *Throttler) generateTabletProbeFunction(ctx context.Context, clu
 		if resp.RecentlyChecked {
 			// We have just probed a tablet, and it reported back that someone just recently "check"ed it.
 			// We therefore renew the heartbeats lease.
-			go throttler.heartbeatWriter.RequestHeartbeats()
+			throttler.requestHeartbeats()
+			go stats.GetOrNewCounter("ThrottlerProbeRecentlyChecked", "probe recently checked").Add(1)
 		}
 		return mySQLThrottleMetric
 	}
@@ -1017,7 +1026,7 @@ func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, rati
 func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.AppThrottle) {
 	throttler.throttledApps.Delete(appName)
 	// the app is likely to check
-	go throttler.heartbeatWriter.RequestHeartbeats()
+	throttler.requestHeartbeats()
 	return base.NewAppThrottle(appName, time.Now(), 0, false)
 }
 
@@ -1164,7 +1173,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
 		return okMetricCheckResult
 	}
 	if !flags.SkipRequestHeartbeats && !throttlerapp.VitessName.Equals(appName) {
-		go throttler.heartbeatWriter.RequestHeartbeats()
+		throttler.requestHeartbeats()
 		// This check was made by someone other than the throttler itself, i.e. this came from online-ddl or vreplication or other.
 		// We mark the fact that someone just made a check. If this is a REPLICA or RDONLY tables, this will be reported back
 		// to the PRIMARY so that it knows it must renew the heartbeat lease.
@@ -1178,6 +1187,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
 		// If this tablet is a REPLICA or RDONLY, we want to advertise to the PRIMARY that someone did a recent check,
 		// so that the PRIMARY knows it must renew the heartbeat lease.
 		checkResult.RecentlyChecked = true
+		go stats.GetOrNewCounter("ThrottlerRecentlyChecked", "recently checked").Add(1)
 	}
 
 	return checkResult