diff --git a/go/cmd/vtctldclient/command/throttler.go b/go/cmd/vtctldclient/command/throttler.go index 84943cc9962..2276ba37d62 100644 --- a/go/cmd/vtctldclient/command/throttler.go +++ b/go/cmd/vtctldclient/command/throttler.go @@ -86,6 +86,7 @@ func init() { UpdateThrottlerConfig.Flags().StringVar(&throttledAppRule.Name, "throttle-app", "", "an app name to throttle") UpdateThrottlerConfig.Flags().Float64Var(&throttledAppRule.Ratio, "throttle-app-ratio", throttle.DefaultThrottleRatio, "ratio to throttle app (app specififed in --throttled-app)") UpdateThrottlerConfig.Flags().DurationVar(&throttledAppDuration, "throttle-app-duration", throttle.DefaultAppThrottleDuration, "duration after which throttled app rule expires (app specififed in --throttled-app)") + UpdateThrottlerConfig.Flags().BoolVar(&throttledAppRule.Exempt, "throttle-app-exempt", throttledAppRule.Exempt, "exempt this app from being at all throttled. WARNING: use with extreme care, as this is likely to push metrics beyond the throttler's threshold, and starve other apps") Root.AddCommand(UpdateThrottlerConfig) } diff --git a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go index 79cb24d22c3..ab2775dc89d 100644 --- a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go +++ b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go @@ -249,7 +249,7 @@ func TestSchemaChange(t *testing.T) { } }) t.Run("updating throttler config", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, customThreshold, noCustomQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, customThreshold, noCustomQuery, nil) require.NoError(t, err) }) diff --git a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go index 654870fae97..0df79c55d46 100644 --- a/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go @@ -28,10 +28,11 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/throttler" + "vitess.io/vitess/go/vt/logutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,6 +46,7 @@ const ( onDemandHeartbeatDuration = 5 * time.Second throttlerEnabledTimeout = 60 * time.Second useDefaultQuery = "" + testAppName = "test" ) var ( @@ -170,12 +172,12 @@ func throttledApps(tablet *cluster.Vttablet) (resp *http.Response, respBody stri } func throttleCheck(tablet *cluster.Vttablet, skipRequestHeartbeats bool) (*http.Response, error) { - resp, err := httpClient.Get(fmt.Sprintf("http://localhost:%d/%s?app=test&s=%t", tablet.HTTPPort, checkAPIPath, skipRequestHeartbeats)) + resp, err := httpClient.Get(fmt.Sprintf("http://localhost:%d/%s?app=%s&s=%t", tablet.HTTPPort, checkAPIPath, testAppName, skipRequestHeartbeats)) return resp, err } func throttleCheckSelf(tablet *cluster.Vttablet) (*http.Response, error) { - return httpClient.Get(fmt.Sprintf("http://localhost:%d/%s?app=test", tablet.HTTPPort, checkSelfAPIPath)) + return httpClient.Get(fmt.Sprintf("http://localhost:%d/%s?app=%s", tablet.HTTPPort, checkSelfAPIPath, testAppName)) } func warmUpHeartbeat(t *testing.T) (respStatus int) { @@ -245,7 +247,7 @@ func TestInitialThrottler(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, http.StatusOK) }) t.Run("enabling throttler with very low threshold", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, unreasonablyLowThreshold.Seconds(), useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, unreasonablyLowThreshold.Seconds(), useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be enabled everywhere with the new config. @@ -257,7 +259,7 @@ func TestInitialThrottler(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, http.StatusTooManyRequests) }) t.Run("disabling throttler", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, true, unreasonablyLowThreshold.Seconds(), useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, true, unreasonablyLowThreshold.Seconds(), useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be disabled everywhere. @@ -271,7 +273,7 @@ func TestInitialThrottler(t *testing.T) { t.Run("enabling throttler, again", func(t *testing.T) { // Enable the throttler again with the default query which also moves us back // to the default threshold. - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, 0, useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, 0, useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be enabled everywhere again with the default config. @@ -283,7 +285,7 @@ func TestInitialThrottler(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, http.StatusTooManyRequests) }) t.Run("setting high threshold", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, extremelyHighThreshold.Seconds(), useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, extremelyHighThreshold.Seconds(), useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be enabled everywhere with new config. @@ -295,7 +297,7 @@ func TestInitialThrottler(t *testing.T) { waitForThrottleCheckStatus(t, primaryTablet, http.StatusOK) }) t.Run("setting low threshold", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be enabled everywhere with new config. @@ -392,6 +394,26 @@ func TestLag(t *testing.T) { defer resp.Body.Close() assert.Equalf(t, http.StatusTooManyRequests, resp.StatusCode, "Unexpected response from throttler: %s", getResponseBody(resp)) }) + t.Run("exempting test app", func(t *testing.T) { + appRule := &topodatapb.ThrottledAppRule{ + Name: testAppName, + ExpiresAt: logutil.TimeToProto(time.Now().Add(time.Hour)), + Exempt: true, + } + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery, appRule) + assert.NoError(t, err) + waitForThrottleCheckStatus(t, primaryTablet, http.StatusOK) + }) + t.Run("unexempting test app", func(t *testing.T) { + appRule := &topodatapb.ThrottledAppRule{ + Name: testAppName, + ExpiresAt: logutil.TimeToProto(time.Now()), + } + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, false, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery, appRule) + assert.NoError(t, err) + waitForThrottleCheckStatus(t, primaryTablet, http.StatusTooManyRequests) + }) + t.Run("starting replication", func(t *testing.T) { err := clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", replicaTablet.Alias) assert.NoError(t, err) @@ -436,7 +458,7 @@ func TestCustomQuery(t *testing.T) { defer cluster.PanicHandler(t) t.Run("enabling throttler with custom query and threshold", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, customThreshold, customQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, customThreshold, customQuery, nil) assert.NoError(t, err) // Wait for the throttler to be enabled everywhere with new custom config. @@ -504,7 +526,7 @@ func TestRestoreDefaultQuery(t *testing.T) { // Validate going back from custom-query to default-query (replication lag) still works. t.Run("enabling throttler with default query and threshold", func(t *testing.T) { - _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery) + _, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, true, false, throttler.DefaultThreshold.Seconds(), useDefaultQuery, nil) assert.NoError(t, err) // Wait for the throttler to be up and running everywhere again with the default config. diff --git a/go/test/endtoend/throttler/util.go b/go/test/endtoend/throttler/util.go index 45d6b094f9c..f938090aafe 100644 --- a/go/test/endtoend/throttler/util.go +++ b/go/test/endtoend/throttler/util.go @@ -33,6 +33,8 @@ import ( "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" ) @@ -57,7 +59,7 @@ var DefaultConfig = &Config{ // This retries the command until it succeeds or times out as the // SrvKeyspace record may not yet exist for a newly created // Keyspace that is still initializing before it becomes serving. -func UpdateThrottlerTopoConfigRaw(vtctldProcess *cluster.VtctldClientProcess, keyspaceName string, enable bool, disable bool, threshold float64, metricsQuery string) (result string, err error) { +func UpdateThrottlerTopoConfigRaw(vtctldProcess *cluster.VtctldClientProcess, keyspaceName string, enable bool, disable bool, threshold float64, metricsQuery string, appRule *topodatapb.ThrottledAppRule) (result string, err error) { args := []string{} args = append(args, "UpdateThrottlerConfig") if enable { @@ -75,6 +77,14 @@ func UpdateThrottlerTopoConfigRaw(vtctldProcess *cluster.VtctldClientProcess, ke } else { args = append(args, "--check-as-check-shard") } + if appRule != nil { + args = append(args, "--throttle-app", appRule.Name) + args = append(args, "--throttle-app-duration", logutil.ProtoToTime(appRule.ExpiresAt).Sub(time.Now()).String()) + args = append(args, "--throttle-app-ratio", fmt.Sprintf("%f", appRule.Ratio)) + if appRule.Exempt { + args = append(args, "--throttle-app-exempt") + } + } args = append(args, keyspaceName) ctx, cancel := context.WithTimeout(context.Background(), ConfigTimeout) @@ -100,14 +110,14 @@ func UpdateThrottlerTopoConfigRaw(vtctldProcess *cluster.VtctldClientProcess, ke // This retries the command until it succeeds or times out as the // SrvKeyspace record may not yet exist for a newly created // Keyspace that is still initializing before it becomes serving. -func UpdateThrottlerTopoConfig(clusterInstance *cluster.LocalProcessCluster, enable bool, disable bool, threshold float64, metricsQuery string) (string, error) { +func UpdateThrottlerTopoConfig(clusterInstance *cluster.LocalProcessCluster, enable bool, disable bool, threshold float64, metricsQuery string, appRule *topodatapb.ThrottledAppRule) (string, error) { rec := concurrency.AllErrorRecorder{} var ( err error res strings.Builder ) for _, ks := range clusterInstance.Keyspaces { - ires, err := UpdateThrottlerTopoConfigRaw(&clusterInstance.VtctldClientProcess, ks.Name, enable, disable, threshold, metricsQuery) + ires, err := UpdateThrottlerTopoConfigRaw(&clusterInstance.VtctldClientProcess, ks.Name, enable, disable, threshold, metricsQuery, appRule) if err != nil { rec.RecordError(err) } @@ -335,7 +345,7 @@ func WaitForThrottledApp(t *testing.T, tablet *cluster.Vttablet, throttlerApp th // The throttler is configued to use the standard replication lag metric. The function waits until the throttler is confirmed // to be running on all tablets. func EnableLagThrottlerAndWaitForStatus(t *testing.T, clusterInstance *cluster.LocalProcessCluster, lag time.Duration) { - _, err := UpdateThrottlerTopoConfig(clusterInstance, true, false, lag.Seconds(), "") + _, err := UpdateThrottlerTopoConfig(clusterInstance, true, false, lag.Seconds(), "", nil) require.NoError(t, err) for _, ks := range clusterInstance.Keyspaces { diff --git a/go/test/endtoend/vreplication/cluster_test.go b/go/test/endtoend/vreplication/cluster_test.go index 12539b778de..4c6f86820df 100644 --- a/go/test/endtoend/vreplication/cluster_test.go +++ b/go/test/endtoend/vreplication/cluster_test.go @@ -369,7 +369,7 @@ func (vc *VitessCluster) AddKeyspace(t *testing.T, cells []*Cell, ksName string, } log.Infof("Applying throttler config for keyspace %s", keyspace.Name) - res, err := throttler.UpdateThrottlerTopoConfigRaw(vc.VtctldClient, keyspace.Name, true, false, throttlerConfig.Threshold, throttlerConfig.Query) + res, err := throttler.UpdateThrottlerTopoConfigRaw(vc.VtctldClient, keyspace.Name, true, false, throttlerConfig.Threshold, throttlerConfig.Query, nil) require.NoError(t, err, res) cellsToWatch := "" diff --git a/go/vt/proto/topodata/topodata.pb.go b/go/vt/proto/topodata/topodata.pb.go index 0405dc6b6da..43ecdbce963 100644 --- a/go/vt/proto/topodata/topodata.pb.go +++ b/go/vt/proto/topodata/topodata.pb.go @@ -1011,6 +1011,8 @@ type ThrottledAppRule struct { Ratio float64 `protobuf:"fixed64,2,opt,name=ratio,proto3" json:"ratio,omitempty"` // ExpiresAt is the time at which the rule expires. ExpiresAt *vttime.Time `protobuf:"bytes,3,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + // Exempt indicates the app should never be throttled, even if the throttler is, in general, throttling other apps. + Exempt bool `protobuf:"varint,4,opt,name=exempt,proto3" json:"exempt,omitempty"` } func (x *ThrottledAppRule) Reset() { @@ -1066,6 +1068,13 @@ func (x *ThrottledAppRule) GetExpiresAt() *vttime.Time { return nil } +func (x *ThrottledAppRule) GetExempt() bool { + if x != nil { + return x.Exempt + } + return false +} + type ThrottlerConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2066,110 +2075,112 @@ var file_topodata_proto_rawDesc = []byte{ 0x34, 0x0a, 0x16, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x71, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x69, 0x73, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x69, 0x0a, 0x10, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, - 0x65, 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x12, 0x2b, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, - 0x22, 0xce, 0x02, 0x0a, 0x0f, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, - 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x01, 0x52, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x21, 0x0a, 0x0c, - 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, - 0x2d, 0x0a, 0x13, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x61, 0x73, 0x5f, 0x63, 0x68, 0x65, 0x63, - 0x6b, 0x5f, 0x73, 0x65, 0x6c, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x63, 0x68, - 0x65, 0x63, 0x6b, 0x41, 0x73, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x53, 0x65, 0x6c, 0x66, 0x12, 0x53, - 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, - 0x70, 0x70, 0x73, 0x1a, 0x5c, 0x0a, 0x12, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, - 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x6f, 0x70, - 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, - 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xb6, 0x04, 0x0a, 0x0b, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x12, 0x47, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2e, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x64, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x4b, 0x65, - 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x64, 0x46, 0x72, 0x6f, - 0x6d, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x44, 0x0a, - 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x0f, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x1a, 0xe1, 0x01, 0x0a, 0x11, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, - 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x43, 0x0a, 0x10, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x6f, 0x70, - 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x15, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x52, 0x13, 0x73, 0x68, 0x61, 0x72, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x1a, 0x5f, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x35, 0x0a, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, - 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, - 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x4b, 0x0a, 0x08, 0x43, 0x65, - 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, - 0x74, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x22, 0x22, 0x0a, 0x0a, 0x43, 0x65, 0x6c, 0x6c, 0x73, - 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x22, 0x55, 0x0a, 0x0a, 0x54, - 0x6f, 0x70, 0x6f, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x6f, 0x70, - 0x6f, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x6f, - 0x70, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, - 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, - 0x6f, 0x74, 0x22, 0x4e, 0x0a, 0x15, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x56, 0x69, - 0x74, 0x65, 0x73, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x35, 0x0a, 0x0b, 0x74, - 0x6f, 0x70, 0x6f, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x6f, 0x70, 0x6f, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x74, 0x6f, 0x70, 0x6f, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x22, 0x5a, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x46, 0x0a, 0x0e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x56, 0x69, 0x74, 0x65, 0x73, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, - 0x0d, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2a, 0x28, - 0x0a, 0x0c, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, - 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x4e, - 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x10, 0x01, 0x2a, 0x9d, 0x01, 0x0a, 0x0a, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x49, 0x4d, 0x41, 0x52, 0x59, 0x10, - 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, 0x53, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0b, 0x0a, - 0x07, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x44, - 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, - 0x03, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x50, 0x41, 0x52, 0x45, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, - 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x05, 0x12, 0x0a, - 0x0a, 0x06, 0x42, 0x41, 0x43, 0x4b, 0x55, 0x50, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, - 0x53, 0x54, 0x4f, 0x52, 0x45, 0x10, 0x07, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x52, 0x41, 0x49, 0x4e, - 0x45, 0x44, 0x10, 0x08, 0x1a, 0x02, 0x10, 0x01, 0x42, 0x38, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x76, - 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x25, 0x76, 0x69, 0x74, - 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, - 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, - 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, + 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x12, 0x2b, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, + 0x65, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x06, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x74, 0x22, 0xce, 0x02, 0x0a, 0x0f, 0x54, 0x68, + 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, + 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, + 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x74, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2d, 0x0a, 0x13, 0x63, 0x68, 0x65, 0x63, + 0x6b, 0x5f, 0x61, 0x73, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x6c, 0x66, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x41, 0x73, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x53, 0x65, 0x6c, 0x66, 0x12, 0x53, 0x0a, 0x0e, 0x74, 0x68, 0x72, 0x6f, 0x74, + 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2c, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, + 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x68, 0x72, 0x6f, 0x74, + 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x74, + 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x1a, 0x5c, 0x0a, 0x12, + 0x54, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6c, 0x65, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb6, 0x04, 0x0a, 0x0b, 0x53, + 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x47, 0x0a, 0x0a, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, + 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x2e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x66, 0x72, + 0x6f, 0x6d, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2e, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x44, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, + 0x6c, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x74, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0xe1, 0x01, 0x0a, + 0x11, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x0f, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x50, + 0x0a, 0x15, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x52, 0x13, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x73, + 0x1a, 0x5f, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x35, + 0x0a, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, + 0x05, 0x10, 0x06, 0x22, 0x4b, 0x0a, 0x08, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, + 0x22, 0x22, 0x0a, 0x0a, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, + 0x65, 0x6c, 0x6c, 0x73, 0x22, 0x55, 0x0a, 0x0a, 0x54, 0x6f, 0x70, 0x6f, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x6f, 0x70, 0x6f, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x6f, 0x70, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x22, 0x4e, 0x0a, 0x15, 0x45, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x56, 0x69, 0x74, 0x65, 0x73, 0x73, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x12, 0x35, 0x0a, 0x0b, 0x74, 0x6f, 0x70, 0x6f, 0x5f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x6f, 0x70, 0x6f, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x0a, 0x74, 0x6f, 0x70, 0x6f, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x5a, 0x0a, 0x10, 0x45, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x46, 0x0a, 0x0e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x56, 0x69, 0x74, 0x65, 0x73, + 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x0d, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2a, 0x28, 0x0a, 0x0c, 0x4b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, + 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x10, + 0x01, 0x2a, 0x9d, 0x01, 0x0a, 0x0a, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, + 0x07, 0x50, 0x52, 0x49, 0x4d, 0x41, 0x52, 0x59, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x41, + 0x53, 0x54, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, + 0x41, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x52, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x03, 0x12, + 0x09, 0x0a, 0x05, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x50, + 0x41, 0x52, 0x45, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, 0x45, 0x58, 0x50, 0x45, 0x52, 0x49, 0x4d, + 0x45, 0x4e, 0x54, 0x41, 0x4c, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x43, 0x4b, 0x55, + 0x50, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x53, 0x54, 0x4f, 0x52, 0x45, 0x10, 0x07, + 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x08, 0x1a, 0x02, 0x10, + 0x01, 0x42, 0x38, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x25, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, + 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/topodata/topodata_vtproto.pb.go b/go/vt/proto/topodata/topodata_vtproto.pb.go index e17a955a8fb..5ed87445845 100644 --- a/go/vt/proto/topodata/topodata_vtproto.pb.go +++ b/go/vt/proto/topodata/topodata_vtproto.pb.go @@ -919,6 +919,16 @@ func (m *ThrottledAppRule) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Exempt { + i-- + if m.Exempt { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if m.ExpiresAt != nil { size, err := m.ExpiresAt.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -1812,6 +1822,9 @@ func (m *ThrottledAppRule) SizeVT() (n int) { l = m.ExpiresAt.SizeVT() n += 1 + l + sov(uint64(l)) } + if m.Exempt { + n += 2 + } n += len(m.unknownFields) return n } @@ -4502,6 +4515,26 @@ func (m *ThrottledAppRule) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Exempt", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Exempt = bool(v != 0) default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 454780eb50d..9aec6995fdf 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -705,7 +705,7 @@ var commands = []commandGroup{ { name: "UpdateThrottlerConfig", method: commandUpdateThrottlerConfig, - params: "[--enable|--disable] [--threshold=] [--custom-query=] [--check-as-check-self|--check-as-check-shard] [--throttle-app|unthrottle-app=] [--throttle-app-ratio=] [--throttle-app-duration=] ", + params: "[--enable|--disable] [--threshold=] [--custom-query=] [--check-as-check-self|--check-as-check-shard] [--throttle-app|unthrottle-app=] [--throttle-app-ratio=] [--throttle-app-duration=] [--throttle-app-exempt] ", help: "Update the table throttler configuration for all cells and tablets of a given keyspace", }, { @@ -3571,6 +3571,7 @@ func commandUpdateThrottlerConfig(ctx context.Context, wr *wrangler.Wrangler, su throttledApp := subFlags.String("throttle-app", "", "an app name to throttle") throttledAppRatio := subFlags.Float64("throttle-app-ratio", throttle.DefaultThrottleRatio, "ratio to throttle app (app specififed in --throttled-app)") throttledAppDuration := subFlags.Duration("throttle-app-duration", throttle.DefaultAppThrottleDuration, "duration after which throttled app rule expires (app specified in --throttled-app)") + throttledAppExempt := subFlags.Bool("throttle-app-exempt", false, "exempt this app from being at all throttled. WARNING: use with extreme care, as this is likely to push metrics beyond the throttler's threshold, and starve other apps (app specified in --throttled-app)") if err := subFlags.Parse(args); err != nil { return err } @@ -3594,6 +3595,9 @@ func commandUpdateThrottlerConfig(ctx context.Context, wr *wrangler.Wrangler, su if subFlags.Changed("throttle-app-duration") && *throttledApp == "" { return fmt.Errorf("--throttle-app-duration requires --throttle-app") } + if subFlags.Changed("throttle-app-exempt") && *throttledApp == "" { + return fmt.Errorf("--throttle-app-exempt requires --throttle-app") + } keyspace := subFlags.Arg(0) @@ -3611,6 +3615,7 @@ func commandUpdateThrottlerConfig(ctx context.Context, wr *wrangler.Wrangler, su req.ThrottledApp = &topodatapb.ThrottledAppRule{ Name: *throttledApp, Ratio: *throttledAppRatio, + Exempt: *throttledAppExempt, ExpiresAt: logutil.TimeToProto(time.Now().Add(*throttledAppDuration)), } } else if *unthrottledApp != "" { diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index cbf287ac090..244aa8ce8b2 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -2115,7 +2115,7 @@ func (e *Executor) validateThrottleParams(ctx context.Context, expireString stri return duration, ratio, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid EXPIRE value: %s. Try '120s', '30m', '1h', etc. Allowed units are (s)ec, (m)in, (h)hour", expireString) } } - ratio = 1.0 + ratio = throttle.DefaultThrottleRatio if ratioLiteral != nil { ratio, err = strconv.ParseFloat(ratioLiteral.Val, 64) if err != nil || ratio < 0 || ratio > 1 { @@ -2134,7 +2134,7 @@ func (e *Executor) ThrottleMigration(ctx context.Context, uuid string, expireStr if err := e.lagThrottler.CheckIsOpen(); err != nil { return nil, err } - _ = e.lagThrottler.ThrottleApp(uuid, time.Now().Add(duration), ratio) + _ = e.lagThrottler.ThrottleApp(uuid, time.Now().Add(duration), ratio, false) return emptyResult, nil } @@ -2147,7 +2147,7 @@ func (e *Executor) ThrottleAllMigrations(ctx context.Context, expireString strin if err := e.lagThrottler.CheckIsOpen(); err != nil { return nil, err } - _ = e.lagThrottler.ThrottleApp(throttlerapp.OnlineDDLName.String(), time.Now().Add(duration), ratio) + _ = e.lagThrottler.ThrottleApp(throttlerapp.OnlineDDLName.String(), time.Now().Add(duration), ratio, false) return emptyResult, nil } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 4ccf3d4b71f..ea439314683 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -1849,7 +1849,7 @@ func (tsv *TabletServer) registerThrottlerThrottleAppHandler() { http.Error(w, fmt.Sprintf("not ok: %v", err), http.StatusInternalServerError) return } - var ratio = 1.0 + var ratio = throttle.DefaultThrottleRatio if ratioParam := r.URL.Query().Get("ratio"); ratioParam != "" { ratio, err = strconv.ParseFloat(ratioParam, 64) if err != nil { @@ -1857,7 +1857,7 @@ func (tsv *TabletServer) registerThrottlerThrottleAppHandler() { return } } - appThrottle := tsv.lagThrottler.ThrottleApp(appName, time.Now().Add(d), ratio) + appThrottle := tsv.lagThrottler.ThrottleApp(appName, time.Now().Add(d), ratio, false) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(appThrottle) diff --git a/go/vt/vttablet/tabletserver/throttle/base/app_throttle.go b/go/vt/vttablet/tabletserver/throttle/base/app_throttle.go index 0633e99c95c..36d57bdc973 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/app_throttle.go +++ b/go/vt/vttablet/tabletserver/throttle/base/app_throttle.go @@ -26,14 +26,16 @@ type AppThrottle struct { AppName string ExpireAt time.Time Ratio float64 + Exempt bool } // NewAppThrottle creates an AppThrottle struct -func NewAppThrottle(appName string, expireAt time.Time, ratio float64) *AppThrottle { +func NewAppThrottle(appName string, expireAt time.Time, ratio float64, exempt bool) *AppThrottle { result := &AppThrottle{ AppName: appName, ExpireAt: expireAt, Ratio: ratio, + Exempt: exempt, } return result } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 125f361e7cc..d492472959f 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -348,7 +348,7 @@ func (throttler *Throttler) applyThrottlerConfig(ctx context.Context, throttlerC throttler.StoreMetricsThreshold(throttlerConfig.Threshold) throttler.checkAsCheckSelf.Store(throttlerConfig.CheckAsCheckSelf) for _, appRule := range throttlerConfig.ThrottledApps { - throttler.ThrottleApp(appRule.Name, logutil.ProtoToTime(appRule.ExpiresAt), appRule.Ratio) + throttler.ThrottleApp(appRule.Name, logutil.ProtoToTime(appRule.ExpiresAt), appRule.Ratio, appRule.Exempt) } if throttlerConfig.Enabled { go throttler.Enable(ctx) @@ -445,7 +445,7 @@ func (throttler *Throttler) Open() error { throttler.pool.Open(throttler.env.Config().DB.AppWithDB(), throttler.env.Config().DB.DbaWithDB(), throttler.env.Config().DB.AppDebugWithDB()) atomic.StoreInt64(&throttler.isOpen, 1) - throttler.ThrottleApp("always-throttled-app", time.Now().Add(time.Hour*24*365*10), DefaultThrottleRatio) + throttler.ThrottleApp("always-throttled-app", time.Now().Add(time.Hour*24*365*10), DefaultThrottleRatio, false) log.Infof("Throttler: throttler-config-via-topo detected") // We want to read throttler config from topo and apply it. @@ -934,7 +934,7 @@ func (throttler *Throttler) expireThrottledApps() { } // ThrottleApp instructs the throttler to begin throttling an app, to som eperiod and with some ratio. -func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, ratio float64) (appThrottle *base.AppThrottle) { +func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, ratio float64, exempt bool) (appThrottle *base.AppThrottle) { throttler.throttledAppsMutex.Lock() defer throttler.throttledAppsMutex.Unlock() @@ -947,6 +947,7 @@ func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, rati if ratio >= 0 { appThrottle.Ratio = ratio } + appThrottle.Exempt = exempt } else { if expireAt.IsZero() { expireAt = now.Add(DefaultAppThrottleDuration) @@ -954,7 +955,7 @@ func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, rati if ratio < 0 { ratio = DefaultThrottleRatio } - appThrottle = base.NewAppThrottle(appName, expireAt, ratio) + appThrottle = base.NewAppThrottle(appName, expireAt, ratio, exempt) } if now.Before(appThrottle.ExpireAt) { throttler.throttledApps.Set(appName, appThrottle, cache.DefaultExpiration) @@ -969,7 +970,7 @@ func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.App throttler.throttledApps.Delete(appName) // the app is likely to check go throttler.heartbeatWriter.RequestHeartbeats() - return base.NewAppThrottle(appName, time.Now(), 0) + return base.NewAppThrottle(appName, time.Now(), 0, false) } // IsAppThrottled tells whether some app should be throttled. @@ -977,16 +978,21 @@ func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.App // on the throttle ratio func (throttler *Throttler) IsAppThrottled(appName string) bool { isSingleAppNameThrottled := func(singleAppName string) bool { - if object, found := throttler.throttledApps.Get(singleAppName); found { - appThrottle := object.(*base.AppThrottle) - if appThrottle.ExpireAt.Before(time.Now()) { - // throttling cleanup hasn't purged yet, but it is expired - return false - } - // handle ratio - if rand.Float64() < appThrottle.Ratio { - return true - } + object, found := throttler.throttledApps.Get(singleAppName) + if !found { + return false + } + appThrottle := object.(*base.AppThrottle) + if !appThrottle.ExpireAt.After(time.Now()) { + // throttling cleanup hasn't purged yet, but it is expired + return false + } + if appThrottle.Exempt { + return false + } + // handle ratio + if rand.Float64() < appThrottle.Ratio { + return true } return false } @@ -1004,6 +1010,40 @@ func (throttler *Throttler) IsAppThrottled(appName string) bool { return false } +// IsAppExempt +func (throttler *Throttler) IsAppExempted(appName string) bool { + isSingleAppNameExempted := func(singleAppName string) bool { + if throttlerapp.ExemptFromChecks(appName) { // well known statically exempted apps + return true + } + object, found := throttler.throttledApps.Get(singleAppName) + if !found { + return false + } + appThrottle := object.(*base.AppThrottle) + if !appThrottle.ExpireAt.After(time.Now()) { + // throttling cleanup hasn't purged yet, but it is expired + return false + } + if appThrottle.Exempt { + return true + } + return false + } + if isSingleAppNameExempted(appName) { + return true + } + for _, singleAppName := range strings.Split(appName, ":") { + if singleAppName == "" { + continue + } + if isSingleAppNameExempted(singleAppName) { + return true + } + } + return false +} + // ThrottledAppsMap returns a (copy) map of currently throttled apps func (throttler *Throttler) ThrottledAppsMap() (result map[string](*base.AppThrottle)) { result = make(map[string](*base.AppThrottle)) @@ -1070,7 +1110,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor if !throttler.IsRunning() { return okMetricCheckResult } - if throttlerapp.ExemptFromChecks(appName) { + if throttler.IsAppExempted(appName) { // Some apps are exempt from checks. They are always responded with OK. This is because those apps are // continuous and do not generate a substantial load. return okMetricCheckResult diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go new file mode 100644 index 00000000000..2995e03b654 --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -0,0 +1,77 @@ +/* + Copyright 2017 GitHub Inc. + + Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +*/ + +package throttle + +import ( + "testing" + "time" + + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" +) + +type FakeHeartbeatWriter struct { +} + +func (w FakeHeartbeatWriter) RequestHeartbeats() { +} + +func TestIsAppThrottled(t *testing.T) { + throttler := Throttler{ + throttledApps: cache.New(cache.NoExpiration, 0), + heartbeatWriter: FakeHeartbeatWriter{}, + } + assert.False(t, throttler.IsAppThrottled("app1")) + assert.False(t, throttler.IsAppThrottled("app2")) + assert.False(t, throttler.IsAppThrottled("app3")) + assert.False(t, throttler.IsAppThrottled("app4")) + // + throttler.ThrottleApp("app1", time.Now().Add(time.Hour), DefaultThrottleRatio, true) + throttler.ThrottleApp("app2", time.Now(), DefaultThrottleRatio, false) + throttler.ThrottleApp("app3", time.Now().Add(time.Hour), DefaultThrottleRatio, false) + throttler.ThrottleApp("app4", time.Now().Add(time.Hour), 0, false) + assert.False(t, throttler.IsAppThrottled("app1")) // exempted + assert.False(t, throttler.IsAppThrottled("app2")) // expired + assert.True(t, throttler.IsAppThrottled("app3")) + assert.False(t, throttler.IsAppThrottled("app4")) // ratio is zero + // + throttler.UnthrottleApp("app1") + throttler.UnthrottleApp("app2") + throttler.UnthrottleApp("app3") + throttler.UnthrottleApp("app4") + assert.False(t, throttler.IsAppThrottled("app1")) + assert.False(t, throttler.IsAppThrottled("app2")) + assert.False(t, throttler.IsAppThrottled("app3")) + assert.False(t, throttler.IsAppThrottled("app4")) +} + +func TestIsAppExempted(t *testing.T) { + throttler := Throttler{ + throttledApps: cache.New(cache.NoExpiration, 0), + heartbeatWriter: FakeHeartbeatWriter{}, + } + assert.False(t, throttler.IsAppExempted("app1")) + assert.False(t, throttler.IsAppExempted("app2")) + assert.False(t, throttler.IsAppExempted("app3")) + // + throttler.ThrottleApp("app1", time.Now().Add(time.Hour), DefaultThrottleRatio, true) + throttler.ThrottleApp("app2", time.Now(), DefaultThrottleRatio, true) // instantly expire + assert.True(t, throttler.IsAppExempted("app1")) + assert.True(t, throttler.IsAppExempted("app1:other-tag")) + assert.False(t, throttler.IsAppExempted("app2")) // expired + assert.False(t, throttler.IsAppExempted("app3")) + // + throttler.UnthrottleApp("app1") + throttler.ThrottleApp("app2", time.Now().Add(time.Hour), DefaultThrottleRatio, false) + assert.False(t, throttler.IsAppExempted("app1")) + assert.False(t, throttler.IsAppExempted("app2")) + assert.False(t, throttler.IsAppExempted("app3")) + // + assert.True(t, throttler.IsAppExempted("schema-tracker")) + throttler.UnthrottleApp("schema-tracker") // meaningless. App is statically exempted + assert.True(t, throttler.IsAppExempted("schema-tracker")) +} diff --git a/proto/topodata.proto b/proto/topodata.proto index 7275e7cfd58..c921f72dfa4 100644 --- a/proto/topodata.proto +++ b/proto/topodata.proto @@ -375,6 +375,8 @@ message ThrottledAppRule { double ratio = 2; // ExpiresAt is the time at which the rule expires. vttime.Time expires_at = 3; + // Exempt indicates the app should never be throttled, even if the throttler is, in general, throttling other apps. + bool exempt = 4; } message ThrottlerConfig { diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 71979452d5c..9061eb82477 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -15865,6 +15865,9 @@ export namespace topodata { /** ThrottledAppRule expires_at */ expires_at?: (vttime.ITime|null); + + /** ThrottledAppRule exempt */ + exempt?: (boolean|null); } /** Represents a ThrottledAppRule. */ @@ -15885,6 +15888,9 @@ export namespace topodata { /** ThrottledAppRule expires_at. */ public expires_at?: (vttime.ITime|null); + /** ThrottledAppRule exempt. */ + public exempt: boolean; + /** * Creates a new ThrottledAppRule instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index f527711b04a..b640de318a7 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -36946,6 +36946,7 @@ export const topodata = $root.topodata = (() => { * @property {string|null} [name] ThrottledAppRule name * @property {number|null} [ratio] ThrottledAppRule ratio * @property {vttime.ITime|null} [expires_at] ThrottledAppRule expires_at + * @property {boolean|null} [exempt] ThrottledAppRule exempt */ /** @@ -36987,6 +36988,14 @@ export const topodata = $root.topodata = (() => { */ ThrottledAppRule.prototype.expires_at = null; + /** + * ThrottledAppRule exempt. + * @member {boolean} exempt + * @memberof topodata.ThrottledAppRule + * @instance + */ + ThrottledAppRule.prototype.exempt = false; + /** * Creates a new ThrottledAppRule instance using the specified properties. * @function create @@ -37017,6 +37026,8 @@ export const topodata = $root.topodata = (() => { writer.uint32(/* id 2, wireType 1 =*/17).double(message.ratio); if (message.expires_at != null && Object.hasOwnProperty.call(message, "expires_at")) $root.vttime.Time.encode(message.expires_at, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.exempt != null && Object.hasOwnProperty.call(message, "exempt")) + writer.uint32(/* id 4, wireType 0 =*/32).bool(message.exempt); return writer; }; @@ -37063,6 +37074,10 @@ export const topodata = $root.topodata = (() => { message.expires_at = $root.vttime.Time.decode(reader, reader.uint32()); break; } + case 4: { + message.exempt = reader.bool(); + break; + } default: reader.skipType(tag & 7); break; @@ -37109,6 +37124,9 @@ export const topodata = $root.topodata = (() => { if (error) return "expires_at." + error; } + if (message.exempt != null && message.hasOwnProperty("exempt")) + if (typeof message.exempt !== "boolean") + return "exempt: boolean expected"; return null; }; @@ -37133,6 +37151,8 @@ export const topodata = $root.topodata = (() => { throw TypeError(".topodata.ThrottledAppRule.expires_at: object expected"); message.expires_at = $root.vttime.Time.fromObject(object.expires_at); } + if (object.exempt != null) + message.exempt = Boolean(object.exempt); return message; }; @@ -37153,6 +37173,7 @@ export const topodata = $root.topodata = (() => { object.name = ""; object.ratio = 0; object.expires_at = null; + object.exempt = false; } if (message.name != null && message.hasOwnProperty("name")) object.name = message.name; @@ -37160,6 +37181,8 @@ export const topodata = $root.topodata = (() => { object.ratio = options.json && !isFinite(message.ratio) ? String(message.ratio) : message.ratio; if (message.expires_at != null && message.hasOwnProperty("expires_at")) object.expires_at = $root.vttime.Time.toObject(message.expires_at, options); + if (message.exempt != null && message.hasOwnProperty("exempt")) + object.exempt = message.exempt; return object; };