From cb3ee6bb6b128c10fea0f67514318f91212d4204 Mon Sep 17 00:00:00 2001 From: whywaita Date: Fri, 14 Jan 2022 12:35:32 +0900 Subject: [PATCH 1/4] Add metrics for rate limit --- pkg/gh/github.go | 7 +++++++ pkg/gh/ratelimit.go | 41 +++++++++++++++++++++++++++++++++++++ pkg/metric/scrape_memory.go | 20 ++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 pkg/gh/ratelimit.go diff --git a/pkg/gh/github.go b/pkg/gh/github.go index a661491..b8967c1 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -8,6 +8,7 @@ import ( "net/url" "path" "strings" + "sync" "time" "github.com/bradleyfalzon/ghinstallation/v2" @@ -25,11 +26,16 @@ var ( // ResponseCache is cache variable responseCache *cache.Cache + + // rateLimitCount is count of Rate limit, for metrics + rateLimitCount = sync.Map{} ) func init() { c := cache.New(5*time.Minute, 10*time.Minute) responseCache = c + + rateLimitCount = sync.Map{} } // NewClient create a client of GitHub @@ -170,6 +176,7 @@ func ListRunners(ctx context.Context, client *github.Client, owner, repo string) if err != nil { return nil, fmt.Errorf("failed to list runners: %w", err) } + storeRateLimit(owner, repo, resp.Rate.Limit) rs = append(rs, runners.Runners...) if resp.NextPage == 0 { diff --git a/pkg/gh/ratelimit.go b/pkg/gh/ratelimit.go new file mode 100644 index 0000000..44659a0 --- /dev/null +++ b/pkg/gh/ratelimit.go @@ -0,0 +1,41 @@ +package gh + +import ( + "fmt" + "sync" +) + +func storeRateLimit(org, repo string, rateLimit int) { + rateLimitCount.Store(getRateLimitKey(org, repo), rateLimit) +} + +func getRateLimitKey(org, repo string) string { + if repo == "" { + return org + } + return fmt.Sprintf("%s/%s", org, repo) +} + +// GetRateLimit get a list of rate limit +func GetRateLimit() map[string]int { + m := map[string]int{} + mu := sync.Mutex{} + + rateLimitCount.Range(func(key, value interface{}) bool { + k, ok := key.(string) + if !ok { + return false + } + v, ok := value.(int) + if !ok { + return false + } + + mu.Lock() + m[k] = v + mu.Unlock() + return true + }) + + return m +} diff --git a/pkg/metric/scrape_memory.go b/pkg/metric/scrape_memory.go index 832e31a..a06f9c3 100644 --- a/pkg/metric/scrape_memory.go +++ b/pkg/metric/scrape_memory.go @@ -8,6 +8,7 @@ import ( "github.com/whywaita/myshoes/internal/config" "github.com/whywaita/myshoes/pkg/datastore" + "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/starter" ) @@ -29,6 +30,11 @@ var ( "waiting queue in starter", []string{"starter"}, nil, ) + memoryGitHubRateLimit = prometheus.NewDesc( + prometheus.BuildFQName(namespace, memoryName, "github_rate_limit"), + "The number of rate limit", + []string{"scope"}, nil, + ) ) // ScraperMemory is scraper implement for memory @@ -49,6 +55,9 @@ func (ScraperMemory) Scrape(ctx context.Context, ds datastore.Datastore, ch chan if err := scrapeStarterValues(ch); err != nil { return fmt.Errorf("failed to scrape starter values: %w", err) } + if err := scrapeGitHubValues(ch); err != nil { + return fmt.Errorf("failed to scrape GitHub values: %w", err) + } return nil } @@ -72,4 +81,15 @@ func scrapeStarterValues(ch chan<- prometheus.Metric) error { return nil } +func scrapeGitHubValues(ch chan<- prometheus.Metric) error { + rateLimitCount := gh.GetRateLimit() + for scope, count := range rateLimitCount { + ch <- prometheus.MustNewConstMetric( + memoryGitHubRateLimit, prometheus.GaugeValue, float64(count), scope, + ) + } + + return nil +} + var _ Scraper = ScraperMemory{} From 6d8af935d63d10acdc67a38a3d4ededa86d3ad6a Mon Sep 17 00:00:00 2001 From: whywaita Date: Fri, 14 Jan 2022 17:31:08 +0900 Subject: [PATCH 2/4] Update metrics of rate limit when token update --- pkg/gh/github.go | 2 +- pkg/gh/jwt.go | 11 ++++++----- pkg/gh/jwt_test.go | 2 +- pkg/gh/ratelimit.go | 4 ++-- pkg/runner/token_update.go | 2 +- pkg/web/target_create.go | 2 +- pkg/web/target_test.go | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/gh/github.go b/pkg/gh/github.go index b8967c1..9c8357d 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -176,7 +176,7 @@ func ListRunners(ctx context.Context, client *github.Client, owner, repo string) if err != nil { return nil, fmt.Errorf("failed to list runners: %w", err) } - storeRateLimit(owner, repo, resp.Rate.Limit) + storeRateLimit(getRateLimitKey(owner, repo), resp.Rate.Limit) rs = append(rs, runners.Runners...) if resp.NextPage == 0 { diff --git a/pkg/gh/jwt.go b/pkg/gh/jwt.go index 077be61..d9de5c6 100644 --- a/pkg/gh/jwt.go +++ b/pkg/gh/jwt.go @@ -22,11 +22,12 @@ var ( // GenerateGitHubAppsToken generate token of GitHub Apps using private key // clientApps needs to response of `NewClientGitHubApps()` -func GenerateGitHubAppsToken(ctx context.Context, clientApps *github.Client, installationID int64) (string, *time.Time, error) { - token, _, err := clientApps.Apps.CreateInstallationToken(ctx, installationID, nil) +func GenerateGitHubAppsToken(ctx context.Context, clientApps *github.Client, installationID int64, scope string) (string, *time.Time, error) { + token, resp, err := clientApps.Apps.CreateInstallationToken(ctx, installationID, nil) if err != nil { return "", nil, fmt.Errorf("failed to generate token from API: %w", err) } + storeRateLimit(scope, resp.Rate.Limit) return *token.Token, token.ExpiresAt, nil } @@ -97,7 +98,7 @@ func IsInstalledGitHubApp(ctx context.Context, gheDomain, inputScope string) (in } func isInstalledGitHubAppSelected(ctx context.Context, gheDomain, inputScope string, installationID int64) error { - lr, err := GHlistAppsInstalledRepo(ctx, gheDomain, installationID) + lr, err := GHlistAppsInstalledRepo(ctx, gheDomain, installationID, inputScope) if err != nil { return fmt.Errorf("failed to get list of installed repositories: %w", err) } @@ -123,12 +124,12 @@ func isInstalledGitHubAppSelected(ctx context.Context, gheDomain, inputScope str return fmt.Errorf("not found") } -func listAppsInstalledRepo(ctx context.Context, gheDomain string, installationID int64) (*github.ListRepositories, error) { +func listAppsInstalledRepo(ctx context.Context, gheDomain string, installationID int64, inputScope string) (*github.ListRepositories, error) { clientApps, err := NewClientGitHubApps(gheDomain, config.Config.GitHub.AppID, config.Config.GitHub.PEMByte) if err != nil { return nil, fmt.Errorf("failed to create github.Client from installationID: %w", err) } - token, _, err := GenerateGitHubAppsToken(ctx, clientApps, installationID) + token, _, err := GenerateGitHubAppsToken(ctx, clientApps, installationID, inputScope) if err != nil { return nil, fmt.Errorf("failed to generate GitHub Apps Token: %w", err) } diff --git a/pkg/gh/jwt_test.go b/pkg/gh/jwt_test.go index 05b0eab..cc816d9 100644 --- a/pkg/gh/jwt_test.go +++ b/pkg/gh/jwt_test.go @@ -49,7 +49,7 @@ func setStubFunctions() { }, nil } - GHlistAppsInstalledRepo = func(ctx context.Context, gheDomain string, installationID int64) (*github.ListRepositories, error) { + GHlistAppsInstalledRepo = func(ctx context.Context, gheDomain string, installationID int64, scope string) (*github.ListRepositories, error) { total := 1 fullName1 := "example-selected/sample-registered" return &github.ListRepositories{ diff --git a/pkg/gh/ratelimit.go b/pkg/gh/ratelimit.go index 44659a0..3fe91e1 100644 --- a/pkg/gh/ratelimit.go +++ b/pkg/gh/ratelimit.go @@ -5,8 +5,8 @@ import ( "sync" ) -func storeRateLimit(org, repo string, rateLimit int) { - rateLimitCount.Store(getRateLimitKey(org, repo), rateLimit) +func storeRateLimit(scope string, rateLimit int) { + rateLimitCount.Store(scope, rateLimit) } func getRateLimitKey(org, repo string) string { diff --git a/pkg/runner/token_update.go b/pkg/runner/token_update.go index 6ea2439..f61ed46 100644 --- a/pkg/runner/token_update.go +++ b/pkg/runner/token_update.go @@ -38,7 +38,7 @@ func (m *Manager) doTargetToken(ctx context.Context) error { logger.Logf(false, "failed to create client of GitHub installation: %+v", err) continue } - token, expiredAt, err := gh.GenerateGitHubAppsToken(ctx, clientApps, installationID) + token, expiredAt, err := gh.GenerateGitHubAppsToken(ctx, clientApps, installationID, target.Scope) if err != nil { logger.Logf(false, "failed to get Apps Token: %+v", err) continue diff --git a/pkg/web/target_create.go b/pkg/web/target_create.go index 3cde0f9..8a58509 100644 --- a/pkg/web/target_create.go +++ b/pkg/web/target_create.go @@ -44,7 +44,7 @@ func handleTargetCreate(w http.ResponseWriter, r *http.Request, ds datastore.Dat outputErrorMsg(w, http.StatusInternalServerError, "failed to client GitHub Apps") return } - token, expiredAt, err := GHGenerateGitHubAppsToken(ctx, clientApps, installationID) + token, expiredAt, err := GHGenerateGitHubAppsToken(ctx, clientApps, installationID, inputTarget.Scope) if err != nil { logger.Logf(false, "failed to generate GitHub Apps Token: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "failed to generate GitHub Apps token") diff --git a/pkg/web/target_test.go b/pkg/web/target_test.go index 7dfcfd6..e95a971 100644 --- a/pkg/web/target_test.go +++ b/pkg/web/target_test.go @@ -47,7 +47,7 @@ func setStubFunctions() { return testInstallationID, nil } - web.GHGenerateGitHubAppsToken = func(ctx context.Context, clientInstallation *github.Client, installationID int64) (string, *time.Time, error) { + web.GHGenerateGitHubAppsToken = func(ctx context.Context, clientInstallation *github.Client, installationID int64, scope string) (string, *time.Time, error) { return testGitHubAppToken, &testTime, nil } From 6376ac15ff37505dc3611bc1884a217b2d9a8043 Mon Sep 17 00:00:00 2001 From: whywaita Date: Fri, 14 Jan 2022 17:51:46 +0900 Subject: [PATCH 3/4] don't need to store metrics in not configured rate limit --- pkg/gh/github.go | 2 +- pkg/gh/jwt.go | 2 +- pkg/gh/ratelimit.go | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/gh/github.go b/pkg/gh/github.go index 9c8357d..cbcf0ad 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -176,7 +176,7 @@ func ListRunners(ctx context.Context, client *github.Client, owner, repo string) if err != nil { return nil, fmt.Errorf("failed to list runners: %w", err) } - storeRateLimit(getRateLimitKey(owner, repo), resp.Rate.Limit) + storeRateLimit(getRateLimitKey(owner, repo), resp.Rate) rs = append(rs, runners.Runners...) if resp.NextPage == 0 { diff --git a/pkg/gh/jwt.go b/pkg/gh/jwt.go index d9de5c6..7797f25 100644 --- a/pkg/gh/jwt.go +++ b/pkg/gh/jwt.go @@ -27,7 +27,7 @@ func GenerateGitHubAppsToken(ctx context.Context, clientApps *github.Client, ins if err != nil { return "", nil, fmt.Errorf("failed to generate token from API: %w", err) } - storeRateLimit(scope, resp.Rate.Limit) + storeRateLimit(scope, resp.Rate) return *token.Token, token.ExpiresAt, nil } diff --git a/pkg/gh/ratelimit.go b/pkg/gh/ratelimit.go index 3fe91e1..dc3796a 100644 --- a/pkg/gh/ratelimit.go +++ b/pkg/gh/ratelimit.go @@ -3,10 +3,18 @@ package gh import ( "fmt" "sync" + + "github.com/google/go-github/v35/github" ) -func storeRateLimit(scope string, rateLimit int) { - rateLimitCount.Store(scope, rateLimit) +func storeRateLimit(scope string, rateLimit github.Rate) { + fmt.Printf("%+v\n", rateLimit) + if rateLimit.Limit == 0 && rateLimit.Reset.IsZero() { + // Not configure rate limit, don't need to store + return + } + + rateLimitCount.Store(scope, rateLimit.Limit) } func getRateLimitKey(org, repo string) string { From fc27460d4abd922a09328d8c4a06087078c36ea2 Mon Sep 17 00:00:00 2001 From: whywaita Date: Fri, 14 Jan 2022 18:14:09 +0900 Subject: [PATCH 4/4] need to store remaining --- pkg/gh/github.go | 9 ++++++--- pkg/gh/ratelimit.go | 40 ++++++++++++++++++++++++++++++------- pkg/metric/scrape_memory.go | 24 ++++++++++++++++------ 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/pkg/gh/github.go b/pkg/gh/github.go index cbcf0ad..7d37d76 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -27,15 +27,18 @@ var ( // ResponseCache is cache variable responseCache *cache.Cache - // rateLimitCount is count of Rate limit, for metrics - rateLimitCount = sync.Map{} + // rateLimitRemain is remaining of Rate limit, for metrics + rateLimitRemain = sync.Map{} + // rateLimitLimit is limit of Rate limit, for metrics + rateLimitLimit = sync.Map{} ) func init() { c := cache.New(5*time.Minute, 10*time.Minute) responseCache = c - rateLimitCount = sync.Map{} + rateLimitRemain = sync.Map{} + rateLimitLimit = sync.Map{} } // NewClient create a client of GitHub diff --git a/pkg/gh/ratelimit.go b/pkg/gh/ratelimit.go index dc3796a..3e939e4 100644 --- a/pkg/gh/ratelimit.go +++ b/pkg/gh/ratelimit.go @@ -8,13 +8,13 @@ import ( ) func storeRateLimit(scope string, rateLimit github.Rate) { - fmt.Printf("%+v\n", rateLimit) - if rateLimit.Limit == 0 && rateLimit.Reset.IsZero() { - // Not configure rate limit, don't need to store + if rateLimit.Reset.IsZero() { + // Not configure rate limit, don't need to store (e.g. GHES) return } - rateLimitCount.Store(scope, rateLimit.Limit) + rateLimitLimit.Store(scope, rateLimit.Limit) + rateLimitRemain.Store(scope, rateLimit.Remaining) } func getRateLimitKey(org, repo string) string { @@ -24,12 +24,38 @@ func getRateLimitKey(org, repo string) string { return fmt.Sprintf("%s/%s", org, repo) } -// GetRateLimit get a list of rate limit -func GetRateLimit() map[string]int { +// GetRateLimitRemain get a list of rate limit remaining +// key: scope, value: remain +func GetRateLimitRemain() map[string]int { m := map[string]int{} mu := sync.Mutex{} - rateLimitCount.Range(func(key, value interface{}) bool { + rateLimitRemain.Range(func(key, value interface{}) bool { + k, ok := key.(string) + if !ok { + return false + } + v, ok := value.(int) + if !ok { + return false + } + + mu.Lock() + m[k] = v + mu.Unlock() + return true + }) + + return m +} + +// GetRateLimitLimit get a list of rate limit +// key: scope, value: remain +func GetRateLimitLimit() map[string]int { + m := map[string]int{} + mu := sync.Mutex{} + + rateLimitLimit.Range(func(key, value interface{}) bool { k, ok := key.(string) if !ok { return false diff --git a/pkg/metric/scrape_memory.go b/pkg/metric/scrape_memory.go index a06f9c3..cd303ce 100644 --- a/pkg/metric/scrape_memory.go +++ b/pkg/metric/scrape_memory.go @@ -30,9 +30,14 @@ var ( "waiting queue in starter", []string{"starter"}, nil, ) - memoryGitHubRateLimit = prometheus.NewDesc( - prometheus.BuildFQName(namespace, memoryName, "github_rate_limit"), - "The number of rate limit", + memoryGitHubRateLimitRemaining = prometheus.NewDesc( + prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_remaining"), + "The number of rate limit remaining", + []string{"scope"}, nil, + ) + memoryGitHubRateLimitLimiting = prometheus.NewDesc( + prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_limiting"), + "The number of rate limit max", []string{"scope"}, nil, ) ) @@ -82,10 +87,17 @@ func scrapeStarterValues(ch chan<- prometheus.Metric) error { } func scrapeGitHubValues(ch chan<- prometheus.Metric) error { - rateLimitCount := gh.GetRateLimit() - for scope, count := range rateLimitCount { + rateLimitRemain := gh.GetRateLimitRemain() + for scope, remain := range rateLimitRemain { + ch <- prometheus.MustNewConstMetric( + memoryGitHubRateLimitRemaining, prometheus.GaugeValue, float64(remain), scope, + ) + } + + rateLimitLimit := gh.GetRateLimitLimit() + for scope, limit := range rateLimitLimit { ch <- prometheus.MustNewConstMetric( - memoryGitHubRateLimit, prometheus.GaugeValue, float64(count), scope, + memoryGitHubRateLimitLimiting, prometheus.GaugeValue, float64(limit), scope, ) }