From 9209601f410ce05eea4146c2340d780eda62cf59 Mon Sep 17 00:00:00 2001 From: whywaita Date: Thu, 26 Aug 2021 14:15:21 +0900 Subject: [PATCH 1/3] add strict parameter, using #84 mode if strict is true --- internal/config/config.go | 6 ++++-- internal/config/init.go | 7 +++++++ pkg/starter/starter.go | 24 ++++++++++++++---------- pkg/web/config.go | 23 +++++++++++++++++++++-- pkg/web/http.go | 4 ++++ 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 17df61d..63a0f6a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,9 +16,10 @@ type conf struct { MySQLDSN string Port int ShoesPluginPath string + RunnerUser string - RunnerUser string - Debug bool + Debug bool + Strict bool // check to registered runner before delete job } // Config Environment keys @@ -30,4 +31,5 @@ const ( EnvPort = "PORT" EnvShoesPluginPath = "PLUGIN" EnvDebug = "DEBUG" + EnvStrict = "STRICT" ) diff --git a/internal/config/init.go b/internal/config/init.go index dab83c3..6d3cdee 100644 --- a/internal/config/init.go +++ b/internal/config/init.go @@ -85,6 +85,13 @@ func Load() { } else { Config.Debug = false } + + strict := os.Getenv(EnvStrict) + if strict == "true" { + Config.Strict = true + } else { + Config.Strict = false + } } func checkBinary(p string) (string, error) { diff --git a/pkg/starter/starter.go b/pkg/starter/starter.go index 38a2d1a..bce3b8c 100644 --- a/pkg/starter/starter.go +++ b/pkg/starter/starter.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/whywaita/myshoes/internal/config" + uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" @@ -107,20 +109,22 @@ func (s *Starter) do(ctx context.Context) error { return } - if err := s.checkRegisteredRunner(ctx, cloudID, *target); err != nil { - logger.Logf(false, "failed to check to register runner (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + if config.Config.Strict { + if err := s.checkRegisteredRunner(ctx, cloudID, *target); err != nil { + logger.Logf(false, "failed to check to register runner (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) - if err := deleteInstance(ctx, cloudID); err != nil { - logger.Logf(false, "failed to delete an instance that not registered instance (target ID: %s, cloud ID: %s): %+v\n", job.TargetID, cloudID, err) - // not return, need to update target status if err. - } + if err := deleteInstance(ctx, cloudID); err != nil { + logger.Logf(false, "failed to delete an instance that not registered instance (target ID: %s, cloud ID: %s): %+v\n", job.TargetID, cloudID, err) + // not return, need to update target status if err. + } + + if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("cannot register runner to GitHub (job ID: %s)", job.UUID)); err != nil { + logger.Logf(false, "failed to update target status (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) + return + } - if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("cannot register runner to GitHub (job ID: %s)", job.UUID)); err != nil { - logger.Logf(false, "failed to update target status (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) return } - - return } r := datastore.Runner{ diff --git a/pkg/web/config.go b/pkg/web/config.go index 634cd8f..72c0851 100644 --- a/pkg/web/config.go +++ b/pkg/web/config.go @@ -8,12 +8,16 @@ import ( "github.com/whywaita/myshoes/pkg/logger" ) -type inputConfig struct { +type inputConfigDebug struct { Debug bool `json:"debug"` } +type inputConfigStrict struct { + Strict bool `json:"strict"` +} + func handleConfigDebug(w http.ResponseWriter, r *http.Request) { - i := inputConfig{} + i := inputConfigDebug{} if err := json.NewDecoder(r.Body).Decode(&i); err != nil { logger.Logf(false, "failed to decode request body: %+v", err) @@ -26,3 +30,18 @@ func handleConfigDebug(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) return } + +func handleConfigStrict(w http.ResponseWriter, r *http.Request) { + i := inputConfigStrict{} + + if err := json.NewDecoder(r.Body).Decode(&i); err != nil { + logger.Logf(false, "failed to decode request body: %+v", err) + outputErrorMsg(w, http.StatusBadRequest, "json decode error") + return + } + + config.Config.Strict = i.Strict + logger.Logf(false, "switch strict mode to %t", i.Strict) + w.WriteHeader(http.StatusNoContent) + return +} diff --git a/pkg/web/http.go b/pkg/web/http.go index 726cebb..c32699e 100644 --- a/pkg/web/http.go +++ b/pkg/web/http.go @@ -68,6 +68,10 @@ func NewMux(ds datastore.Datastore) *goji.Mux { apacheLogging(r) handleConfigDebug(w, r) }) + mux.HandleFunc(pat.Post("/config/strict"), func(w http.ResponseWriter, r *http.Request) { + apacheLogging(r) + handleConfigStrict(w, r) + }) return mux } From 53d5b3d16b2b8696a54a1f0b5b2a2e98fe6f0bdb Mon Sep 17 00:00:00 2001 From: whywaita Date: Thu, 26 Aug 2021 16:02:00 +0900 Subject: [PATCH 2/3] cache API response --- go.mod | 1 + go.sum | 2 ++ pkg/gh/github.go | 21 ++++++++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2a39c3d..2eab575 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/lestrrat-go/jwx v1.2.0 // indirect github.com/ory/dockertest/v3 v3.6.2 + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/prometheus/client_golang v1.10.0 // indirect github.com/satori/go.uuid v1.2.0 goji.io v2.0.2+incompatible diff --git a/go.sum b/go.sum index 42a1dc3..af2ccc0 100644 --- a/go.sum +++ b/go.sum @@ -416,6 +416,8 @@ github.com/pact-foundation/pact-go v1.0.4 h1:OYkFijGHoZAYbOIb1LWXrwKQbMMRUv1oQ89 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/performancecopilot/speed v3.0.0+incompatible h1:2WnRzIquHa5QxaJKShDkLM+sc0JPuwhXzK8OYOyt3Vg= diff --git a/pkg/gh/github.go b/pkg/gh/github.go index 0fffa55..1da5733 100644 --- a/pkg/gh/github.go +++ b/pkg/gh/github.go @@ -8,6 +8,9 @@ import ( "net/url" "path" "strings" + "time" + + "github.com/patrickmn/go-cache" "github.com/bradleyfalzon/ghinstallation" "github.com/google/go-github/v35/github" @@ -19,8 +22,16 @@ import ( var ( // ErrNotFound is error for not found ErrNotFound = fmt.Errorf("not found") + + // ResponseCache is cache variable + responseCache *cache.Cache ) +func init() { + c := cache.New(5*time.Minute, 10*time.Minute) + responseCache = c +} + // NewClient create a client of GitHub func NewClient(ctx context.Context, personalToken, gheDomain string) (*github.Client, error) { ts := oauth2.StaticTokenSource( @@ -96,13 +107,16 @@ func ExistGitHubRunner(ctx context.Context, client *github.Client, owner, repo, // ListRunners get runners that registered repository or org func ListRunners(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) { - var rs []*github.Runner + if cachedRs, found := responseCache.Get(getCacheKey(owner, repo)); found { + return cachedRs.([]*github.Runner), nil + } var opts = &github.ListOptions{ Page: 0, PerPage: 10, } + var rs []*github.Runner for { logger.Logf(true, "get runners from GitHub, page: %d, now all runners: %d", opts.Page, len(rs)) runners, resp, err := listRunners(ctx, client, owner, repo, opts) @@ -117,11 +131,16 @@ func ListRunners(ctx context.Context, client *github.Client, owner, repo string) opts.Page = resp.NextPage } + responseCache.Set(getCacheKey(owner, repo), rs, 1*time.Second) logger.Logf(true, "found %d runners", len(rs)) return rs, nil } +func getCacheKey(owner, repo string) string { + return fmt.Sprintf("owner-%s-repo-%s", owner, repo) +} + func listRunners(ctx context.Context, client *github.Client, owner, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) { if repo == "" { runners, resp, err := client.Actions.ListOrganizationRunners(ctx, owner, opts) From cfa1453fca1e859cc30c2f78cc5816a3e3134ae3 Mon Sep 17 00:00:00 2001 From: whywaita Date: Thu, 26 Aug 2021 19:22:23 +0900 Subject: [PATCH 3/3] strict mode is true in default --- internal/config/init.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/config/init.go b/internal/config/init.go index 6d3cdee..8964404 100644 --- a/internal/config/init.go +++ b/internal/config/init.go @@ -86,10 +86,8 @@ func Load() { Config.Debug = false } - strict := os.Getenv(EnvStrict) - if strict == "true" { - Config.Strict = true - } else { + Config.Strict = true + if os.Getenv(EnvStrict) == "false" { Config.Strict = false } }