Skip to content

Commit

Permalink
Merge pull request #137 from whywaita/fix/121
Browse files Browse the repository at this point in the history
Add validate in modify target
  • Loading branch information
whywaita authored Apr 12, 2022
2 parents e864147 + 946925b commit b0f514d
Show file tree
Hide file tree
Showing 10 changed files with 304 additions and 148 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/ory/dockertest/v3 v3.8.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/prometheus/client_golang v1.11.0
github.com/r3labs/diff/v2 v2.15.1 // indirect
github.com/satori/go.uuid v1.2.0
goji.io v2.0.2+incompatible
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg=
github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
Expand All @@ -222,12 +224,15 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
Expand Down Expand Up @@ -323,6 +328,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
Expand Down
2 changes: 1 addition & 1 deletion pkg/datastore/mysql/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func (m *MySQL) EnqueueJob(ctx context.Context, job datastore.Job) error {
// ListJobs get all jobs
func (m *MySQL) ListJobs(ctx context.Context) ([]datastore.Job, error) {
var jobs []datastore.Job
query := `SELECT uuid, ghe_domain, repository, check_event, target_id FROM jobs`
query := `SELECT uuid, ghe_domain, repository, check_event, target_id, created_at, updated_at FROM jobs`
if err := m.Conn.SelectContext(ctx, &jobs, query); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, datastore.ErrNotFound
Expand Down
6 changes: 3 additions & 3 deletions pkg/datastore/mysql/job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,9 @@ func TestMySQL_ListJobs(t *testing.T) {
if len(test.want) != len(got) {
t.Fatalf("incorrect length jobs, want: %d but got: %d", len(test.want), len(got))
}
for _, g := range got {
g.CreatedAt = time.Time{}
g.UpdatedAt = time.Time{}
for i := range got {
got[i].CreatedAt = time.Time{}
got[i].UpdatedAt = time.Time{}
}

if diff := cmp.Diff(test.want, got); diff != "" {
Expand Down
96 changes: 18 additions & 78 deletions pkg/gh/github.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package gh

import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
Expand All @@ -15,7 +13,6 @@ import (
"github.com/google/go-github/v35/github"
"github.com/m4ns0ur/httpcache"
"github.com/patrickmn/go-cache"
"github.com/whywaita/myshoes/pkg/logger"
"golang.org/x/oauth2"
)

Expand Down Expand Up @@ -138,6 +135,23 @@ func CheckSignature(installationID int64) error {
return nil
}

// ExistRunnerReleases check exist of runner file
func ExistRunnerReleases(runnerVersion string) error {
releasesURL := fmt.Sprintf("https://github.com/actions/runner/releases/tag/%s", runnerVersion)
resp, err := http.Get(releasesURL)
if err != nil {
return fmt.Errorf("failed to GET from %s: %w", releasesURL, ErrNotFound)
}

if resp.StatusCode == http.StatusOK {
return nil
} else if resp.StatusCode == http.StatusNotFound {
return ErrNotFound
}

return fmt.Errorf("invalid response code (%d)", resp.StatusCode)
}

// ExistGitHubRepository check exist of GitHub repository
func ExistGitHubRepository(scope, gheDomain string, accessToken string) error {
repoURL, err := getRepositoryURL(scope, gheDomain)
Expand All @@ -160,86 +174,12 @@ func ExistGitHubRepository(scope, gheDomain string, accessToken string) error {
if resp.StatusCode == http.StatusOK {
return nil
} else if resp.StatusCode == http.StatusNotFound {
return errors.New("not found")
return ErrNotFound
}

return fmt.Errorf("invalid response code (%d)", resp.StatusCode)
}

// ExistGitHubRunner check exist registered of GitHub runner
func ExistGitHubRunner(ctx context.Context, client *github.Client, owner, repo, runnerName string) (*github.Runner, error) {
runners, err := ListRunners(ctx, client, owner, repo)
if err != nil {
return nil, fmt.Errorf("failed to get list of runners: %w", err)
}

return ExistGitHubRunnerWithRunner(runners, runnerName)
}

// ExistGitHubRunnerWithRunner check exist registered of GitHub runner from a list of runner
func ExistGitHubRunnerWithRunner(runners []*github.Runner, runnerName string) (*github.Runner, error) {
for _, r := range runners {
if strings.EqualFold(r.GetName(), runnerName) {
return r, nil
}
}

return nil, ErrNotFound
}

// ListRunners get runners that registered repository or org
func ListRunners(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) {
if cachedRs, found := responseCache.Get(getCacheKey(owner, repo)); found {
return cachedRs.([]*github.Runner), nil
}

var opts = &github.ListOptions{
Page: 0,
PerPage: 100,
}

var rs []*github.Runner
for {
logger.Logf(true, "get runners from GitHub, page: %d, now all runners: %d", opts.Page, len(rs))
runners, resp, err := listRunners(ctx, client, owner, repo, opts)
if err != nil {
return nil, fmt.Errorf("failed to list runners: %w", err)
}
storeRateLimit(getRateLimitKey(owner, repo), resp.Rate)

rs = append(rs, runners.Runners...)
if resp.NextPage == 0 {
break
}
opts.Page = resp.NextPage
}

responseCache.Set(getCacheKey(owner, repo), rs, 1*time.Second)
logger.Logf(true, "found %d runners in GitHub", len(rs))

return rs, nil
}

func getCacheKey(owner, repo string) string {
return fmt.Sprintf("owner-%s-repo-%s", owner, repo)
}

func listRunners(ctx context.Context, client *github.Client, owner, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) {
if repo == "" {
runners, resp, err := client.Actions.ListOrganizationRunners(ctx, owner, opts)
if err != nil {
return nil, nil, fmt.Errorf("failed to list organization runners: %w", err)
}
return runners, resp, nil
}

runners, resp, err := client.Actions.ListRunners(ctx, owner, repo, opts)
if err != nil {
return nil, nil, fmt.Errorf("failed to list repository runners: %w", err)
}
return runners, resp, nil
}

func getRepositoryURL(scope, gheDomain string) (string, error) {
// github.com
// => https://api.github.com/repos/:owner/:repo
Expand Down
85 changes: 85 additions & 0 deletions pkg/gh/runner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package gh

import (
"context"
"fmt"
"strings"
"time"

"github.com/google/go-github/v35/github"
"github.com/whywaita/myshoes/pkg/logger"
)

// ExistGitHubRunner check exist registered of GitHub runner
func ExistGitHubRunner(ctx context.Context, client *github.Client, owner, repo, runnerName string) (*github.Runner, error) {
runners, err := ListRunners(ctx, client, owner, repo)
if err != nil {
return nil, fmt.Errorf("failed to get list of runners: %w", err)
}

return ExistGitHubRunnerWithRunner(runners, runnerName)
}

// ExistGitHubRunnerWithRunner check exist registered of GitHub runner from a list of runner
func ExistGitHubRunnerWithRunner(runners []*github.Runner, runnerName string) (*github.Runner, error) {
for _, r := range runners {
if strings.EqualFold(r.GetName(), runnerName) {
return r, nil
}
}

return nil, ErrNotFound
}

// ListRunners get runners that registered repository or org
func ListRunners(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) {
if cachedRs, found := responseCache.Get(getCacheKey(owner, repo)); found {
return cachedRs.([]*github.Runner), nil
}

var opts = &github.ListOptions{
Page: 0,
PerPage: 100,
}

var rs []*github.Runner
for {
logger.Logf(true, "get runners from GitHub, page: %d, now all runners: %d", opts.Page, len(rs))
runners, resp, err := listRunners(ctx, client, owner, repo, opts)
if err != nil {
return nil, fmt.Errorf("failed to list runners: %w", err)
}
storeRateLimit(getRateLimitKey(owner, repo), resp.Rate)

rs = append(rs, runners.Runners...)
if resp.NextPage == 0 {
break
}
opts.Page = resp.NextPage
}

responseCache.Set(getCacheKey(owner, repo), rs, 1*time.Second)
logger.Logf(true, "found %d runners in GitHub", len(rs))

return rs, nil
}

func getCacheKey(owner, repo string) string {
return fmt.Sprintf("owner-%s-repo-%s", owner, repo)
}

func listRunners(ctx context.Context, client *github.Client, owner, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) {
if repo == "" {
runners, resp, err := client.Actions.ListOrganizationRunners(ctx, owner, opts)
if err != nil {
return nil, nil, fmt.Errorf("failed to list organization runners: %w", err)
}
return runners, resp, nil
}

runners, resp, err := client.Actions.ListRunners(ctx, owner, repo, opts)
if err != nil {
return nil, nil, fmt.Errorf("failed to list repository runners: %w", err)
}
return runners, resp, nil
}
16 changes: 16 additions & 0 deletions pkg/metric/scrape_datastore.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package metric
import (
"context"
"fmt"
"sort"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/whywaita/myshoes/pkg/datastore"
Expand All @@ -21,6 +23,11 @@ var (
"Number of targets",
[]string{"resource_type"}, nil,
)
datastoreJobDurationOldest = prometheus.NewDesc(
prometheus.BuildFQName(namespace, datastoreName, "job_duration_oldest_seconds"),
"Duration time of oldest job",
[]string{"job_id"}, nil,
)
)

// ScraperDatastore is scraper implement for datastore.Datastore
Expand Down Expand Up @@ -61,6 +68,15 @@ func scrapeJobs(ctx context.Context, ds datastore.Datastore, ch chan<- prometheu
return nil
}

sort.SliceStable(jobs, func(i, j int) bool {
// oldest job is first
return jobs[i].CreatedAt.Before(jobs[j].CreatedAt)
})

oldestJob := jobs[0]
ch <- prometheus.MustNewConstMetric(
datastoreJobDurationOldest, prometheus.GaugeValue, time.Since(oldestJob.CreatedAt).Seconds(), oldestJob.UUID.String())

result := map[string]float64{} // key: target_id, value: number
for _, j := range jobs {
result[j.TargetID.String()]++
Expand Down
Loading

0 comments on commit b0f514d

Please sign in to comment.