From 6592f5d7f390ff39cd34a3bc13d85abcbe97072c Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Mon, 6 May 2024 20:11:38 +0800 Subject: [PATCH] feat(tibuild): support build with tekton engine for pull requests (#138) Signed-off-by: wuhuizuo Signed-off-by: wuhuizuo --- .../pkg/rest/service/cloud_event_client.go | 89 ++++++++++++++----- .../rest/service/cloud_event_client_test.go | 2 +- tibuild/pkg/rest/service/dev_build_service.go | 26 +++++- tibuild/pkg/rest/service/gh_client.go | 34 ++++--- tibuild/pkg/rest/service/gh_client_test.go | 4 +- tibuild/pkg/rest/service/interface.go | 9 +- tibuild/pkg/rest/service/model.go | 8 +- 7 files changed, 127 insertions(+), 45 deletions(-) diff --git a/tibuild/pkg/rest/service/cloud_event_client.go b/tibuild/pkg/rest/service/cloud_event_client.go index d458d7e..cd23537 100644 --- a/tibuild/pkg/rest/service/cloud_event_client.go +++ b/tibuild/pkg/rest/service/cloud_event_client.go @@ -4,12 +4,18 @@ import ( "context" "fmt" "log/slog" + "strings" cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/cloudevents/sdk-go/v2/protocol" "github.com/google/go-github/v61/github" ) +const ( + ceTypeFakeGHPushDevBuild = "net.pingcap.tibuild.devbuild.push" + ceTypeFakeGHPRDevBuild = "net.pingcap.tibuild.devbuild.pull_request" +) + type BuildTrigger interface { TriggerDevBuild(ctx context.Context, dev DevBuild) error } @@ -31,7 +37,7 @@ type CloudEventClient struct { } func (s CloudEventClient) TriggerDevBuild(ctx context.Context, dev DevBuild) error { - event, err := NewDevBuildCloudEvent(dev) + event, err := newDevBuildCloudEvent(dev) if err != nil { return err } @@ -43,32 +49,71 @@ func (s CloudEventClient) TriggerDevBuild(ctx context.Context, dev DevBuild) err return nil } -func NewDevBuildCloudEvent(dev DevBuild) (*cloudevents.Event, error) { +func newDevBuildCloudEvent(dev DevBuild) (*cloudevents.Event, error) { + repo := GHRepoToStruct(dev.Spec.GithubRepo) + + var eventType string + var eventData interface{} + switch { + case strings.HasPrefix(dev.Spec.GitRef, "branch/"): + ref := strings.Replace(dev.Spec.GitRef, "branch/", "refs/heads/", 1) + eventType = ceTypeFakeGHPushDevBuild + eventData = newFakeGitHubPushEventPayload(repo.Owner, repo.Repo, ref, dev.Spec.GitHash) + case strings.HasPrefix(dev.Spec.GitRef, "pull/"): + eventType = ceTypeFakeGHPRDevBuild + eventData = newFakeGitHubPullRequestPayload(repo.Owner, repo.Repo, dev.Spec.prBaseRef, + dev.Spec.GitHash, dev.Spec.prNumber) + default: + return nil, fmt.Errorf("unkown git ref format") + } + event := cloudevents.NewEvent() - event.SetType(devbuild_ce_type) + event.SetType(eventType) + event.SetData(cloudevents.ApplicationJSON, eventData) event.SetSubject(fmt.Sprint(dev.ID)) - event.SetExtension("user", dev.Meta.CreatedBy) event.SetSource("tibuild.pingcap.net/api/devbuilds/" + fmt.Sprint(dev.ID)) - repo := GHRepoToStruct(dev.Spec.GithubRepo) + event.SetExtension("user", dev.Meta.CreatedBy) + + return &event, nil +} - if ref := GitRefToGHRef(dev.Spec.GitRef); ref != "" { - eventData := &github.PushEvent{ - Ref: github.String(ref), - After: github.String(dev.Spec.GitHash), - Before: github.String("00000000000000000000000000000000000000000"), - Repo: &github.PushEventRepository{ - Name: &repo.Repo, - CloneURL: github.String(repo.URL()), - Owner: &github.User{ - Login: &repo.Owner, - }, +func newFakeGitHubPushEventPayload(owner, repo, ref, sha string) *github.PushEvent { + return &github.PushEvent{ + Ref: github.String(ref), + After: github.String(sha), + Before: github.String("00000000000000000000000000000000000000000"), + Repo: &github.PushEventRepository{ + FullName: github.String(fmt.Sprintf("%s/%s", owner, repo)), + Name: github.String(repo), + CloneURL: github.String(fmt.Sprintf("https://github.com/%s/%s.git", owner, repo)), + Owner: &github.User{ + Login: github.String(owner), }, - } - event.SetData(cloudevents.ApplicationJSON, eventData) - return &event, nil - } else { - return nil, fmt.Errorf("unkown git ref format") + }, } } -const devbuild_ce_type = "net.pingcap.tibuild.devbuild.push" +func newFakeGitHubPullRequestPayload(owner, repo, baseRef, headSHA string, number int) *github.PullRequestEvent { + return &github.PullRequestEvent{ + Action: github.String("opened"), + Number: github.Int(number), + PullRequest: &github.PullRequest{ + Number: github.Int(number), + State: github.String("open"), + Head: &github.PullRequestBranch{ + SHA: github.String(headSHA), + }, + Base: &github.PullRequestBranch{ + Ref: github.String(baseRef), + }, + }, + Repo: &github.Repository{ + FullName: github.String(fmt.Sprintf("%s/%s", owner, repo)), + Name: github.String(repo), + CloneURL: github.String(fmt.Sprintf("https://github.com/%s/%s.git", owner, repo)), + Owner: &github.User{ + Login: github.String(owner), + }, + }, + } +} diff --git a/tibuild/pkg/rest/service/cloud_event_client_test.go b/tibuild/pkg/rest/service/cloud_event_client_test.go index 48c8542..3cb7f3f 100644 --- a/tibuild/pkg/rest/service/cloud_event_client_test.go +++ b/tibuild/pkg/rest/service/cloud_event_client_test.go @@ -28,7 +28,7 @@ func TestNewEvent(t *testing.T) { dev := sampleDevBuild() dev.Spec.GitHash = "754095a9f460dcf31f053045cfedfb00b9ad8e81" - ev, err := NewDevBuildCloudEvent(dev) + ev, err := newDevBuildCloudEvent(dev) require.NoError(t, err) js, err := json.Marshal(ev) require.NoError(t, err) diff --git a/tibuild/pkg/rest/service/dev_build_service.go b/tibuild/pkg/rest/service/dev_build_service.go index 1dd8b03..8ce6c7e 100644 --- a/tibuild/pkg/rest/service/dev_build_service.go +++ b/tibuild/pkg/rest/service/dev_build_service.go @@ -40,7 +40,7 @@ func (s DevbuildServer) Create(ctx context.Context, req DevBuild, option DevBuil if req.Meta.CreatedBy == "" { return nil, fmt.Errorf("unkown submitter%w", ErrAuth) } - err := fillGitHash(ctx, s.GHClient, &req) + err := fillDetailInfoForTekton(ctx, s.GHClient, &req) if err != nil { return nil, err } @@ -116,15 +116,35 @@ func validatePermission(ctx context.Context, req *DevBuild) error { return nil } -func fillGitHash(ctx context.Context, client GHClient, req *DevBuild) error { +func fillDetailInfoForTekton(ctx context.Context, client GHClient, req *DevBuild) error { + repo := GHRepoToStruct(req.Spec.GithubRepo) + if strings.HasPrefix(req.Spec.GitRef, "pull/") { + prNumber, err := strconv.ParseInt(strings.Replace(req.Spec.GitRef, "pull/", "", 1), 10, 32) + if err != nil { + return err + } + req.Spec.prNumber = int(prNumber) + + pr, err := client.GetPullRequestInfo(ctx, repo.Owner, repo.Repo, req.Spec.prNumber) + if err != nil { + return err + } + req.Spec.GitHash = pr.Head.GetSHA() + req.Spec.prBaseRef = pr.Base.GetRef() + + return nil + } + if req.Spec.GitHash != "" { return nil } - commit, err := client.GetHash(ctx, *GHRepoToStruct(req.Spec.GithubRepo), req.Spec.GitRef) + + commit, err := client.GetHash(ctx, repo.Owner, repo.Repo, req.Spec.GitRef) if err != nil { return fmt.Errorf("get hash from github failed%s%w", err.Error(), ErrServerRefuse) } req.Spec.GitHash = commit + return nil } diff --git a/tibuild/pkg/rest/service/gh_client.go b/tibuild/pkg/rest/service/gh_client.go index 38807e9..cfb7051 100644 --- a/tibuild/pkg/rest/service/gh_client.go +++ b/tibuild/pkg/rest/service/gh_client.go @@ -10,30 +10,31 @@ import ( "github.com/google/go-github/v61/github" ) -type GitHubClient struct{ *github.Client } +var _ GHClient = (*GitHubClient)(nil) +var sha1regex *regexp.Regexp = regexp.MustCompile(`^[0-9a-fA-F]{40}$`) -func GitRefToGHRef(ref string) string { - if branchName, found := strings.CutPrefix(ref, "branch/"); found { - return fmt.Sprintf("refs/heads/%s", branchName) - } - return "" -} +type GitHubClient struct{ *github.Client } -func (c GitHubClient) GetHash(ctx context.Context, repo GithubRepo, ref string) (string, error) { +func (c GitHubClient) GetHash(ctx context.Context, owner, repo, ref string) (string, error) { if sha1regex.MatchString(ref) { return ref, nil } - gref := GitRefToGHRef(ref) + gref := convertParamGitRefToGitHubRef(ref) if gref == "" { return "", fmt.Errorf("bad git ref:%s", ref) } - rt, _, err := c.Git.GetRef(ctx, repo.Owner, repo.Repo, gref) + rt, _, err := c.Git.GetRef(ctx, owner, repo, gref) if err != nil { return "", err } return *rt.Object.SHA, nil } +func (c GitHubClient) GetPullRequestInfo(ctx context.Context, owner, repo string, prNum int) (*github.PullRequest, error) { + pr, _, err := c.PullRequests.Get(ctx, owner, repo, prNum) + return pr, err +} + func NewGHClient(token string) GHClient { client := github.NewClient(http.DefaultClient) if token != "" { @@ -42,4 +43,15 @@ func NewGHClient(token string) GHClient { return GitHubClient{client} } -var sha1regex *regexp.Regexp = regexp.MustCompile(`^[0-9a-fA-F]{40}$`) +func convertParamGitRefToGitHubRef(ref string) string { + switch { + case strings.HasPrefix(ref, "branch/"): + return strings.Replace(ref, "branch/", "refs/heads/", 1) + case strings.HasPrefix(ref, "tag/"): + return strings.Replace(ref, "tag/", "refs/tags/", 1) + case strings.HasPrefix(ref, "pull/"), strings.HasPrefix(ref, "pr/"): + return strings.Join([]string{strings.Replace(ref, "pull/", "refs/pulls/", 1), "head"}, "/") + default: + return "" + } +} diff --git a/tibuild/pkg/rest/service/gh_client_test.go b/tibuild/pkg/rest/service/gh_client_test.go index 3305139..3dff257 100644 --- a/tibuild/pkg/rest/service/gh_client_test.go +++ b/tibuild/pkg/rest/service/gh_client_test.go @@ -13,14 +13,14 @@ func TestGetHash(t *testing.T) { t.Skip("Skipping send event") } token := os.Getenv("GHTOKEN") - hash, err := NewGHClient(token).GetHash(context.TODO(), RepoPd, "branch/master") + hash, err := NewGHClient(token).GetHash(context.TODO(), RepoPd.Owner, RepoPd.Repo, "branch/master") require.NoError(t, err) require.NotEmpty(t, hash) } func TestGetHashSha1(t *testing.T) { s := "754095a9f460dcf31f053045cfedfb00b9ad8e81" - hash, err := NewGHClient("").GetHash(context.TODO(), RepoPd, s) + hash, err := NewGHClient("").GetHash(context.TODO(), RepoPd.Owner, RepoPd.Repo, s) require.NoError(t, err) require.Equal(t, s, hash) } diff --git a/tibuild/pkg/rest/service/interface.go b/tibuild/pkg/rest/service/interface.go index b4555e3..670d2ed 100644 --- a/tibuild/pkg/rest/service/interface.go +++ b/tibuild/pkg/rest/service/interface.go @@ -1,6 +1,10 @@ package service -import "context" +import ( + "context" + + "github.com/google/go-github/v61/github" +) type HotfixService interface { CreateBranch(ctx context.Context, req BranchCreateReq) (resp *BranchCreateResp, err error) @@ -21,5 +25,6 @@ type ArtifactHelperService interface { } type GHClient interface { - GetHash(ctx context.Context, repo GithubRepo, ref string) (string, error) + GetHash(ctx context.Context, owner, repo, ref string) (string, error) + GetPullRequestInfo(ctx context.Context, owner, repo string, prNum int) (*github.PullRequest, error) } diff --git a/tibuild/pkg/rest/service/model.go b/tibuild/pkg/rest/service/model.go index 2fde0b2..e1a88db 100644 --- a/tibuild/pkg/rest/service/model.go +++ b/tibuild/pkg/rest/service/model.go @@ -110,9 +110,7 @@ func StringToProduct(s string) Product { func ProdToRepo(prod Product) *GithubRepo { switch prod { - case ProductBr, ProductTidbLightning, ProductDumpling: - fallthrough - case ProductTidb: + case ProductBr, ProductTidbLightning, ProductDumpling, ProductTidb: return &RepoTidb case ProductTikv: return &RepoTikv @@ -167,6 +165,7 @@ type DevBuildSaveOption struct { type DevBuildSpec struct { Product Product `json:"product"` GitRef string `json:"gitRef"` + GitHash string `json:"gitHash,omitempty" gorm:"type:varchar(64)"` Version string `json:"version"` Edition ProductEdition `json:"edition"` PluginGitRef string `json:"pluginGitRef,omitempty"` @@ -180,7 +179,8 @@ type DevBuildSpec struct { IsHotfix bool `json:"isHotfix,omitempty"` TargetImg string `json:"targetImg,omitempty" gorm:"type:varchar(128)"` PipelineEngine PipelineEngine `json:"pipelineEngine,omitempty" gorm:"type:varchar(16)"` - GitHash string `json:"gitHash,omitempty" gorm:"type:varchar(64)"` + prNumber int + prBaseRef string } type PipelineEngine string