Skip to content

Commit

Permalink
feat(tibuild): send cloudevents whtn trigger builds
Browse files Browse the repository at this point in the history
Signed-off-by: wuhuizuo <wuhuizuo@126.com>
  • Loading branch information
wuhuizuo committed Dec 9, 2023
1 parent 96b217b commit ac24749
Show file tree
Hide file tree
Showing 5 changed files with 147 additions and 71 deletions.
7 changes: 6 additions & 1 deletion tibuild/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ go 1.21

require (
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/cloudevents/sdk-go/v2 v2.14.0
github.com/gin-contrib/requestid v0.0.6
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.9.1
github.com/google/go-github/v57 v57.0.0
github.com/jinzhu/configor v1.2.2
github.com/stretchr/testify v1.8.4
github.com/swaggo/files v1.0.1
Expand Down Expand Up @@ -34,7 +36,7 @@ require (
github.com/go-playground/validator/v10 v10.14.0 // indirect
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
Expand All @@ -50,6 +52,9 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.10.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/sys v0.8.0 // indirect
Expand Down
23 changes: 21 additions & 2 deletions tibuild/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
github.com/cloudevents/sdk-go/v2 v2.14.0 h1:Nrob4FwVgi5L4tV9lhjzZcjYqFVyJzsA56CwPaPfv6s=
github.com/cloudevents/sdk-go/v2 v2.14.0/go.mod h1:xDmKfzNjM8gBvjaF8ijFjM1VYOVUEeUfapHMUX1T5To=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -72,9 +74,14 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs=
github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
Expand Down Expand Up @@ -132,6 +139,8 @@ github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZO
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
Expand Down Expand Up @@ -172,8 +181,16 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
Expand Down Expand Up @@ -230,6 +247,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down
45 changes: 45 additions & 0 deletions tibuild/internal/controller/events.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package controller

import (
"context"
"fmt"

cloudevents "github.com/cloudevents/sdk-go/v2"
"github.com/google/go-github/v57/github"
)

func (pt *PipelineTriggerStruct) NewCloudEvent(buildID int) *cloudevents.Event {
/*
ArtifactType string `form:"artifact_type" json:"artifact_type" validate:"required"`
Branch string `form:"branch" json:"branch" validate:"required"`
Component string `form:"component" json:"component" validate:"required"`
PipelineId int `form:"pipeline_id" json:"pipeline_id" validate:"required,numeric"`
Version string `form:"version" json:"version" validate:"required,startswith=v"`
TriggeredBy string `form:"triggered_by" json:"triggered_by" validate:"required"`
PushGCR string `form:"push_gcr" json:"push_gcr" validate:"required"`
*/
event := cloudevents.NewEvent()
// TODO: fill them
event.SetSubject(fmt.Sprint(buildID))
event.SetExtension("user", pt.TriggeredBy)
event.SetSource("https://tibuild.pingcap/com/dev-build/")
eventData := &github.PushEvent{
Ref: github.String(fmt.Sprintf("refs/heads/%s", pt.Branch)),
After: github.String(pt.Branch),
Repo: &github.PushEventRepository{
Name: &pt.Component,
CloneURL: github.String(fmt.Sprintf("https://github.com/tikv/%s", pt.Component)),
Owner: &github.User{
Login: github.String("pingcap"),
},
},
}
event.SetData(cloudevents.ApplicationJSON, eventData)

return &event
}

func sendEventsForDevBuild(sinkURL string, event cloudevents.Event) cloudevents.Result {
client, _ := cloudevents.NewClientHTTP(cloudevents.WithTarget(sinkURL))
return client.Send(context.Background(), event)
}
139 changes: 73 additions & 66 deletions tibuild/internal/controller/pipeline_trigger.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,72 @@ type PipelineTriggerStruct struct {
PushGCR string `form:"push_gcr" json:"push_gcr" validate:"required"`
}

func (pt *PipelineTriggerStruct) BuildParams() map[string]string {
ret := make(map[string]string)

switch pt.PipelineId {
case 1, 2, 3, 4, 5, 6:
fmt.Println("触发构建的是多分支流水线,没有传入参数")
case 7:
fmt.Println("Tab展示名:Nightly Image Build For QA")
if pt.Branch == "master" {
ret["GIT_BRANCH"] = "master"
ret["NEED_MULTIARCH"] = "true"
} else {
ret["GIT_BRANCH"] = pt.Branch
ret["NEED_MULTIARCH"] = "false"
}
ret["FORCE_REBUILD"] = "false"

case 8:
fmt.Println("Tab展示名:Nightly Image Build to Dockerhub")

case 9:
fmt.Println("Tab展示名:Nightly TiUP Build")
case 10:
ret["RELEASE_BRANCH"] = pt.Branch
ret["RELEASE_TAG"] = pt.Version
case 11:
ret["RELEASE_TAG"] = pt.Version
ret["RELEASE_BRANCH"] = pt.Branch
ret["NEED_MULTIARCH"] = "true"
ret["DEBUG_MODE"] = "false"
case 12: // dev-build
ret["PRODUCT"] = pt.Component
if pt.Component == "ticdc" || pt.Component == "dm" {
ret["REPO"] = "tiflow"
} else if pt.Component == "tidb" || pt.Component == "dumpling" || pt.Component == "lightning" || pt.Component == "br" {
ret["REPO"] = "tidb"
} else {
ret["REPO"] = pt.Component
}
ret["HOTFIX_TAG"] = pt.Version
if pt.PushGCR == "Yes" {
ret["PUSH_GCR"] = "true"
} else {
ret["PUSH_GCR"] = "false"
}
if pt.ArtifactType == "enterprise image" {
ret["PUSH_DOCKER_HUB"] = "false"
ret["EDITION"] = "enterprise"
} else {
ret["PUSH_DOCKER_HUB"] = "true"
ret["EDITION"] = "community"
}
ret["FORCE_REBUILD"] = "true"
ret["DEBUG"] = "false"
if pt.Arch == "All" {
ret["ARCH"] = "both"
} else if pt.Arch == "linux-amd64" {
ret["ARCH"] = "amd64"
} else {
ret["ARCH"] = "arm64"
}
}

return ret
}

type TibuildInfo struct {
PipelineId int `form:"pipeline_id"`
BuildType string `form:"build_type"`
Expand Down Expand Up @@ -123,11 +189,9 @@ func PipelineTrigger(c *gin.Context) {
fmt.Println("pipeline_build_id : ", ps.PipelineBuildId)
fmt.Println("begin_time: ", ps.BeginTime)

params_trans := make(map[string]string)
params_trans["PIPELINE_BUILD_ID"] = strconv.Itoa(ps.PipelineBuildId)
log.Println(params_trans)

go triggerJenkinsJob(ctx, &tibuildInfo[0], &params, params_trans, jenkins, int64(ps.PipelineBuildId))
go triggerJenkinsJob(ctx, ps.PipelineBuildId, &tibuildInfo[0], &params, jenkins)
result := sendEventsForDevBuild("https://cloudevents-server.apps.svc/events", *params.NewCloudEvent(ps.PipelineBuildId))
log.Println(result.Error())

c.JSON(http.StatusOK, gin.H{
"code": 200,
Expand All @@ -136,67 +200,10 @@ func PipelineTrigger(c *gin.Context) {
})
}

func triggerJenkinsJob(ctx context.Context, tibuildInfo *TibuildInfo, params *PipelineTriggerStruct, params_trans map[string]string, jenkins *gojenkins.Jenkins, pipelineBuildID int64) {
switch params.PipelineId {
case 1, 2, 3, 4, 5, 6:
fmt.Println("触发构建的是多分支流水线,没有传入参数")
case 7:
fmt.Println("Tab展示名:Nightly Image Build For QA")
if params.Branch == "master" {
params_trans["GIT_BRANCH"] = "master"
params_trans["NEED_MULTIARCH"] = "true"
} else {
params_trans["GIT_BRANCH"] = params.Branch
params_trans["NEED_MULTIARCH"] = "false"
}
params_trans["FORCE_REBUILD"] = "false"

case 8:
fmt.Println("Tab展示名:Nightly Image Build to Dockerhub")

case 9:
fmt.Println("Tab展示名:Nightly TiUP Build")
case 10:
params_trans["RELEASE_BRANCH"] = params.Branch
params_trans["RELEASE_TAG"] = params.Version
case 11:
params_trans["RELEASE_TAG"] = params.Version
params_trans["RELEASE_BRANCH"] = params.Branch
params_trans["NEED_MULTIARCH"] = "true"
params_trans["DEBUG_MODE"] = "false"
case 12: // dev-build
params_trans["PRODUCT"] = params.Component
if params.Component == "ticdc" || params.Component == "dm" {
params_trans["REPO"] = "tiflow"
} else if params.Component == "tidb" || params.Component == "dumpling" || params.Component == "lightning" || params.Component == "br" {
params_trans["REPO"] = "tidb"
} else {
params_trans["REPO"] = params.Component
}
params_trans["HOTFIX_TAG"] = params.Version
if params.PushGCR == "Yes" {
params_trans["PUSH_GCR"] = "true"
} else {
params_trans["PUSH_GCR"] = "false"
}
if params.ArtifactType == "enterprise image" {
params_trans["PUSH_DOCKER_HUB"] = "false"
params_trans["EDITION"] = "enterprise"
} else {
params_trans["PUSH_DOCKER_HUB"] = "true"
params_trans["EDITION"] = "community"
}
params_trans["FORCE_REBUILD"] = "true"
params_trans["DEBUG"] = "false"
if params.Arch == "All" {
params_trans["ARCH"] = "both"
} else if params.Arch == "linux-amd64" {
params_trans["ARCH"] = "amd64"
} else {
params_trans["ARCH"] = "arm64"
}

}
func triggerJenkinsJob(ctx context.Context, pipelineBuildID int, tibuildInfo *TibuildInfo, params *PipelineTriggerStruct, jenkins *gojenkins.Jenkins) {
params_trans := params.BuildParams()
params_trans["PIPELINE_BUILD_ID"] = strconv.Itoa(pipelineBuildID)
log.Println(params_trans)

job_name_tmp := strings.Split(tibuildInfo.Pipeline, "/job/")
if len(job_name_tmp) <= 1 {
Expand Down
4 changes: 2 additions & 2 deletions tibuild/internal/service/job_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"github.com/PingCAP-QE/ee-apps/tibuild/internal/entity"
)

func Job_Build(jenkins *gojenkins.Jenkins, ctx context.Context, jobname string, params map[string]string, pipeline_build_id int64) {
func Job_Build(jenkins *gojenkins.Jenkins, ctx context.Context, jobname string, params map[string]string, pipeline_build_id int) {
jobarr := strings.Split(jobname, "/")
newjobname := strings.TrimRight(strings.Join(jobarr, "/job/"), "/job/")
println("*************** 输入参数是 **************")
Expand All @@ -39,7 +39,7 @@ func Job_Build(jenkins *gojenkins.Jenkins, ctx context.Context, jobname string,
jenkins_log = "https://cd.pingcap.net/blue/organizations/jenkins/" + jobarr[0] + "/detail/" + jobarr[0] + "/" + strconv.FormatInt(build.GetBuildNumber(), 10) + "/pipeline"
}

err = database.DBConn.DB.Model(new(entity.PipelinesListShow)).Where("pipeline_build_id = ?", int(pipeline_build_id)).Update("jenkins_log", jenkins_log).Error
err = database.DBConn.DB.Model(new(entity.PipelinesListShow)).Where("pipeline_build_id = ?", pipeline_build_id).Update("jenkins_log", jenkins_log).Error

if err != nil {
panic(err)
Expand Down

0 comments on commit ac24749

Please sign in to comment.