From ed099a3c76bc4cea69f470023dc0add93bfe19b8 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Thu, 17 Oct 2024 17:30:17 +0800 Subject: [PATCH] feat(publisher): update publish state with redis (#183) Signed-off-by: wuhuizuo Signed-off-by: wuhuizuo --- publisher/cmd/publisher/main.go | 20 ++++---- publisher/cmd/worker/main.go | 2 +- publisher/design/design.go | 4 +- publisher/gen/http/cli/publisher/cli.go | 18 +++---- publisher/gen/http/openapi.json | 2 +- publisher/gen/http/openapi.yaml | 23 +++++---- publisher/gen/http/openapi3.json | 2 +- publisher/gen/http/openapi3.yaml | 50 ++++++++++--------- publisher/gen/http/tiup/client/cli.go | 2 +- .../gen/http/tiup/client/encode_decode.go | 7 +++ publisher/pkg/{ => impl}/tiup/funcs.go | 0 publisher/pkg/{ => impl}/tiup/funcs_test.go | 19 +++++++ publisher/pkg/{ => impl}/tiup/publisher.go | 23 ++++++--- .../{tiup.go => pkg/impl/tiup/service.go} | 19 +++---- publisher/pkg/{ => impl}/tiup/types.go | 10 ++++ 15 files changed, 127 insertions(+), 74 deletions(-) rename publisher/pkg/{ => impl}/tiup/funcs.go (100%) rename publisher/pkg/{ => impl}/tiup/funcs_test.go (95%) rename publisher/pkg/{ => impl}/tiup/publisher.go (83%) rename publisher/{tiup.go => pkg/impl/tiup/service.go} (86%) rename publisher/pkg/{ => impl}/tiup/types.go (80%) diff --git a/publisher/cmd/publisher/main.go b/publisher/cmd/publisher/main.go index cd3b8a7..c4aab12 100644 --- a/publisher/cmd/publisher/main.go +++ b/publisher/cmd/publisher/main.go @@ -18,9 +18,9 @@ import ( "goa.design/clue/log" "gopkg.in/yaml.v3" - "github.com/PingCAP-QE/ee-apps/publisher" - tiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup" + gentiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup" "github.com/PingCAP-QE/ee-apps/publisher/pkg/config" + "github.com/PingCAP-QE/ee-apps/publisher/pkg/impl/tiup" ) func main() { @@ -56,7 +56,7 @@ func main() { zerolog.SetGlobalLevel(logLevel) // Initialize the services. - tiupSvc, err := initService(*configFile) + tiupSvc, err := initTiupService(*configFile) if err != nil { log.Fatalf(ctx, err, "failed to initialize service") } @@ -64,10 +64,10 @@ func main() { // Wrap the services in endpoints that can be invoked from other services // potentially running in different processes. var ( - tiupEndpoints *tiup.Endpoints + tiupEndpoints *gentiup.Endpoints ) { - tiupEndpoints = tiup.NewEndpoints(tiupSvc) + tiupEndpoints = gentiup.NewEndpoints(tiupSvc) tiupEndpoints.Use(debug.LogPayloads()) tiupEndpoints.Use(log.Endpoint) } @@ -128,20 +128,20 @@ func main() { log.Printf(ctx, "exited") } -func initService(configFile string) (tiup.Service, error) { +func initTiupService(configFile string) (gentiup.Service, error) { // Load and parse configuration var config config.Service { configData, err := os.ReadFile(configFile) if err != nil { - return nil, fmt.Errorf("Error reading config file: %v", err) + return nil, fmt.Errorf("error reading config file: %v", err) } if err := yaml.Unmarshal(configData, &config); err != nil { - return nil, fmt.Errorf("Error parsing config file: %v", err) + return nil, fmt.Errorf("error parsing config file: %v", err) } } - logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", tiup.ServiceName).Logger() + logger := zerolog.New(os.Stderr).With().Timestamp().Str("service", gentiup.ServiceName).Logger() // Configure Kafka kafkaWriter kafkaWriter := kafka.NewWriter(kafka.WriterConfig{ @@ -159,5 +159,5 @@ func initService(configFile string) (tiup.Service, error) { DB: config.Redis.DB, }) - return publisher.NewTiup(&logger, kafkaWriter, redisClient, config.EventSource), nil + return tiup.NewTiup(&logger, kafkaWriter, redisClient, config.EventSource), nil } diff --git a/publisher/cmd/worker/main.go b/publisher/cmd/worker/main.go index 429bae4..966cc47 100644 --- a/publisher/cmd/worker/main.go +++ b/publisher/cmd/worker/main.go @@ -17,7 +17,7 @@ import ( "gopkg.in/yaml.v3" "github.com/PingCAP-QE/ee-apps/publisher/pkg/config" - "github.com/PingCAP-QE/ee-apps/publisher/pkg/tiup" + "github.com/PingCAP-QE/ee-apps/publisher/pkg/impl/tiup" ) func main() { diff --git a/publisher/design/design.go b/publisher/design/design.go index 267e31f..cd7797a 100644 --- a/publisher/design/design.go +++ b/publisher/design/design.go @@ -56,7 +56,9 @@ var _ = Service("tiup", func() { Attribute("request_id", String, "request track id") Required("request_id") }) - Result(String, "request state") + Result(String, "request state", func() { + Enum("queued", "processing", "success", "failed") + }) HTTP(func() { GET("/publish-request/{request_id}") Response(StatusOK) diff --git a/publisher/gen/http/cli/publisher/cli.go b/publisher/gen/http/cli/publisher/cli.go index 4211972..4d1162d 100644 --- a/publisher/gen/http/cli/publisher/cli.go +++ b/publisher/gen/http/cli/publisher/cli.go @@ -29,10 +29,10 @@ func UsageCommands() string { // UsageExamples produces an example of a valid invocation of the CLI tool. func UsageExamples() string { return os.Args[0] + ` tiup request-to-publish --body '{ - "artifact_url": "A facilis.", - "request_id": "Sequi placeat blanditiis est iusto quia eum.", - "tiup-mirror": "Sunt voluptates.", - "version": "Rerum consectetur deleniti." + "artifact_url": "Omnis expedita.", + "request_id": "Est sequi placeat.", + "tiup-mirror": "Pariatur rerum consectetur deleniti architecto sunt.", + "version": "Dicta id perferendis rem a." }'` + "\n" + "" } @@ -162,10 +162,10 @@ RequestToPublish implements request-to-publish. Example: %[1]s tiup request-to-publish --body '{ - "artifact_url": "A facilis.", - "request_id": "Sequi placeat blanditiis est iusto quia eum.", - "tiup-mirror": "Sunt voluptates.", - "version": "Rerum consectetur deleniti." + "artifact_url": "Omnis expedita.", + "request_id": "Est sequi placeat.", + "tiup-mirror": "Pariatur rerum consectetur deleniti architecto sunt.", + "version": "Dicta id perferendis rem a." }' `, os.Args[0]) } @@ -177,6 +177,6 @@ QueryPublishingStatus implements query-publishing-status. -request-id STRING: request track id Example: - %[1]s tiup query-publishing-status --request-id "Necessitatibus quidem." + %[1]s tiup query-publishing-status --request-id "Aspernatur quis voluptas." `, os.Args[0]) } diff --git a/publisher/gen/http/openapi.json b/publisher/gen/http/openapi.json index 5b4b84c..cd062e8 100644 --- a/publisher/gen/http/openapi.json +++ b/publisher/gen/http/openapi.json @@ -1 +1 @@ -{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"wuhui.zuo@pingcap.com","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Sed illo harum quis voluptatem."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string"}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et ullam."},"request_id":{"type":"string","description":"The request id","example":"Assumenda tempora autem accusantium est."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Ab expedita repellendus."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Et suscipit et."}},"example":{"artifact_url":"Corrupti ut natus aut ipsam reprehenderit.","request_id":"Rerum tempore voluptas.","tiup-mirror":"Laboriosam optio omnis cupiditate magnam nisi.","version":"Et eum odit eos ratione."},"required":["artifact_url","tiup-mirror"]}}} \ No newline at end of file +{"swagger":"2.0","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","email":"wuhui.zuo@pingcap.com","url":"https://github.com/wuhuizuo"},"version":"1.0.0"},"host":"0.0.0.0:80","consumes":["application/json","application/xml","application/gob"],"produces":["application/json","application/xml","application/gob"],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","parameters":[{"name":"Request-To-PublishRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/TiupRequestToPublishRequestBody","required":["artifact_url","tiup-mirror"]}}],"responses":{"200":{"description":"OK response.","schema":{"type":"array","items":{"type":"string","example":"Cupiditate suscipit hic quidem voluptates nostrum necessitatibus."}}}},"schemes":["http"]}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"type":"string","enum":["queued","processing","success","failed"]}}},"schemes":["http"]}}},"definitions":{"TiupRequestToPublishRequestBody":{"title":"TiupRequestToPublishRequestBody","type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Optio necessitatibus ipsa incidunt."},"request_id":{"type":"string","description":"The request id","example":"Et ullam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Illo harum quis voluptatem."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Expedita et necessitatibus ut molestias."}},"example":{"artifact_url":"Et suscipit et.","request_id":"Corrupti ut natus aut ipsam reprehenderit.","tiup-mirror":"Assumenda tempora autem accusantium est.","version":"Ab expedita repellendus."},"required":["artifact_url","tiup-mirror"]}}} \ No newline at end of file diff --git a/publisher/gen/http/openapi.yaml b/publisher/gen/http/openapi.yaml index 65aa191..09aeffa 100644 --- a/publisher/gen/http/openapi.yaml +++ b/publisher/gen/http/openapi.yaml @@ -39,7 +39,7 @@ paths: type: array items: type: string - example: Sed illo harum quis voluptatem. + example: Cupiditate suscipit hic quidem voluptates nostrum necessitatibus. schemes: - http /tiup/publish-request/{request_id}: @@ -59,6 +59,11 @@ paths: description: OK response. schema: type: string + enum: + - queued + - processing + - success + - failed schemes: - http definitions: @@ -69,25 +74,25 @@ definitions: artifact_url: type: string description: The full url of the pushed image, contain the tag part. It will parse the repo from it. - example: Et ullam. + example: Optio necessitatibus ipsa incidunt. request_id: type: string description: The request id - example: Assumenda tempora autem accusantium est. + example: Et ullam. tiup-mirror: type: string description: Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987. default: http://tiup.pingcap.net:8988 - example: Ab expedita repellendus. + example: Illo harum quis voluptatem. version: type: string description: Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config. - example: Et suscipit et. + example: Expedita et necessitatibus ut molestias. example: - artifact_url: Corrupti ut natus aut ipsam reprehenderit. - request_id: Rerum tempore voluptas. - tiup-mirror: Laboriosam optio omnis cupiditate magnam nisi. - version: Et eum odit eos ratione. + artifact_url: Et suscipit et. + request_id: Corrupti ut natus aut ipsam reprehenderit. + tiup-mirror: Assumenda tempora autem accusantium est. + version: Ab expedita repellendus. required: - artifact_url - tiup-mirror diff --git a/publisher/gen/http/openapi3.json b/publisher/gen/http/openapi3.json index ca6437d..b73223a 100644 --- a/publisher/gen/http/openapi3.json +++ b/publisher/gen/http/openapi3.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"wuhui.zuo@pingcap.com"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"A facilis.","request_id":"Sequi placeat blanditiis est iusto quia eum.","tiup-mirror":"Sunt voluptates.","version":"Rerum consectetur deleniti."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Distinctio voluptates voluptatem accusamus nisi omnis quia."},"description":"request track ids","example":["Voluptates possimus pariatur consequuntur itaque est dignissimos.","Atque amet optio et aliquam ut.","Nulla at error qui dolores et ipsa.","Rem nisi non."]},"example":["Molestiae vitae ut commodi voluptatem consectetur.","Magnam aut et labore."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Facere maiores tenetur mollitia consequatur perferendis sint."},"example":"Nisi earum earum pariatur et animi."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"Necessitatibus officia distinctio officia voluptas laudantium iure."},"example":"Aperiam consequatur consequatur."}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Voluptas ratione hic libero nisi."},"request_id":{"type":"string","description":"The request id","example":"Quibusdam nisi quam."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Magni quia adipisci excepturi."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Rerum voluptate accusantium optio."}},"example":{"artifact_url":"Atque vero in molestiae odit consequatur.","request_id":"Eos eveniet vero.","tiup-mirror":"Nemo rerum voluptas.","version":"Eum saepe nihil omnis dolorem eveniet."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]} \ No newline at end of file +{"openapi":"3.0.3","info":{"title":"Publish API","description":"Publish API","contact":{"name":"WuHui Zuo","url":"https://github.com/wuhuizuo","email":"wuhui.zuo@pingcap.com"},"version":"1.0.0"},"servers":[{"url":"http://0.0.0.0:80"}],"paths":{"/tiup/publish-request":{"post":{"tags":["tiup"],"summary":"request-to-publish tiup","operationId":"tiup#request-to-publish","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RequestToPublishRequestBody"},"example":{"artifact_url":"Omnis expedita.","request_id":"Est sequi placeat.","tiup-mirror":"Pariatur rerum consectetur deleniti architecto sunt.","version":"Dicta id perferendis rem a."}}}},"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"array","items":{"type":"string","example":"Eum saepe nihil omnis dolorem eveniet."},"description":"request track ids","example":["Rerum voluptas dolore.","Eveniet vero voluptas.","Voluptates voluptatem accusamus nisi omnis quia molestias."]},"example":["Pariatur consequuntur itaque est.","Exercitationem atque amet optio."]}}}}}},"/tiup/publish-request/{request_id}":{"get":{"tags":["tiup"],"summary":"query-publishing-status tiup","operationId":"tiup#query-publishing-status","parameters":[{"name":"request_id","in":"path","description":"request track id","required":true,"schema":{"type":"string","description":"request track id","example":"Aliquam ut laborum nulla."},"example":"Error qui dolores et."}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"type":"string","description":"request state","example":"queued","enum":["queued","processing","success","failed"]},"example":"failed"}}}}}}},"components":{"schemas":{"RequestToPublishRequestBody":{"type":"object","properties":{"artifact_url":{"type":"string","description":"The full url of the pushed image, contain the tag part. It will parse the repo from it.","example":"Et eum odit eos ratione."},"request_id":{"type":"string","description":"The request id","example":"Voluptas ratione hic libero nisi."},"tiup-mirror":{"type":"string","description":"Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.","default":"http://tiup.pingcap.net:8988","example":"Rerum tempore voluptas."},"version":{"type":"string","description":"Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config.","example":"Laboriosam optio omnis cupiditate magnam nisi."}},"example":{"artifact_url":"Rerum voluptate accusantium optio.","request_id":"Atque vero in molestiae odit consequatur.","tiup-mirror":"Quibusdam nisi quam.","version":"Magni quia adipisci excepturi."},"required":["artifact_url","tiup-mirror"]}}},"tags":[{"name":"tiup","description":"TiUP Publisher service"}]} \ No newline at end of file diff --git a/publisher/gen/http/openapi3.yaml b/publisher/gen/http/openapi3.yaml index fa4ac9c..cff1125 100644 --- a/publisher/gen/http/openapi3.yaml +++ b/publisher/gen/http/openapi3.yaml @@ -23,10 +23,10 @@ paths: schema: $ref: '#/components/schemas/RequestToPublishRequestBody' example: - artifact_url: A facilis. - request_id: Sequi placeat blanditiis est iusto quia eum. - tiup-mirror: Sunt voluptates. - version: Rerum consectetur deleniti. + artifact_url: Omnis expedita. + request_id: Est sequi placeat. + tiup-mirror: Pariatur rerum consectetur deleniti architecto sunt. + version: Dicta id perferendis rem a. responses: "200": description: OK response. @@ -36,16 +36,15 @@ paths: type: array items: type: string - example: Distinctio voluptates voluptatem accusamus nisi omnis quia. + example: Eum saepe nihil omnis dolorem eveniet. description: request track ids example: - - Voluptates possimus pariatur consequuntur itaque est dignissimos. - - Atque amet optio et aliquam ut. - - Nulla at error qui dolores et ipsa. - - Rem nisi non. + - Rerum voluptas dolore. + - Eveniet vero voluptas. + - Voluptates voluptatem accusamus nisi omnis quia molestias. example: - - Molestiae vitae ut commodi voluptatem consectetur. - - Magnam aut et labore. + - Pariatur consequuntur itaque est. + - Exercitationem atque amet optio. /tiup/publish-request/{request_id}: get: tags: @@ -60,8 +59,8 @@ paths: schema: type: string description: request track id - example: Facere maiores tenetur mollitia consequatur perferendis sint. - example: Nisi earum earum pariatur et animi. + example: Aliquam ut laborum nulla. + example: Error qui dolores et. responses: "200": description: OK response. @@ -70,8 +69,13 @@ paths: schema: type: string description: request state - example: Necessitatibus officia distinctio officia voluptas laudantium iure. - example: Aperiam consequatur consequatur. + example: queued + enum: + - queued + - processing + - success + - failed + example: failed components: schemas: RequestToPublishRequestBody: @@ -80,25 +84,25 @@ components: artifact_url: type: string description: The full url of the pushed image, contain the tag part. It will parse the repo from it. - example: Voluptas ratione hic libero nisi. + example: Et eum odit eos ratione. request_id: type: string description: The request id - example: Quibusdam nisi quam. + example: Voluptas ratione hic libero nisi. tiup-mirror: type: string description: Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987. default: http://tiup.pingcap.net:8988 - example: Magni quia adipisci excepturi. + example: Rerum tempore voluptas. version: type: string description: Force set the version. Default is the artifact version read from `org.opencontainers.image.version` of the manifest config. - example: Rerum voluptate accusantium optio. + example: Laboriosam optio omnis cupiditate magnam nisi. example: - artifact_url: Atque vero in molestiae odit consequatur. - request_id: Eos eveniet vero. - tiup-mirror: Nemo rerum voluptas. - version: Eum saepe nihil omnis dolorem eveniet. + artifact_url: Rerum voluptate accusantium optio. + request_id: Atque vero in molestiae odit consequatur. + tiup-mirror: Quibusdam nisi quam. + version: Magni quia adipisci excepturi. required: - artifact_url - tiup-mirror diff --git a/publisher/gen/http/tiup/client/cli.go b/publisher/gen/http/tiup/client/cli.go index 74a4dd7..a7e8465 100644 --- a/publisher/gen/http/tiup/client/cli.go +++ b/publisher/gen/http/tiup/client/cli.go @@ -22,7 +22,7 @@ func BuildRequestToPublishPayload(tiupRequestToPublishBody string) (*tiup.Reques { err = json.Unmarshal([]byte(tiupRequestToPublishBody), &body) if err != nil { - return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"artifact_url\": \"A facilis.\",\n \"request_id\": \"Sequi placeat blanditiis est iusto quia eum.\",\n \"tiup-mirror\": \"Sunt voluptates.\",\n \"version\": \"Rerum consectetur deleniti.\"\n }'") + return nil, fmt.Errorf("invalid JSON for body, \nerror: %s, \nexample of valid JSON:\n%s", err, "'{\n \"artifact_url\": \"Omnis expedita.\",\n \"request_id\": \"Est sequi placeat.\",\n \"tiup-mirror\": \"Pariatur rerum consectetur deleniti architecto sunt.\",\n \"version\": \"Dicta id perferendis rem a.\"\n }'") } } v := &tiup.RequestToPublishPayload{ diff --git a/publisher/gen/http/tiup/client/encode_decode.go b/publisher/gen/http/tiup/client/encode_decode.go index 59bf657..1a84eed 100644 --- a/publisher/gen/http/tiup/client/encode_decode.go +++ b/publisher/gen/http/tiup/client/encode_decode.go @@ -16,6 +16,7 @@ import ( tiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup" goahttp "goa.design/goa/v3/http" + goa "goa.design/goa/v3/pkg" ) // BuildRequestToPublishRequest instantiates a HTTP request object with method @@ -137,6 +138,12 @@ func DecodeQueryPublishingStatusResponse(decoder func(*http.Response) goahttp.De if err != nil { return nil, goahttp.ErrDecodingError("tiup", "query-publishing-status", err) } + if !(body == "queued" || body == "processing" || body == "success" || body == "failed") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body", body, []any{"queued", "processing", "success", "failed"})) + } + if err != nil { + return nil, goahttp.ErrValidationError("tiup", "query-publishing-status", err) + } return body, nil default: body, _ := io.ReadAll(resp.Body) diff --git a/publisher/pkg/tiup/funcs.go b/publisher/pkg/impl/tiup/funcs.go similarity index 100% rename from publisher/pkg/tiup/funcs.go rename to publisher/pkg/impl/tiup/funcs.go diff --git a/publisher/pkg/tiup/funcs_test.go b/publisher/pkg/impl/tiup/funcs_test.go similarity index 95% rename from publisher/pkg/tiup/funcs_test.go rename to publisher/pkg/impl/tiup/funcs_test.go index be17b77..19fb703 100644 --- a/publisher/pkg/tiup/funcs_test.go +++ b/publisher/pkg/impl/tiup/funcs_test.go @@ -3,8 +3,27 @@ package tiup import ( "reflect" "testing" + + cloudevents "github.com/cloudevents/sdk-go/v2" ) +func Test_xxx(t *testing.T) { + if !cloudevents.IsACK(nil) { + t.Errorf("IsACK(nil) = false, want true") + } + + if !cloudevents.IsACK(cloudevents.NewReceipt(true, "xxxx")) { + t.Errorf("xxx") + } + if !cloudevents.IsNACK(cloudevents.NewReceipt(true, "xxxx")) { + t.Errorf("xxx") + } + + if !cloudevents.IsNACK(cloudevents.NewReceipt(false, "xxxx")) { + t.Errorf("xxx") + } +} + func Test_fetchOCIArtifactConfig(t *testing.T) { type args struct { repo string diff --git a/publisher/pkg/tiup/publisher.go b/publisher/pkg/impl/tiup/publisher.go similarity index 83% rename from publisher/pkg/tiup/publisher.go rename to publisher/pkg/impl/tiup/publisher.go index cc1e3da..4f247b7 100644 --- a/publisher/pkg/tiup/publisher.go +++ b/publisher/pkg/impl/tiup/publisher.go @@ -2,6 +2,7 @@ package tiup import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -10,6 +11,7 @@ import ( "slices" cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/go-redis/redis/v8" "github.com/rs/zerolog" ) @@ -17,6 +19,7 @@ type Publisher struct { mirrorURL string larkWebhookURL string logger zerolog.Logger + redisClient *redis.Client } func NewPublisher(mirrorURL, larkWebhookURL string, logger *zerolog.Logger) (*Publisher, error) { @@ -39,13 +42,23 @@ func (p *Publisher) Handle(event cloudevents.Event) cloudevents.Result { if !slices.Contains(p.SupportEventTypes(), event.Type()) { return cloudevents.ResultNACK } + p.redisClient.Set(context.Background(), event.ID(), PublishStateProcessing, redis.KeepTTL) data := new(PublishRequest) if err := event.DataAs(&data); err != nil { return cloudevents.NewReceipt(false, "invalid data: %v", err) } - return p.handleImpl(data) + result := p.handleImpl(data) + switch { + case cloudevents.IsACK(result): + p.redisClient.Set(context.Background(), event.ID(), PublishStateSuccess, redis.KeepTTL) + default: + p.redisClient.Set(context.Background(), event.ID(), PublishStateFailed, redis.KeepTTL) + p.notifyLark(&data.Publish, result) + } + + return result } func (p *Publisher) handleImpl(data *PublishRequest) cloudevents.Result { @@ -53,16 +66,14 @@ func (p *Publisher) handleImpl(data *PublishRequest) cloudevents.Result { saveTo, err := downloadFile(data) if err != nil { p.logger.Err(err).Msg("download file failed") - // h.notifyLark(&data.Publish, err) - return cloudevents.NewReceipt(true, "download file failed: %v", err) + return cloudevents.NewReceipt(false, "download file failed: %v", err) } p.logger.Info().Msg("download file success") // 2. publish the tarball to the mirror. if err := p.publish(saveTo, &data.Publish); err != nil { p.logger.Err(err).Msg("publish to mirror failed") - p.notifyLark(&data.Publish, err) - return cloudevents.NewReceipt(true, "publish to mirror failed: %v", err) + return cloudevents.NewReceipt(false, "publish to mirror failed: %v", err) } p.logger.Info().Msg("publish to mirror success") @@ -71,7 +82,7 @@ func (p *Publisher) handleImpl(data *PublishRequest) cloudevents.Result { remoteURL := fmt.Sprintf("%s/%s-%s-%s-%s.tar.gz", p.mirrorURL, data.Publish.Name, data.Publish.Version, data.Publish.OS, data.Publish.Arch) if err := postCheck(saveTo, remoteURL); err != nil { p.logger.Err(err).Str("remote", remoteURL).Msg("post check failed") - return cloudevents.NewReceipt(true, "post check failed: %v", err) + return cloudevents.NewReceipt(false, "post check failed: %v", err) } p.logger.Info().Str("remote", remoteURL).Msg("post check success") diff --git a/publisher/tiup.go b/publisher/pkg/impl/tiup/service.go similarity index 86% rename from publisher/tiup.go rename to publisher/pkg/impl/tiup/service.go index 971cc91..3852385 100644 --- a/publisher/tiup.go +++ b/publisher/pkg/impl/tiup/service.go @@ -1,4 +1,4 @@ -package publisher +package tiup import ( "context" @@ -8,15 +8,10 @@ import ( cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/go-redis/redis/v8" "github.com/google/uuid" - gentiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup" - "github.com/PingCAP-QE/ee-apps/publisher/pkg/tiup" - "github.com/rs/zerolog" "github.com/segmentio/kafka-go" -) -const ( - DefaultStateTTL = 12 * time.Hour + gentiup "github.com/PingCAP-QE/ee-apps/publisher/gen/tiup" ) // tiup service example implementation. @@ -44,7 +39,7 @@ func NewTiup(logger *zerolog.Logger, kafkaWriter *kafka.Writer, redisClient *red func (s *tiupsrvc) RequestToPublish(ctx context.Context, p *gentiup.RequestToPublishPayload) (res []string, err error) { s.logger.Info().Msgf("tiup.request-to-publish") // 1. Analyze the artifact_url to get the repo and tag and the tiup package information. - publishRequests, err := tiup.AnalyzeFromOciArtifactUrl(p.ArtifactURL) + publishRequests, err := AnalyzeFromOciArtifactUrl(p.ArtifactURL) if err != nil { return nil, err } @@ -79,7 +74,7 @@ func (s *tiupsrvc) RequestToPublish(ctx context.Context, p *gentiup.RequestToPub // 5. Init the request dealing status in redis with the request id. for _, requestID := range requestIDs { - if err := s.redisClient.SetXX(ctx, requestID, "pending", s.stateTTL).Err(); err != nil { + if err := s.redisClient.SetXX(ctx, requestID, PublishStateQueued, s.stateTTL).Err(); err != nil { return nil, fmt.Errorf("failed to set initial status in Redis: %v", err) } } @@ -105,14 +100,14 @@ func (s *tiupsrvc) QueryPublishingStatus(ctx context.Context, p *gentiup.QueryPu return status, nil } -func (s *tiupsrvc) composeEvents(requests []tiup.PublishRequest) []cloudevents.Event { +func (s *tiupsrvc) composeEvents(requests []PublishRequest) []cloudevents.Event { var ret []cloudevents.Event for _, request := range requests { event := cloudevents.NewEvent() event.SetID(uuid.New().String()) - event.SetType(tiup.EventTypeTiupPublishRequest) + event.SetType(EventTypeTiupPublishRequest) event.SetSource(s.eventSource) - event.SetSubject(tiup.EventTypeTiupPublishRequest) + event.SetSubject(EventTypeTiupPublishRequest) event.SetData(cloudevents.ApplicationJSON, request) ret = append(ret, event) } diff --git a/publisher/pkg/tiup/types.go b/publisher/pkg/impl/tiup/types.go similarity index 80% rename from publisher/pkg/tiup/types.go rename to publisher/pkg/impl/tiup/types.go index 87dfb63..23d9faa 100644 --- a/publisher/pkg/tiup/types.go +++ b/publisher/pkg/impl/tiup/types.go @@ -1,10 +1,20 @@ package tiup +import "time" + const ( EventTypeTiupPublishRequest = "net.pingcap.tibuild.tiup-publish-request" FromTypeOci = "oci" FromTypeHTTP = "http" + + PublishStateQueued = "queued" + PublishStateProcessing = "processing" + PublishStateSuccess = "success" + PublishStateFailed = "failed" + PublishStateCanceled = "canceled" + + DefaultStateTTL = 12 * time.Hour ) type PublishRequest struct {