From ba2130adc589100285f64ee81548ebbff2f7775a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Mar 2024 18:39:22 +0000 Subject: [PATCH 1/8] Support returning error from `buildGenericIssueRequest` --- submit.go | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/submit.go b/submit.go index 38a0ec8..0980064 100644 --- a/submit.go +++ b/submit.go @@ -580,9 +580,12 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing } owner, repo := splits[0], splits[1] - issueReq := buildGithubIssueRequest(p, listingURL) + issueReq, err := buildGithubIssueRequest(p, listingURL) + if err != nil { + return err + } - issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq) + issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, issueReq) if err != nil { return err } @@ -602,7 +605,10 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub glProj := s.cfg.GitlabProjectMappings[p.AppName] glLabels := s.cfg.GitlabProjectLabels[p.AppName] - issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential) + issueReq, err := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential) + if err != nil { + return err + } issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq) @@ -665,7 +671,7 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer { return &bodyBuf } -func buildGenericIssueRequest(p payload, listingURL string) (title, body string) { +func buildGenericIssueRequest(p payload, listingURL string) (title, body string, err error) { bodyBuf := buildReportBody(p, " \n", "`") // Add log links to the body @@ -688,23 +694,29 @@ func buildGenericIssueRequest(p payload, listingURL string) (title, body string) return } -func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest { - title, body := buildGenericIssueRequest(p, listingURL) +func buildGithubIssueRequest(p payload, listingURL string) (*github.IssueRequest, error) { + title, body, err := buildGenericIssueRequest(p, listingURL) + if err != nil { + return nil, err + } labels := p.Labels // go-github doesn't like nils if labels == nil { labels = []string{} } - return github.IssueRequest{ + return &github.IssueRequest{ Title: &title, Body: &body, Labels: &labels, - } + }, nil } -func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions { - title, body := buildGenericIssueRequest(p, listingURL) +func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) { + title, body, err := buildGenericIssueRequest(p, listingURL) + if err != nil { + return nil, err + } if p.Labels != nil { labels = append(labels, p.Labels...) @@ -715,7 +727,7 @@ func buildGitlabIssueRequest(p payload, listingURL string, labels []string, conf Description: &body, Confidential: &confidential, Labels: labels, - } + }, nil } func (s *submitServer) sendEmail(p payload, reportDir string) error { From 4e15c4b0578a67bf386124403ee7483e74bf6d05 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 5 Mar 2024 13:13:25 +0000 Subject: [PATCH 2/8] Generate github/gitlab issue bodies from a template --- CHANGES.md | 9 +++++ README.md | 23 +++++++++++ main.go | 40 ++++++++++++++++++- rageshake.sample.yaml | 14 ++++++- submit.go | 51 +++++++++++++++---------- submit_test.go | 89 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 194 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f775d4e..2129e2b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +UNRELEASED +========== + +Features +-------- + +- Allow configuration of the body of created Github/Gitlab issues via a template in the configuration file. ([\#84](https://github.com/matrix-org/rageshake/issues/84)) + + 1.11.0 (2023-08-11) =================== diff --git a/README.md b/README.md index 10b1f61..d9f1304 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,29 @@ Optional parameters: * `-listen
`: TCP network address to listen for HTTP requests on. Example: `:9110`. +## Issue template + +It is possible to specify a template in the configuration file which will be used to build the +body of any issues created on Github or Gitlab, via the `issue_body_template` setting. +See [rageshake.sample.yaml](rageshake.sample.yaml) for an example. + +See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. + +The following properties are defined on the input (accessible via `.` or `$`): + +| Name | Type | Description | +|--------------|---------------------|---------------------------------------------------------------------------------------------------| +| `ID` | `string` | The unique ID for this rageshake. | +| `UserText` | `string` | A multi-line string containing the user description of the fault (from `text` in the submission). | +| `AppName` | `string` | A short slug to identify the app making the report (from `app` in the submission). | +| `Labels` | `[]string` | A list of labels requested by the application. | +| `Data` | `map[string]string` | A map of other key/value pairs included in the submission. | +| `Logs` | `[]string` | A list of log file names. | +| `LogErrors` | `[]string` | Set if there are log parsing errors. | +| `Files` | `[]string` | A list of other files (not logs) uploaded as part of the rageshake. | +| `FileErrors` | `[]string` | Set if there are file parsing errors. | +| `ListingURL` | `string` | Complete link to the listing URL that contains all uploaded logs. | + ## HTTP endpoints The following HTTP endpoints are exposed: diff --git a/main.go b/main.go index 287e5e6..f9e01a6 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "net/http" "os" "strings" + "text/template" "time" "github.com/google/go-github/github" @@ -37,6 +38,20 @@ import ( "gopkg.in/yaml.v2" ) +// DefaultIssueBodyTemplate is the default template used for `issue_body_template` in the config. +// +// !!! Keep in step with the documentation in `rageshake.sample.yaml` !!! +const DefaultIssueBodyTemplate = `User message: +{{ .UserText }} + +{{ range $key, $val := .Data -}} +{{ $key }}: ` + "`{{ $val }}`" + ` +{{ end }} +[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) +{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) +{{- end }} +` + var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.") var bindAddr = flag.String("listen", ":9110", "The port to listen on.") @@ -63,6 +78,8 @@ type config struct { GitlabProjectLabels map[string][]string `yaml:"gitlab_project_labels"` GitlabIssueConfidential bool `yaml:"gitlab_issue_confidential"` + IssueBodyTemplate string `yaml:"issue_body_template"` + SlackWebhookURL string `yaml:"slack_webhook_url"` EmailAddresses []string `yaml:"email_addresses"` @@ -158,7 +175,16 @@ func main() { log.Printf("Using %s/listing as public URI", apiPrefix) rand.Seed(time.Now().UnixNano()) - http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, appNameMap, cfg}) + http.Handle("/api/submit", &submitServer{ + issueTemplate: parseIssueTemplate(cfg), + ghClient: ghClient, + glClient: glClient, + apiPrefix: apiPrefix, + slack: slack, + genericWebhookClient: genericWebhookClient, + allowedAppNameMap: appNameMap, + cfg: cfg, + }) // Make sure bugs directory exists _ = os.Mkdir("bugs", os.ModePerm) @@ -186,6 +212,18 @@ func main() { log.Fatal(http.ListenAndServe(*bindAddr, nil)) } +func parseIssueTemplate(cfg *config) *template.Template { + issueTemplate := cfg.IssueBodyTemplate + if issueTemplate == "" { + issueTemplate = DefaultIssueBodyTemplate + } + parsedIssueTemplate, err := template.New("issue").Parse(issueTemplate) + if err != nil { + log.Fatalf("Invalid `issue_template` in config file: %s", err) + } + return parsedIssueTemplate +} + func configureAppNameMap(cfg *config) map[string]bool { if len(cfg.AllowedAppNames) == 0 { fmt.Println("Warning: allowed_app_names is empty. Accepting requests from all app names") diff --git a/rageshake.sample.yaml b/rageshake.sample.yaml index 283e812..35be478 100644 --- a/rageshake.sample.yaml +++ b/rageshake.sample.yaml @@ -21,6 +21,19 @@ github_token: secrettoken github_project_mappings: my-app: octocat/HelloWorld +# A template for the body of Github and Gitlab issues. The default template is as shown below. +# +# See `README.md` for more information on what can be specified here. +issue_body_template: | + {{ .UserText }} + + {{ range $key, $val := .Data -}} + {{ $key }}: `{{ $val }}` + {{ end }} + [Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) + {{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) + {{- end }} + # a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which # will be used to create a GitLab issue for each report. It requires # `api` scope. If omitted, no issues will be created. @@ -55,7 +68,6 @@ smtp_server: localhost:25 smtp_username: myemailuser smtp_password: myemailpass - # a list of webhook URLs, (see docs/generic_webhook.md) generic_webhook_urls: - https://server.example.com/your-server/api diff --git a/submit.go b/submit.go index 0980064..79516e0 100644 --- a/submit.go +++ b/submit.go @@ -37,6 +37,7 @@ import ( "sort" "strconv" "strings" + "text/template" "time" "github.com/google/go-github/github" @@ -47,6 +48,9 @@ import ( var maxPayloadSize = 1024 * 1024 * 55 // 55 MB type submitServer struct { + // Template for building github and gitlab issues + issueTemplate *template.Template + // github client for reporting bugs. may be nil, in which case, // reporting is disabled. ghClient *github.Client @@ -78,6 +82,15 @@ type jsonLogEntry struct { Lines string `json:"lines"` } +// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template`. +// +// !!! Keep in step with the documentation in `README.md` !!! +type issueBodyTemplatePayload struct { + payload + // Complete link to the listing URL that contains all uploaded logs + ListingURL string +} + // Stores additional information created during processing of a payload type genericWebhookPayload struct { payload @@ -87,7 +100,10 @@ type genericWebhookPayload struct { ListingURL string `json:"listing_url"` } -// Stores information about a request made to this server +// `payload` stores information about a request made to this server. +// +// !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step +// with the documentation in `README.md` !!! type payload struct { // A unique ID for this payload, generated within this server ID string `json:"id"` @@ -580,7 +596,7 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing } owner, repo := splits[0], splits[1] - issueReq, err := buildGithubIssueRequest(p, listingURL) + issueReq, err := buildGithubIssueRequest(p, listingURL, s.issueTemplate) if err != nil { return err } @@ -605,7 +621,7 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub glProj := s.cfg.GitlabProjectMappings[p.AppName] glLabels := s.cfg.GitlabProjectLabels[p.AppName] - issueReq, err := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential) + issueReq, err := buildGitlabIssueRequest(p, listingURL, s.issueTemplate, glLabels, s.cfg.GitlabIssueConfidential) if err != nil { return err } @@ -671,31 +687,26 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer { return &bodyBuf } -func buildGenericIssueRequest(p payload, listingURL string) (title, body string, err error) { - bodyBuf := buildReportBody(p, " \n", "`") +func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title, body string, err error) { + var bodyBuf bytes.Buffer - // Add log links to the body - fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL) - fmt.Fprintf(bodyBuf, " ([archive](%s))", listingURL+"?format=tar.gz") + issuePayload := issueBodyTemplatePayload{ + payload: p, + ListingURL: listingURL, + } - for _, file := range p.Files { - fmt.Fprintf( - bodyBuf, - " / [%s](%s)", - file, - listingURL+"/"+file, - ) + if err = bodyTemplate.Execute(&bodyBuf, issuePayload); err != nil { + return } title = buildReportTitle(p) - body = bodyBuf.String() return } -func buildGithubIssueRequest(p payload, listingURL string) (*github.IssueRequest, error) { - title, body, err := buildGenericIssueRequest(p, listingURL) +func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (*github.IssueRequest, error) { + title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) if err != nil { return nil, err } @@ -712,8 +723,8 @@ func buildGithubIssueRequest(p payload, listingURL string) (*github.IssueRequest }, nil } -func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) { - title, body, err := buildGenericIssueRequest(p, listingURL) +func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *template.Template, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) { + title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) if err != nil { return nil, err } diff --git a/submit_test.go b/submit_test.go index aa54a33..8c15853 100644 --- a/submit_test.go +++ b/submit_test.go @@ -28,6 +28,7 @@ import ( "strconv" "strings" "testing" + "text/template" ) // testParsePayload builds a /submit request with the given body, and calls @@ -56,11 +57,10 @@ func testParsePayload(t *testing.T, body, contentType string, tempDir string) (* return p, rr.Result() } - func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool, body string) int { - // Submit a request without files to the server and return statusCode - // Could be extended with more complicated config; aimed here just to - // test options for allowedAppNameMap + // Submit a request without files to the server and return statusCode + // Could be extended with more complicated config; aimed here just to + // test options for allowedAppNameMap req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body)) if err != nil { @@ -70,7 +70,7 @@ func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool w := httptest.NewRecorder() var cfg config - s := &submitServer{nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} + s := &submitServer{nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} s.ServeHTTP(w, req) rsp := w.Result() @@ -406,6 +406,63 @@ func mkTempDir(t *testing.T) string { * buildGithubIssueRequest tests */ +// General test of Github issue formatting. +func TestBuildGithubIssue(t *testing.T) { + body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="text" + + +test words. +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="app" + +riot-web +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="User-Agent" + +xxx +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="user_id" + +id +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="device_id" + +id +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="version" + +1 +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="file"; filename="passwd.txt" + +file +------WebKitFormBoundarySsdgl8Nq9voFyhdO-- +` + p, _ := testParsePayload(t, body, + "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", + "", + ) + + if p == nil { + t.Fatal("parseRequest returned nil") + } + + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } + + if *issueReq.Title != "test words." { + t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") + } + expectedBody := "User message:\n\ntest words.\n\nUser-Agent: `xxx`\nVersion: `1`\ndevice_id: `id`\nuser_id: `id`\n\n[Logs](http://test/listing/foo) ([archive](http://test/listing/foo?format=tar.gz)) / [passwd.txt](http://test/listing/foo/passwd.txt)\n" + if *issueReq.Body != expectedBody { + t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) + } +} + func TestBuildGithubIssueLeadingNewline(t *testing.T) { body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO Content-Disposition: form-data; name="text" @@ -427,12 +484,16 @@ riot-web t.Fatal("parseRequest returned nil") } - issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo") + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } if *issueReq.Title != "test words." { t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") } - expectedBody := "User message:\n\n\ntest words.\n" + expectedBody := "User message:\n\ntest words.\n" if !strings.HasPrefix(*issueReq.Body, expectedBody) { t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) } @@ -453,7 +514,11 @@ Content-Disposition: form-data; name="text" t.Fatal("parseRequest returned nil") } - issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo") + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } if *issueReq.Title != "Untitled report" { t.Errorf("Title: got %s, want %s", *issueReq.Title, "Untitled report") @@ -464,7 +529,7 @@ Content-Disposition: form-data; name="text" } } -func TestTestSortDataKeys(t *testing.T) { +func TestSortDataKeys(t *testing.T) { expect := ` Number of logs: 0 Application: @@ -506,9 +571,13 @@ user_id: id } } + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) for k, v := range sample { p := payload{Data: v.data} - res := buildGithubIssueRequest(p, "") + res, err := buildGithubIssueRequest(p, "", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } got := *res.Body if k == 0 { expect = got From 12b8e4b33efe4821bef2dfd0210434fc7b156d96 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Mar 2024 22:36:19 +0000 Subject: [PATCH 3/8] Move default issue template to a file --- README.md | 21 ++------------------- main.go | 16 ++++------------ rageshake.sample.yaml | 2 +- submit.go | 4 ++-- templates/README.md | 25 +++++++++++++++++++++++++ templates/issue_body.tmpl | 9 +++++++++ 6 files changed, 43 insertions(+), 34 deletions(-) create mode 100644 templates/README.md create mode 100644 templates/issue_body.tmpl diff --git a/README.md b/README.md index d9f1304..4272f6f 100644 --- a/README.md +++ b/README.md @@ -21,25 +21,8 @@ Optional parameters: ## Issue template It is possible to specify a template in the configuration file which will be used to build the -body of any issues created on Github or Gitlab, via the `issue_body_template` setting. -See [rageshake.sample.yaml](rageshake.sample.yaml) for an example. - -See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. - -The following properties are defined on the input (accessible via `.` or `$`): - -| Name | Type | Description | -|--------------|---------------------|---------------------------------------------------------------------------------------------------| -| `ID` | `string` | The unique ID for this rageshake. | -| `UserText` | `string` | A multi-line string containing the user description of the fault (from `text` in the submission). | -| `AppName` | `string` | A short slug to identify the app making the report (from `app` in the submission). | -| `Labels` | `[]string` | A list of labels requested by the application. | -| `Data` | `map[string]string` | A map of other key/value pairs included in the submission. | -| `Logs` | `[]string` | A list of log file names. | -| `LogErrors` | `[]string` | Set if there are log parsing errors. | -| `Files` | `[]string` | A list of other files (not logs) uploaded as part of the rageshake. | -| `FileErrors` | `[]string` | Set if there are file parsing errors. | -| `ListingURL` | `string` | Complete link to the listing URL that contains all uploaded logs. | +body of any issues created on Github or Gitlab, via the `issue_body_template` setting. See +[templates/README.md](templates/README.md) for more information. ## HTTP endpoints diff --git a/main.go b/main.go index f9e01a6..6251670 100644 --- a/main.go +++ b/main.go @@ -37,20 +37,12 @@ import ( "gopkg.in/yaml.v2" ) +import _ "embed" -// DefaultIssueBodyTemplate is the default template used for `issue_body_template` in the config. +// DefaultIssueBodyTemplate is the default template used for `issue_body_template_file` in the config. // -// !!! Keep in step with the documentation in `rageshake.sample.yaml` !!! -const DefaultIssueBodyTemplate = `User message: -{{ .UserText }} - -{{ range $key, $val := .Data -}} -{{ $key }}: ` + "`{{ $val }}`" + ` -{{ end }} -[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) -{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) -{{- end }} -` +//go:embed templates/issue_body.tmpl +var DefaultIssueBodyTemplate string var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.") var bindAddr = flag.String("listen", ":9110", "The port to listen on.") diff --git a/rageshake.sample.yaml b/rageshake.sample.yaml index 35be478..b45472f 100644 --- a/rageshake.sample.yaml +++ b/rageshake.sample.yaml @@ -23,7 +23,7 @@ github_project_mappings: # A template for the body of Github and Gitlab issues. The default template is as shown below. # -# See `README.md` for more information on what can be specified here. +# See `templates/README.md` for more information on what can be specified here. issue_body_template: | {{ .UserText }} diff --git a/submit.go b/submit.go index 79516e0..ec56492 100644 --- a/submit.go +++ b/submit.go @@ -84,7 +84,7 @@ type jsonLogEntry struct { // `issueBodyTemplatePayload` contains the data made available to the `issue_body_template`. // -// !!! Keep in step with the documentation in `README.md` !!! +// !!! Keep in step with the documentation in `templates/README.md` !!! type issueBodyTemplatePayload struct { payload // Complete link to the listing URL that contains all uploaded logs @@ -103,7 +103,7 @@ type genericWebhookPayload struct { // `payload` stores information about a request made to this server. // // !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step -// with the documentation in `README.md` !!! +// with the documentation in `templates/README.md` !!! type payload struct { // A unique ID for this payload, generated within this server ID string `json:"id"` diff --git a/templates/README.md b/templates/README.md new file mode 100644 index 0000000..f257fbe --- /dev/null +++ b/templates/README.md @@ -0,0 +1,25 @@ +This directory contains the default templates that are used by the rageshake server. + +The templates can be overridden via settings in the config file. + +The templates are as follows: + +* `issue_body.tmpl`: Used when filing an issue at Github or Gitlab, and gives the issue description. Override via + the `issue_body_template` setting in the configuration file. + +See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. + +The following properties are defined on the input (accessible via `.` or `$`): + +| Name | Type | Description | +|--------------|---------------------|---------------------------------------------------------------------------------------------------| +| `ID` | `string` | The unique ID for this rageshake. | +| `UserText` | `string` | A multi-line string containing the user description of the fault (from `text` in the submission). | +| `AppName` | `string` | A short slug to identify the app making the report (from `app` in the submission). | +| `Labels` | `[]string` | A list of labels requested by the application. | +| `Data` | `map[string]string` | A map of other key/value pairs included in the submission. | +| `Logs` | `[]string` | A list of log file names. | +| `LogErrors` | `[]string` | Set if there are log parsing errors. | +| `Files` | `[]string` | A list of other files (not logs) uploaded as part of the rageshake. | +| `FileErrors` | `[]string` | Set if there are file parsing errors. | +| `ListingURL` | `string` | Complete link to the listing URL that contains all uploaded logs. | diff --git a/templates/issue_body.tmpl b/templates/issue_body.tmpl new file mode 100644 index 0000000..8de4d52 --- /dev/null +++ b/templates/issue_body.tmpl @@ -0,0 +1,9 @@ +User message: +{{ .UserText }} + +{{ range $key, $val := .Data -}} +{{ $key }}: `{{ $val }}` +{{ end }} +[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) +{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) +{{- end }} From 8b155c09e8e659e6b016dd068b87e1c1ac20ba1d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Mar 2024 22:45:34 +0000 Subject: [PATCH 4/8] Use a template file rather than an inline template --- README.md | 2 +- main.go | 15 ++++++++++----- rageshake.sample.yaml | 15 +++------------ templates/README.md | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4272f6f..9f7b8d6 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Optional parameters: ## Issue template It is possible to specify a template in the configuration file which will be used to build the -body of any issues created on Github or Gitlab, via the `issue_body_template` setting. See +body of any issues created on Github or Gitlab, via the `issue_body_template_file` setting. See [templates/README.md](templates/README.md) for more information. ## HTTP endpoints diff --git a/main.go b/main.go index 6251670..2aec899 100644 --- a/main.go +++ b/main.go @@ -70,7 +70,7 @@ type config struct { GitlabProjectLabels map[string][]string `yaml:"gitlab_project_labels"` GitlabIssueConfidential bool `yaml:"gitlab_issue_confidential"` - IssueBodyTemplate string `yaml:"issue_body_template"` + IssueBodyTemplateFile string `yaml:"issue_body_template_file"` SlackWebhookURL string `yaml:"slack_webhook_url"` @@ -205,13 +205,18 @@ func main() { } func parseIssueTemplate(cfg *config) *template.Template { - issueTemplate := cfg.IssueBodyTemplate - if issueTemplate == "" { - issueTemplate = DefaultIssueBodyTemplate + issueTemplate := DefaultIssueBodyTemplate + issueTemplateFile := cfg.IssueBodyTemplateFile + if issueTemplateFile != "" { + issueTemplateBytes, err := os.ReadFile(issueTemplateFile) + if err != nil { + log.Fatalf("Unable to read template file `%s`: %s", issueTemplateFile, err) + } + issueTemplate = string(issueTemplateBytes) } parsedIssueTemplate, err := template.New("issue").Parse(issueTemplate) if err != nil { - log.Fatalf("Invalid `issue_template` in config file: %s", err) + log.Fatalf("Invalid `issue body template` in config file: %s", err) } return parsedIssueTemplate } diff --git a/rageshake.sample.yaml b/rageshake.sample.yaml index b45472f..4166cc4 100644 --- a/rageshake.sample.yaml +++ b/rageshake.sample.yaml @@ -21,18 +21,9 @@ github_token: secrettoken github_project_mappings: my-app: octocat/HelloWorld -# A template for the body of Github and Gitlab issues. The default template is as shown below. -# -# See `templates/README.md` for more information on what can be specified here. -issue_body_template: | - {{ .UserText }} - - {{ range $key, $val := .Data -}} - {{ $key }}: `{{ $val }}` - {{ end }} - [Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) - {{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) - {{- end }} +# The path of a template for the body of Github and Gitlab issues. See `templates/README.md` for more information on +# the templates. +issue_body_template_file: path/to/issue_body.tmpl # a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which # will be used to create a GitLab issue for each report. It requires diff --git a/templates/README.md b/templates/README.md index f257fbe..614b2fe 100644 --- a/templates/README.md +++ b/templates/README.md @@ -5,7 +5,7 @@ The templates can be overridden via settings in the config file. The templates are as follows: * `issue_body.tmpl`: Used when filing an issue at Github or Gitlab, and gives the issue description. Override via - the `issue_body_template` setting in the configuration file. + the `issue_body_template_file` setting in the configuration file. See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. From 058362f2cd871cacf6a044f40bbeee38e3cb36f5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 15 Mar 2024 11:27:06 +0000 Subject: [PATCH 5/8] Replace `parseIssueTemplate` with more generic `parseTemplate` ... so that in future we can use it for other templates --- main.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 2aec899..a2ee0b4 100644 --- a/main.go +++ b/main.go @@ -168,7 +168,7 @@ func main() { rand.Seed(time.Now().UnixNano()) http.Handle("/api/submit", &submitServer{ - issueTemplate: parseIssueTemplate(cfg), + issueTemplate: parseTemplate(DefaultIssueBodyTemplate, cfg.IssueBodyTemplateFile, "issue"), ghClient: ghClient, glClient: glClient, apiPrefix: apiPrefix, @@ -204,21 +204,26 @@ func main() { log.Fatal(http.ListenAndServe(*bindAddr, nil)) } -func parseIssueTemplate(cfg *config) *template.Template { - issueTemplate := DefaultIssueBodyTemplate - issueTemplateFile := cfg.IssueBodyTemplateFile - if issueTemplateFile != "" { - issueTemplateBytes, err := os.ReadFile(issueTemplateFile) +// parseTemplate parses a template file, with fallback to default. +// +// If `configFileSettingValue` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is +// used. +// +// The template text is then parsed into a template named `templateName`. +func parseTemplate(defaultTemplate string, configFileSettingValue string, templateName string) *template.Template { + templateText := defaultTemplate + if configFileSettingValue != "" { + issueTemplateBytes, err := os.ReadFile(configFileSettingValue) if err != nil { - log.Fatalf("Unable to read template file `%s`: %s", issueTemplateFile, err) + log.Fatalf("Unable to read template file `%s`: %s", configFileSettingValue, err) } - issueTemplate = string(issueTemplateBytes) + templateText = string(issueTemplateBytes) } - parsedIssueTemplate, err := template.New("issue").Parse(issueTemplate) + parsedTemplate, err := template.New(templateName).Parse(templateText) if err != nil { - log.Fatalf("Invalid `issue body template` in config file: %s", err) + log.Fatalf("Invalid template file %s in config file: %s", configFileSettingValue, err) } - return parsedIssueTemplate + return parsedTemplate } func configureAppNameMap(cfg *config) map[string]bool { From 28d9c63819903533a86281187ef8c32d2e36650f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 15 Mar 2024 11:35:39 +0000 Subject: [PATCH 6/8] `buildGenericIssueRequest`: return `[]byte` ... to make it more reusable. No point converting `[]byte` to `string` back to `[]byte` again. --- submit.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/submit.go b/submit.go index ec56492..a72dc41 100644 --- a/submit.go +++ b/submit.go @@ -687,7 +687,7 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer { return &bodyBuf } -func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title, body string, err error) { +func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title string, body []byte, err error) { var bodyBuf bytes.Buffer issuePayload := issueBodyTemplatePayload{ @@ -700,7 +700,7 @@ func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *templa } title = buildReportTitle(p) - body = bodyBuf.String() + body = bodyBuf.Bytes() return } @@ -716,9 +716,10 @@ func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *templat if labels == nil { labels = []string{} } + bodyStr := string(body) return &github.IssueRequest{ Title: &title, - Body: &body, + Body: &bodyStr, Labels: &labels, }, nil } @@ -733,9 +734,10 @@ func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *templat labels = append(labels, p.Labels...) } + bodyStr := string(body) return &gitlab.CreateIssueOptions{ Title: &title, - Description: &body, + Description: &bodyStr, Confidential: &confidential, Labels: labels, }, nil From 575d25957f0270a5743d88ef243acc6c0dd051b2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 15 Mar 2024 11:38:32 +0000 Subject: [PATCH 7/8] Use template for email bodies --- README.md | 5 ++--- main.go | 7 +++++++ rageshake.sample.yaml | 9 +++++---- submit.go | 39 +++++++++++++++------------------------ submit_test.go | 2 +- templates/README.md | 1 + templates/email_body.tmpl | 9 +++++++++ 7 files changed, 40 insertions(+), 32 deletions(-) create mode 100644 templates/email_body.tmpl diff --git a/README.md b/README.md index 9f7b8d6..5c64f64 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,8 @@ Optional parameters: ## Issue template -It is possible to specify a template in the configuration file which will be used to build the -body of any issues created on Github or Gitlab, via the `issue_body_template_file` setting. See -[templates/README.md](templates/README.md) for more information. +It is possible to override the templates used to construct emails, and Github and Gitlab issues. +See [templates/README.md](templates/README.md) for more information. ## HTTP endpoints diff --git a/main.go b/main.go index a2ee0b4..13c3aac 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,11 @@ import _ "embed" //go:embed templates/issue_body.tmpl var DefaultIssueBodyTemplate string +// DefaultEmailBodyTemplate is the default template used for `email_body_template_file` in the config. +// +//go:embed templates/email_body.tmpl +var DefaultEmailBodyTemplate string + var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.") var bindAddr = flag.String("listen", ":9110", "The port to listen on.") @@ -71,6 +76,7 @@ type config struct { GitlabIssueConfidential bool `yaml:"gitlab_issue_confidential"` IssueBodyTemplateFile string `yaml:"issue_body_template_file"` + EmailBodyTemplateFile string `yaml:"email_body_template_file"` SlackWebhookURL string `yaml:"slack_webhook_url"` @@ -169,6 +175,7 @@ func main() { rand.Seed(time.Now().UnixNano()) http.Handle("/api/submit", &submitServer{ issueTemplate: parseTemplate(DefaultIssueBodyTemplate, cfg.IssueBodyTemplateFile, "issue"), + emailTemplate: parseTemplate(DefaultEmailBodyTemplate, cfg.EmailBodyTemplateFile, "email"), ghClient: ghClient, glClient: glClient, apiPrefix: apiPrefix, diff --git a/rageshake.sample.yaml b/rageshake.sample.yaml index 4166cc4..204a919 100644 --- a/rageshake.sample.yaml +++ b/rageshake.sample.yaml @@ -21,10 +21,6 @@ github_token: secrettoken github_project_mappings: my-app: octocat/HelloWorld -# The path of a template for the body of Github and Gitlab issues. See `templates/README.md` for more information on -# the templates. -issue_body_template_file: path/to/issue_body.tmpl - # a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which # will be used to create a GitLab issue for each report. It requires # `api` scope. If omitted, no issues will be created. @@ -63,3 +59,8 @@ smtp_password: myemailpass generic_webhook_urls: - https://server.example.com/your-server/api - http://another-server.com/api + +# The paths of template files for the body of Github and Gitlab issues, and emails. +# See `templates/README.md` for more information. +issue_body_template_file: path/to/issue_body.tmpl +email_body_template_file: path/to/email_body.tmpl diff --git a/submit.go b/submit.go index a72dc41..ae68edb 100644 --- a/submit.go +++ b/submit.go @@ -51,6 +51,9 @@ type submitServer struct { // Template for building github and gitlab issues issueTemplate *template.Template + // Template for building emails + emailTemplate *template.Template + // github client for reporting bugs. may be nil, in which case, // reporting is disabled. ghClient *github.Client @@ -82,7 +85,8 @@ type jsonLogEntry struct { Lines string `json:"lines"` } -// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template`. +// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template` and +// `email_body_template`. // // !!! Keep in step with the documentation in `templates/README.md` !!! type issueBodyTemplatePayload struct { @@ -521,7 +525,7 @@ func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, lis return nil, err } - if err := s.sendEmail(p, reportDir); err != nil { + if err := s.sendEmail(p, reportDir, listingURL); err != nil { return nil, err } @@ -671,22 +675,6 @@ func buildReportTitle(p payload) string { return trimmedUserText } -func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer { - var bodyBuf bytes.Buffer - fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText) - var dataKeys []string - for k := range p.Data { - dataKeys = append(dataKeys, k) - } - sort.Strings(dataKeys) - for _, k := range dataKeys { - v := p.Data[k] - fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline) - } - - return &bodyBuf -} - func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title string, body []byte, err error) { var bodyBuf bytes.Buffer @@ -743,11 +731,16 @@ func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *templat }, nil } -func (s *submitServer) sendEmail(p payload, reportDir string) error { +func (s *submitServer) sendEmail(p payload, reportDir string, listingURL string) error { if len(s.cfg.EmailAddresses) == 0 { return nil } + title, body, err := buildGenericIssueRequest(p, listingURL, s.emailTemplate) + if err != nil { + return err + } + e := email.NewEmail() e.From = "Rageshake " @@ -756,10 +749,8 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error { } e.To = s.cfg.EmailAddresses - - e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p)) - - e.Text = buildReportBody(p, "\n", "\"").Bytes() + e.Subject = fmt.Sprintf("[%s] %s", p.AppName, title) + e.Text = body allFiles := append(p.Files, p.Logs...) for _, file := range allFiles { @@ -771,7 +762,7 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error { if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" { auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer) } - err := e.Send(s.cfg.SMTPServer, auth) + err = e.Send(s.cfg.SMTPServer, auth) if err != nil { return err } diff --git a/submit_test.go b/submit_test.go index 8c15853..12ebefe 100644 --- a/submit_test.go +++ b/submit_test.go @@ -70,7 +70,7 @@ func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool w := httptest.NewRecorder() var cfg config - s := &submitServer{nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} + s := &submitServer{nil, nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} s.ServeHTTP(w, req) rsp := w.Result() diff --git a/templates/README.md b/templates/README.md index 614b2fe..9382e2c 100644 --- a/templates/README.md +++ b/templates/README.md @@ -6,6 +6,7 @@ The templates are as follows: * `issue_body.tmpl`: Used when filing an issue at Github or Gitlab, and gives the issue description. Override via the `issue_body_template_file` setting in the configuration file. +* `email_body.tmpl`: Used when sending an email. Override via the `email_body_template_file` configuration setting. See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. diff --git a/templates/email_body.tmpl b/templates/email_body.tmpl new file mode 100644 index 0000000..13cbab2 --- /dev/null +++ b/templates/email_body.tmpl @@ -0,0 +1,9 @@ +User message: +{{ .UserText }} + +{{ range $key, $val := .Data -}} +{{ $key }}: "{{ $val }}" +{{ end }} +[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) +{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) +{{- end }} From dbcf2ddb65a6f19e38641937fba67645d7c45c64 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 18 Mar 2024 13:31:09 +0000 Subject: [PATCH 8/8] Rename variable --- main.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 13c3aac..ffe3636 100644 --- a/main.go +++ b/main.go @@ -213,22 +213,22 @@ func main() { // parseTemplate parses a template file, with fallback to default. // -// If `configFileSettingValue` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is +// If `templateFilePath` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is // used. // // The template text is then parsed into a template named `templateName`. -func parseTemplate(defaultTemplate string, configFileSettingValue string, templateName string) *template.Template { +func parseTemplate(defaultTemplate string, templateFilePath string, templateName string) *template.Template { templateText := defaultTemplate - if configFileSettingValue != "" { - issueTemplateBytes, err := os.ReadFile(configFileSettingValue) + if templateFilePath != "" { + issueTemplateBytes, err := os.ReadFile(templateFilePath) if err != nil { - log.Fatalf("Unable to read template file `%s`: %s", configFileSettingValue, err) + log.Fatalf("Unable to read template file `%s`: %s", templateFilePath, err) } templateText = string(issueTemplateBytes) } parsedTemplate, err := template.New(templateName).Parse(templateText) if err != nil { - log.Fatalf("Invalid template file %s in config file: %s", configFileSettingValue, err) + log.Fatalf("Invalid template file %s in config file: %s", templateFilePath, err) } return parsedTemplate }