From 8ccdeb228e611cba37e653c37746af7c9688a7e6 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 19 Sep 2023 18:36:46 +0545 Subject: [PATCH 1/2] feat: add metrics for notifications --- cmd/server.go | 11 +++++++++-- go.mod | 2 +- notification/metrics.go | 24 ++++++++++++++++++++++++ notification/shoutrrr.go | 6 ++++++ 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 notification/metrics.go diff --git a/cmd/server.go b/cmd/server.go index 5e2faf590..1a5146ebd 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -14,6 +14,8 @@ import ( "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + prom "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" @@ -59,8 +61,13 @@ func createHTTPServer(ctx api.Context) *echo.Echo { } }) - e.Use(echoprometheus.NewMiddleware("mission_control")) - e.GET("/metrics", echoprometheus.NewHandler()) + e.Use(echoprometheus.NewMiddlewareWithConfig(echoprometheus.MiddlewareConfig{ + Registerer: prom.DefaultRegisterer, + })) + + e.GET("/metrics", echoprometheus.NewHandlerWithConfig(echoprometheus.HandlerConfig{ + Gatherer: prom.DefaultGatherer, + })) if authMode != "" { var ( diff --git a/go.mod b/go.mod index 96f2935df..a61ba81bf 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/onsi/ginkgo/v2 v2.9.5 github.com/onsi/gomega v1.27.7 github.com/ory/client-go v1.1.41 + github.com/prometheus/client_golang v1.16.0 github.com/sethvargo/go-retry v0.2.4 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -89,7 +90,6 @@ require ( github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect diff --git a/notification/metrics.go b/notification/metrics.go new file mode 100644 index 000000000..f4b8ea536 --- /dev/null +++ b/notification/metrics.go @@ -0,0 +1,24 @@ +package notification + +import "github.com/prometheus/client_golang/prometheus" + +func init() { + prometheus.MustRegister(notificationSentCounter, notificationSendDuration) +} + +var ( + notificationSentCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "sent_total", + Subsystem: "notification", + Help: "Total number of notifications sent", + }, + []string{"service"}, + ) + + notificationSendDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "send_duration_seconds", + Subsystem: "notification", + Help: "Duration to send a notification.", + }, []string{"service"}) +) diff --git a/notification/shoutrrr.go b/notification/shoutrrr.go index b1d0fdc9c..48f88a050 100644 --- a/notification/shoutrrr.go +++ b/notification/shoutrrr.go @@ -6,6 +6,7 @@ import ( "os" "strconv" "strings" + "time" stripmd "github.com/adityathebe/go-strip-markdown/v2" "github.com/containrrr/shoutrrr" @@ -42,6 +43,8 @@ func setSystemSMTPCredential(shoutrrrURL string) (string, error) { } func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) error { + start := time.Now() + if connectionName != "" { connection, err := ctx.HydrateConnection(connectionName) if err != nil { @@ -126,6 +129,9 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop } } + notificationSentCounter.WithLabelValues(service).Inc() + notificationSendDuration.WithLabelValues(service).Observe(time.Since(start).Seconds()) + return nil } From 6972a92e5633c55eb451789e3e9ef3627eee5e83 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 20 Sep 2023 09:42:15 +0545 Subject: [PATCH 2/2] feat: add more metric labels & a new metric --- events/notifications.go | 4 +++- notification/context.go | 16 +++++++++++++++- notification/metrics.go | 15 ++++++++++++--- notification/shoutrrr.go | 33 ++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/events/notifications.go b/events/notifications.go index 46689ccbc..4c4fd1157 100644 --- a/events/notifications.go +++ b/events/notifications.go @@ -218,7 +218,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error { } if props.PersonID != nil { - ctx.WithPersonID(props.PersonID) + ctx.WithPersonID(props.PersonID).WithRecipientType(pkgNotification.RecipientTypePerson) var emailAddress string if err := ctx.DB().Model(&models.Person{}).Select("email").Where("id = ?", props.PersonID).Find(&emailAddress).Error; err != nil { return fmt.Errorf("failed to get email of person(id=%s); %v", props.PersonID, err) @@ -229,6 +229,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error { } if props.TeamID != "" { + ctx.WithRecipientType(pkgNotification.RecipientTypeTeam) teamSpec, err := teams.GetTeamSpec(ctx.Context, props.TeamID) if err != nil { return fmt.Errorf("failed to get team(id=%s); %v", props.TeamID, err) @@ -248,6 +249,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error { } for _, cn := range notification.CustomNotifications { + ctx.WithRecipientType(pkgNotification.RecipientTypeCustom) if cn.Name != props.NotificationName { continue } diff --git a/notification/context.go b/notification/context.go index 6b0ac14bb..7878bcaf4 100644 --- a/notification/context.go +++ b/notification/context.go @@ -6,9 +6,18 @@ import ( "github.com/google/uuid" ) +type RecipientType string + +const ( + RecipientTypePerson RecipientType = "person" + RecipientTypeTeam RecipientType = "team" + RecipientTypeCustom RecipientType = "custom" +) + type Context struct { api.Context notificationID uuid.UUID + recipientType RecipientType log *models.NotificationSendHistory } @@ -32,6 +41,10 @@ func (t *Context) WithMessage(message string) { t.log.Body = message } +func (t *Context) WithRecipientType(recipientType RecipientType) { + t.recipientType = recipientType +} + func (t *Context) WithError(err string) { t.log.Error = &err } @@ -41,6 +54,7 @@ func (t *Context) WithSource(event string, resourceID uuid.UUID) { t.log.ResourceID = resourceID } -func (t *Context) WithPersonID(id *uuid.UUID) { +func (t *Context) WithPersonID(id *uuid.UUID) *Context { t.log.PersonID = id + return t } diff --git a/notification/metrics.go b/notification/metrics.go index f4b8ea536..780d48f52 100644 --- a/notification/metrics.go +++ b/notification/metrics.go @@ -3,7 +3,7 @@ package notification import "github.com/prometheus/client_golang/prometheus" func init() { - prometheus.MustRegister(notificationSentCounter, notificationSendDuration) + prometheus.MustRegister(notificationSentCounter, notificationSendFailureCounter, notificationSendDuration) } var ( @@ -13,12 +13,21 @@ var ( Subsystem: "notification", Help: "Total number of notifications sent", }, - []string{"service"}, + []string{"service", "recipient_type", "id"}, + ) + + notificationSendFailureCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "send_error_total", + Subsystem: "notification", + Help: "Total number of failure notifications sent", + }, + []string{"service", "recipient_type", "id"}, ) notificationSendDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "send_duration_seconds", Subsystem: "notification", Help: "Duration to send a notification.", - }, []string{"service"}) + }, []string{"service", "recipient_type", "id"}) ) diff --git a/notification/shoutrrr.go b/notification/shoutrrr.go index 48f88a050..4ce5ec2f0 100644 --- a/notification/shoutrrr.go +++ b/notification/shoutrrr.go @@ -45,10 +45,24 @@ func setSystemSMTPCredential(shoutrrrURL string) (string, error) { func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) error { start := time.Now() + service, err := send(ctx, connectionName, shoutrrrURL, title, message, properties...) + if err != nil { + notificationSendFailureCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc() + return err + } + + notificationSentCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc() + notificationSendDuration.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Observe(time.Since(start).Seconds()) + + return nil +} + +// send sends a notification and returns the service it sent the notification to +func send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) (string, error) { if connectionName != "" { connection, err := ctx.HydrateConnection(connectionName) if err != nil { - return err + return "", err } shoutrrrURL = connection.URL @@ -59,18 +73,18 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop var err error shoutrrrURL, err = setSystemSMTPCredential(shoutrrrURL) if err != nil { - return err + return "", err } } sender, err := shoutrrr.CreateSender(shoutrrrURL) if err != nil { - return fmt.Errorf("failed to create a shoutrrr sender client: %w", err) + return "", fmt.Errorf("failed to create a shoutrrr sender client: %w", err) } service, _, err := sender.ExtractServiceName(shoutrrrURL) if err != nil { - return fmt.Errorf("failed to extract service name: %w", err) + return "", fmt.Errorf("failed to extract service name: %w", err) } switch service { @@ -104,7 +118,7 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop if service == "smtp" { parsedURL, err := url.Parse(shoutrrrURL) if err != nil { - return fmt.Errorf("failed to parse shoutrrr URL: %w", err) + return "", fmt.Errorf("failed to parse shoutrrr URL: %w", err) } query := parsedURL.Query() @@ -119,20 +133,17 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop m := mail.New(to, title, message, `text/html; charset="UTF-8"`). SetFrom(fromName, from). SetCredentials(parsedURL.Hostname(), port, parsedURL.User.Username(), password) - return m.Send() + return service, m.Send() } sendErrors := sender.Send(message, params) for _, err := range sendErrors { if err != nil { - return fmt.Errorf("error publishing notification (service=%s): %w", service, err) + return "", fmt.Errorf("error publishing notification (service=%s): %w", service, err) } } - notificationSentCounter.WithLabelValues(service).Inc() - notificationSendDuration.WithLabelValues(service).Observe(time.Since(start).Seconds()) - - return nil + return service, nil } // injectTitleIntoProperties adds the given title to the shoutrrr properties if it's not already set.