feat: add metrics for notifications

cmd/server.go

@@ -14,6 +14,8 @@ import (
 	"github.com/labstack/echo-contrib/echoprometheus"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	prom "github.com/prometheus/client_golang/prometheus"
+
 	"github.com/spf13/cobra"
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -59,8 +61,13 @@ func createHTTPServer(ctx api.Context) *echo.Echo {
 		}
 	})
 
-	e.Use(echoprometheus.NewMiddleware("mission_control"))
-	e.GET("/metrics", echoprometheus.NewHandler())
+	e.Use(echoprometheus.NewMiddlewareWithConfig(echoprometheus.MiddlewareConfig{
+		Registerer: prom.DefaultRegisterer,
+	}))
+
+	e.GET("/metrics", echoprometheus.NewHandlerWithConfig(echoprometheus.HandlerConfig{
+		Gatherer: prom.DefaultGatherer,
+	}))
 
 	if authMode != "" {
 		var (

events/notifications.go

@@ -218,7 +218,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	if props.PersonID != nil {
-		ctx.WithPersonID(props.PersonID)
+		ctx.WithPersonID(props.PersonID).WithRecipientType(pkgNotification.RecipientTypePerson)
 		var emailAddress string
 		if err := ctx.DB().Model(&models.Person{}).Select("email").Where("id = ?", props.PersonID).Find(&emailAddress).Error; err != nil {
 			return fmt.Errorf("failed to get email of person(id=%s); %v", props.PersonID, err)
@@ -229,6 +229,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	if props.TeamID != "" {
+		ctx.WithRecipientType(pkgNotification.RecipientTypeTeam)
 		teamSpec, err := teams.GetTeamSpec(ctx.Context, props.TeamID)
 		if err != nil {
 			return fmt.Errorf("failed to get team(id=%s); %v", props.TeamID, err)
@@ -248,6 +249,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	for _, cn := range notification.CustomNotifications {
+		ctx.WithRecipientType(pkgNotification.RecipientTypeCustom)
 		if cn.Name != props.NotificationName {
 			continue
 		}

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.5
 	github.com/onsi/gomega v1.27.7
 	github.com/ory/client-go v1.1.41
+	github.com/prometheus/client_golang v1.16.0
 	github.com/sethvargo/go-retry v0.2.4
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
@@ -89,7 +90,6 @@ require (
 	github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect

notification/context.go

@@ -6,9 +6,18 @@ import (
 	"github.com/google/uuid"
 )
 
+type RecipientType string
+
+const (
+	RecipientTypePerson RecipientType = "person"
+	RecipientTypeTeam   RecipientType = "team"
+	RecipientTypeCustom RecipientType = "custom"
+)
+
 type Context struct {
 	api.Context
 	notificationID uuid.UUID
+	recipientType  RecipientType
 	log            *models.NotificationSendHistory
 }
 
@@ -32,6 +41,10 @@ func (t *Context) WithMessage(message string) {
 	t.log.Body = message
 }
 
+func (t *Context) WithRecipientType(recipientType RecipientType) {
+	t.recipientType = recipientType
+}
+
 func (t *Context) WithError(err string) {
 	t.log.Error = &err
 }
@@ -41,6 +54,7 @@ func (t *Context) WithSource(event string, resourceID uuid.UUID) {
 	t.log.ResourceID = resourceID
 }
 
-func (t *Context) WithPersonID(id *uuid.UUID) {
+func (t *Context) WithPersonID(id *uuid.UUID) *Context {
 	t.log.PersonID = id
+	return t
 }

notification/metrics.go

@@ -0,0 +1,33 @@
+package notification
+
+import "github.com/prometheus/client_golang/prometheus"
+
+func init() {
+	prometheus.MustRegister(notificationSentCounter, notificationSendFailureCounter, notificationSendDuration)
+}
+
+var (
+	notificationSentCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:      "sent_total",
+			Subsystem: "notification",
+			Help:      "Total number of notifications sent",
+		},
+		[]string{"service", "recipient_type", "id"},
+	)
+
+	notificationSendFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:      "send_error_total",
+			Subsystem: "notification",
+			Help:      "Total number of failure notifications sent",
+		},
+		[]string{"service", "recipient_type", "id"},
+	)
+
+	notificationSendDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:      "send_duration_seconds",
+		Subsystem: "notification",
+		Help:      "Duration to send a notification.",
+	}, []string{"service", "recipient_type", "id"})
+)

notification/shoutrrr.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	stripmd "github.com/adityathebe/go-strip-markdown/v2"
 	"github.com/containrrr/shoutrrr"
@@ -42,10 +43,26 @@ func setSystemSMTPCredential(shoutrrrURL string) (string, error) {
 }
 
 func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) error {
+	start := time.Now()
+
+	service, err := send(ctx, connectionName, shoutrrrURL, title, message, properties...)
+	if err != nil {
+		notificationSendFailureCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+		return err
+	}
+
+	notificationSentCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+	notificationSendDuration.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Observe(time.Since(start).Seconds())
+
+	return nil
+}
+
+// send sends a notification and returns the service it sent the notification to
+func send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) (string, error) {
 	if connectionName != "" {
 		connection, err := ctx.HydrateConnection(connectionName)
 		if err != nil {
-			return err
+			return "", err
 		}
 
 		shoutrrrURL = connection.URL
@@ -56,18 +73,18 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 		var err error
 		shoutrrrURL, err = setSystemSMTPCredential(shoutrrrURL)
 		if err != nil {
-			return err
+			return "", err
 		}
 	}
 
 	sender, err := shoutrrr.CreateSender(shoutrrrURL)
 	if err != nil {
-		return fmt.Errorf("failed to create a shoutrrr sender client: %w", err)
+		return "", fmt.Errorf("failed to create a shoutrrr sender client: %w", err)
 	}
 
 	service, _, err := sender.ExtractServiceName(shoutrrrURL)
 	if err != nil {
-		return fmt.Errorf("failed to extract service name: %w", err)
+		return "", fmt.Errorf("failed to extract service name: %w", err)
 	}
 
 	switch service {
@@ -101,7 +118,7 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 	if service == "smtp" {
 		parsedURL, err := url.Parse(shoutrrrURL)
 		if err != nil {
-			return fmt.Errorf("failed to parse shoutrrr URL: %w", err)
+			return "", fmt.Errorf("failed to parse shoutrrr URL: %w", err)
 		}
 
 		query := parsedURL.Query()
@@ -116,17 +133,17 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 		m := mail.New(to, title, message, `text/html; charset="UTF-8"`).
 			SetFrom(fromName, from).
 			SetCredentials(parsedURL.Hostname(), port, parsedURL.User.Username(), password)
-		return m.Send()
+		return service, m.Send()
 	}
 
 	sendErrors := sender.Send(message, params)
 	for _, err := range sendErrors {
 		if err != nil {
-			return fmt.Errorf("error publishing notification (service=%s): %w", service, err)
+			return "", fmt.Errorf("error publishing notification (service=%s): %w", service, err)
 		}
 	}
 
-	return nil
+	return service, nil
 }
 
 // injectTitleIntoProperties adds the given title to the shoutrrr properties if it's not already set.