feat: add more metric labels & a new metric

events/notifications.go

@@ -218,7 +218,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	if props.PersonID != nil {
-		ctx.WithPersonID(props.PersonID)
+		ctx.WithPersonID(props.PersonID).WithRecipientType(pkgNotification.RecipientTypePerson)
 		var emailAddress string
 		if err := ctx.DB().Model(&models.Person{}).Select("email").Where("id = ?", props.PersonID).Find(&emailAddress).Error; err != nil {
 			return fmt.Errorf("failed to get email of person(id=%s); %v", props.PersonID, err)
@@ -229,6 +229,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	if props.TeamID != "" {
+		ctx.WithRecipientType(pkgNotification.RecipientTypeTeam)
 		teamSpec, err := teams.GetTeamSpec(ctx.Context, props.TeamID)
 		if err != nil {
 			return fmt.Errorf("failed to get team(id=%s); %v", props.TeamID, err)
@@ -248,6 +249,7 @@ func sendNotification(ctx *pkgNotification.Context, event api.Event) error {
 	}
 
 	for _, cn := range notification.CustomNotifications {
+		ctx.WithRecipientType(pkgNotification.RecipientTypeCustom)
 		if cn.Name != props.NotificationName {
 			continue
 		}

notification/context.go

@@ -6,9 +6,18 @@ import (
 	"github.com/google/uuid"
 )
 
+type RecipientType string
+
+const (
+	RecipientTypePerson RecipientType = "person"
+	RecipientTypeTeam   RecipientType = "team"
+	RecipientTypeCustom RecipientType = "custom"
+)
+
 type Context struct {
 	*api.Context
 	notificationID uuid.UUID
+	recipientType  RecipientType
 	log            *models.NotificationSendHistory
 }
 
@@ -32,6 +41,10 @@ func (t *Context) WithMessage(message string) {
 	t.log.Body = message
 }
 
+func (t *Context) WithRecipientType(recipientType RecipientType) {
+	t.recipientType = recipientType
+}
+
 func (t *Context) WithError(err string) {
 	t.log.Error = &err
 }
@@ -41,6 +54,7 @@ func (t *Context) WithSource(event string, resourceID uuid.UUID) {
 	t.log.ResourceID = resourceID
 }
 
-func (t *Context) WithPersonID(id *uuid.UUID) {
+func (t *Context) WithPersonID(id *uuid.UUID) *Context {
 	t.log.PersonID = id
+	return t
 }

notification/metrics.go

@@ -3,7 +3,7 @@ package notification
 import "github.com/prometheus/client_golang/prometheus"
 
 func init() {
-	prometheus.MustRegister(notificationSentCounter, notificationSendDuration)
+	prometheus.MustRegister(notificationSentCounter, notificationSendFailureCounter, notificationSendDuration)
 }
 
 var (
@@ -13,12 +13,21 @@ var (
 			Subsystem: "notification",
 			Help:      "Total number of notifications sent",
 		},
-		[]string{"service"},
+		[]string{"service", "recipient_type", "id"},
+	)
+
+	notificationSendFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name:      "send_error_total",
+			Subsystem: "notification",
+			Help:      "Total number of failure notifications sent",
+		},
+		[]string{"service", "recipient_type", "id"},
 	)
 
 	notificationSendDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Name:      "send_duration_seconds",
 		Subsystem: "notification",
 		Help:      "Duration to send a notification.",
-	}, []string{"service"})
+	}, []string{"service", "recipient_type", "id"})
 )

notification/shoutrrr.go

@@ -44,10 +44,24 @@ func setSystemSMTPCredential(shoutrrrURL string) (string, error) {
 func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) error {
 	start := time.Now()
 
+	service, err := send(ctx, connectionName, shoutrrrURL, title, message, properties...)
+	if err != nil {
+		notificationSendFailureCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+		return err
+	}
+
+	notificationSentCounter.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Inc()
+	notificationSendDuration.WithLabelValues(service, string(ctx.recipientType), ctx.notificationID.String()).Observe(time.Since(start).Seconds())
+
+	return nil
+}
+
+// send sends a notification and returns the service it sent the notification to
+func send(ctx *Context, connectionName, shoutrrrURL, title, message string, properties ...map[string]string) (string, error) {
 	if connectionName != "" {
 		connection, err := ctx.HydrateConnection(connectionName)
 		if err != nil {
-			return err
+			return "", err
 		}
 
 		shoutrrrURL = connection.URL
@@ -58,18 +72,18 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 		var err error
 		shoutrrrURL, err = setSystemSMTPCredential(shoutrrrURL)
 		if err != nil {
-			return err
+			return "", err
 		}
 	}
 
 	sender, err := shoutrrr.CreateSender(shoutrrrURL)
 	if err != nil {
-		return fmt.Errorf("failed to create a shoutrrr sender client: %w", err)
+		return "", fmt.Errorf("failed to create a shoutrrr sender client: %w", err)
 	}
 
 	service, _, err := sender.ExtractServiceName(shoutrrrURL)
 	if err != nil {
-		return fmt.Errorf("failed to extract service name: %w", err)
+		return "", fmt.Errorf("failed to extract service name: %w", err)
 	}
 
 	switch service {
@@ -103,7 +117,7 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 	if service == "smtp" {
 		parsedURL, err := url.Parse(shoutrrrURL)
 		if err != nil {
-			return fmt.Errorf("failed to parse shoutrrr URL: %w", err)
+			return "", fmt.Errorf("failed to parse shoutrrr URL: %w", err)
 		}
 
 		query := parsedURL.Query()
@@ -117,20 +131,17 @@ func Send(ctx *Context, connectionName, shoutrrrURL, title, message string, prop
 		m := mail.New(to, title, message, `text/html; charset="UTF-8"`).
 			SetFrom(from).
 			SetCredentials(parsedURL.Hostname(), port, parsedURL.User.Username(), password)
-		return m.Send()
+		return service, m.Send()
 	}
 
 	sendErrors := sender.Send(message, params)
 	for _, err := range sendErrors {
 		if err != nil {
-			return fmt.Errorf("error publishing notification (service=%s): %w", service, err)
+			return "", fmt.Errorf("error publishing notification (service=%s): %w", service, err)
 		}
 	}
 
-	notificationSentCounter.WithLabelValues(service).Inc()
-	notificationSendDuration.WithLabelValues(service).Observe(time.Since(start).Seconds())
-
-	return nil
+	return service, nil
 }
 
 // injectTitleIntoProperties adds the given title to the shoutrrr properties if it's not already set.