add more precise duration buckets and upstream auth label (#22)

* add more precise duration buckets and upstream auth label to metrics

pkg/auth/authenticator.go

@@ -149,21 +149,21 @@ func (a *Authenticator) UpdateCache(c client.Client, ctx context.Context, readOn
 
 	t := time.Now()
 	err = c.List(ctx, tokens)
-	fetchObjectListLatency.With(KindLabel(MetricsKindAccessToken)).Observe(time.Since(t).Seconds())
+	fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindAccessToken)).Observe(time.Since(t).Seconds())
 	if err != nil {
 		return err
 	}
 
 	t = time.Now()
 	err = c.List(ctx, bindings)
-	fetchObjectListLatency.With(KindLabel(MetricsKindWebserviceAccessBinding)).Observe(time.Since(t).Seconds())
+	fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindWebserviceAccessBinding)).Observe(time.Since(t).Seconds())
 	if err != nil {
 		return err
 	}
 
 	t = time.Now()
 	err = c.List(ctx, webservices)
-	fetchObjectListLatency.With(KindLabel(MetricsKindWebservice)).Observe(time.Since(t).Seconds())
+	fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindWebservice)).Observe(time.Since(t).Seconds())
 	if err != nil {
 		return err
 	}
@@ -175,7 +175,7 @@ func (a *Authenticator) UpdateCache(c client.Client, ctx context.Context, readOn
 		// client.MatchingLabels{"cerberus.snappcloud.io/secret": "true"},
 		listOpts,
 	)
-	fetchObjectListLatency.With(KindLabel(MetricsKindSecret)).Observe(time.Since(t).Seconds())
+	fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindSecret)).Observe(time.Since(t).Seconds())
 	if err != nil {
 		return err
 	}
@@ -332,13 +332,15 @@ func (a *Authenticator) readService(wsvc string) (bool, CerberusReason, Services
 func (a *Authenticator) Check(ctx context.Context, request *Request) (*Response, error) {
 
 	wsvc := request.Context["webservice"]
+	request.Context[HasUpstreamAuth] = "false"
 	var extraHeaders ExtraHeaders
 	var httpStatusCode int
 
 	ok, reason, wsvcCacheEntry := a.readService(wsvc)
 	if ok {
 		ok, reason, extraHeaders = a.TestAccess(request, wsvcCacheEntry)
 		if ok && hasUpstreamAuth(wsvcCacheEntry) {
+			request.Context[HasUpstreamAuth] = "true"
 			ok, reason = a.checkServiceUpstreamAuth(wsvcCacheEntry, request, &extraHeaders)
 		}
 	}
@@ -449,7 +451,7 @@ func (a *Authenticator) checkServiceUpstreamAuth(service ServicesCacheEntry, req
 		return false, CerberusReasonUpstreamAuthFailed
 	}
 
-	labels := StatusLabel(resp.StatusCode)
+	labels := AddStatusLabel(nil, resp.StatusCode)
 	upstreamAuthRequestDuration.With(labels).Observe(reqDuration.Seconds())
 
 	if resp.StatusCode != http.StatusOK {

pkg/auth/metrics.go

@@ -9,6 +9,7 @@ import (
 const (
 	CerberusReasonLabel      = "cerberus_reason"
 	CheckRequestVersionLabel = "check_request_version"
+	HasUpstreamAuth		     = "upstream_auth_enabled"
 	ObjectKindLabel          = "kind"
 
 	MetricsKindSecret                  = "secret"
@@ -22,15 +23,15 @@ const (
 )
 
 var (
-	DurationBuckets      = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5}
+	DurationBuckets      = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.000125, 0.00015, 0.000175, 0.0002,  0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5}
 	SmallDurationBuckets = []float64{0.0000001, 0.000001, 0.0000025, 0.000005, 0.00001, 0.000025, 0.00005, 0.0001, 0.001, 0.01, 0.05, 0.1}
 
 	reqCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "check_request_count",
 			Help: "CheckRequest count",
 		},
-		[]string{CerberusReasonLabel, CheckRequestVersionLabel},
+		[]string{CerberusReasonLabel, CheckRequestVersionLabel, HasUpstreamAuth},
 	)
 
 	reqLatency = prometheus.NewHistogramVec(
@@ -39,7 +40,7 @@ var (
 			Help:    "CheckRequest durations (response times)",
 			Buckets: DurationBuckets,
 		},
-		[]string{CerberusReasonLabel, CheckRequestVersionLabel},
+		[]string{CerberusReasonLabel, CheckRequestVersionLabel, HasUpstreamAuth},
 	)
 
 	cacheUpdateCount = prometheus.NewCounter(
@@ -136,20 +137,34 @@ func init() {
 	)
 }
 
-func ReasonLabel(reason CerberusReason) prometheus.Labels {
-	labels := prometheus.Labels{}
+func AddReasonLabel(labels prometheus.Labels, reason CerberusReason) prometheus.Labels {
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
 	labels[CerberusReasonLabel] = string(reason)
 	return labels
 }
 
-func KindLabel(kind string) prometheus.Labels {
-	labels := prometheus.Labels{}
+func AddKindLabel(labels prometheus.Labels, kind string) prometheus.Labels {
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
 	labels[ObjectKindLabel] = kind
 	return labels
 }
 
-func StatusLabel(status int) prometheus.Labels {
-	labels := prometheus.Labels{}
+func AddStatusLabel(labels prometheus.Labels, status int) prometheus.Labels {
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
 	labels[StatusCode] = strconv.Itoa(status)
 	return labels
 }
+
+func AddUpstreamAuthLabel(labels prometheus.Labels, hasUpstreamAuth string) prometheus.Labels {
+	if labels == nil {
+		labels = prometheus.Labels{}
+	}
+	labels[HasUpstreamAuth] = hasUpstreamAuth
+	return labels
+}

pkg/auth/server.go

@@ -45,7 +45,8 @@ func (a *authV2) Check(ctx context.Context, check *CheckRequestV2) (*CheckRespon
 
 	// update metrics
 	reason := CerberusReason(response.Response.Header.Get("X-Cerberus-Reason"))
-	labels := ReasonLabel(reason)
+	labels := AddReasonLabel(nil, reason)
+	labels = AddUpstreamAuthLabel(labels, request.Context[HasUpstreamAuth])
 	labels[CheckRequestVersionLabel] = MetricsCheckRequestVersion2
 	reqCount.With(labels).Inc()
 	reqLatency.With(labels).Observe(time.Since(reqStartTime).Seconds())
@@ -70,7 +71,8 @@ func (a *authV3) Check(ctx context.Context, check *CheckRequestV3) (*CheckRespon
 
 	// update metrics
 	reason := CerberusReason(response.Response.Header.Get("X-Cerberus-Reason"))
-	labels := ReasonLabel(reason)
+	labels := AddReasonLabel(nil, reason)
+	labels = AddUpstreamAuthLabel(labels, request.Context[HasUpstreamAuth])
 	labels[CheckRequestVersionLabel] = MetricsCheckRequestVersion3
 	reqCount.With(labels).Inc()
 	reqLatency.With(labels).Observe(time.Since(reqStartTime).Seconds())