From d6d43065e6c435e9f4f2a2c8bb66480089861ade Mon Sep 17 00:00:00 2001 From: Masoud Golestaneh <52936110+Cypherspark@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:47:18 +0330 Subject: [PATCH] add more precise duration buckets and upstream auth label (#22) * add more precise duration buckets and upstream auth label to metrics --- pkg/auth/authenticator.go | 12 +++++++----- pkg/auth/metrics.go | 33 ++++++++++++++++++++++++--------- pkg/auth/server.go | 6 ++++-- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/pkg/auth/authenticator.go b/pkg/auth/authenticator.go index 96aebc4..903cc8c 100644 --- a/pkg/auth/authenticator.go +++ b/pkg/auth/authenticator.go @@ -149,21 +149,21 @@ func (a *Authenticator) UpdateCache(c client.Client, ctx context.Context, readOn t := time.Now() err = c.List(ctx, tokens) - fetchObjectListLatency.With(KindLabel(MetricsKindAccessToken)).Observe(time.Since(t).Seconds()) + fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindAccessToken)).Observe(time.Since(t).Seconds()) if err != nil { return err } t = time.Now() err = c.List(ctx, bindings) - fetchObjectListLatency.With(KindLabel(MetricsKindWebserviceAccessBinding)).Observe(time.Since(t).Seconds()) + fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindWebserviceAccessBinding)).Observe(time.Since(t).Seconds()) if err != nil { return err } t = time.Now() err = c.List(ctx, webservices) - fetchObjectListLatency.With(KindLabel(MetricsKindWebservice)).Observe(time.Since(t).Seconds()) + fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindWebservice)).Observe(time.Since(t).Seconds()) if err != nil { return err } @@ -175,7 +175,7 @@ func (a *Authenticator) UpdateCache(c client.Client, ctx context.Context, readOn // client.MatchingLabels{"cerberus.snappcloud.io/secret": "true"}, listOpts, ) - fetchObjectListLatency.With(KindLabel(MetricsKindSecret)).Observe(time.Since(t).Seconds()) + fetchObjectListLatency.With(AddKindLabel(nil, MetricsKindSecret)).Observe(time.Since(t).Seconds()) if err != nil { return err } @@ -332,6 +332,7 @@ func (a *Authenticator) readService(wsvc string) (bool, CerberusReason, Services func (a *Authenticator) Check(ctx context.Context, request *Request) (*Response, error) { wsvc := request.Context["webservice"] + request.Context[HasUpstreamAuth] = "false" var extraHeaders ExtraHeaders var httpStatusCode int @@ -339,6 +340,7 @@ func (a *Authenticator) Check(ctx context.Context, request *Request) (*Response, if ok { ok, reason, extraHeaders = a.TestAccess(request, wsvcCacheEntry) if ok && hasUpstreamAuth(wsvcCacheEntry) { + request.Context[HasUpstreamAuth] = "true" ok, reason = a.checkServiceUpstreamAuth(wsvcCacheEntry, request, &extraHeaders) } } @@ -449,7 +451,7 @@ func (a *Authenticator) checkServiceUpstreamAuth(service ServicesCacheEntry, req return false, CerberusReasonUpstreamAuthFailed } - labels := StatusLabel(resp.StatusCode) + labels := AddStatusLabel(nil, resp.StatusCode) upstreamAuthRequestDuration.With(labels).Observe(reqDuration.Seconds()) if resp.StatusCode != http.StatusOK { diff --git a/pkg/auth/metrics.go b/pkg/auth/metrics.go index be4be60..d828058 100644 --- a/pkg/auth/metrics.go +++ b/pkg/auth/metrics.go @@ -9,6 +9,7 @@ import ( const ( CerberusReasonLabel = "cerberus_reason" CheckRequestVersionLabel = "check_request_version" + HasUpstreamAuth = "upstream_auth_enabled" ObjectKindLabel = "kind" MetricsKindSecret = "secret" @@ -22,7 +23,7 @@ const ( ) var ( - DurationBuckets = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5} + DurationBuckets = []float64{0.000005, 0.00001, 0.000015, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.000125, 0.00015, 0.000175, 0.0002, 0.00025, .0005, .001, .002, .003, .004, .005, .006, .007, .008, .009, .01, .02, .05, .1, 1, 2.5, 5} SmallDurationBuckets = []float64{0.0000001, 0.000001, 0.0000025, 0.000005, 0.00001, 0.000025, 0.00005, 0.0001, 0.001, 0.01, 0.05, 0.1} reqCount = prometheus.NewCounterVec( @@ -30,7 +31,7 @@ var ( Name: "check_request_count", Help: "CheckRequest count", }, - []string{CerberusReasonLabel, CheckRequestVersionLabel}, + []string{CerberusReasonLabel, CheckRequestVersionLabel, HasUpstreamAuth}, ) reqLatency = prometheus.NewHistogramVec( @@ -39,7 +40,7 @@ var ( Help: "CheckRequest durations (response times)", Buckets: DurationBuckets, }, - []string{CerberusReasonLabel, CheckRequestVersionLabel}, + []string{CerberusReasonLabel, CheckRequestVersionLabel, HasUpstreamAuth}, ) cacheUpdateCount = prometheus.NewCounter( @@ -136,20 +137,34 @@ func init() { ) } -func ReasonLabel(reason CerberusReason) prometheus.Labels { - labels := prometheus.Labels{} +func AddReasonLabel(labels prometheus.Labels, reason CerberusReason) prometheus.Labels { + if labels == nil { + labels = prometheus.Labels{} + } labels[CerberusReasonLabel] = string(reason) return labels } -func KindLabel(kind string) prometheus.Labels { - labels := prometheus.Labels{} +func AddKindLabel(labels prometheus.Labels, kind string) prometheus.Labels { + if labels == nil { + labels = prometheus.Labels{} + } labels[ObjectKindLabel] = kind return labels } -func StatusLabel(status int) prometheus.Labels { - labels := prometheus.Labels{} +func AddStatusLabel(labels prometheus.Labels, status int) prometheus.Labels { + if labels == nil { + labels = prometheus.Labels{} + } labels[StatusCode] = strconv.Itoa(status) return labels } + +func AddUpstreamAuthLabel(labels prometheus.Labels, hasUpstreamAuth string) prometheus.Labels { + if labels == nil { + labels = prometheus.Labels{} + } + labels[HasUpstreamAuth] = hasUpstreamAuth + return labels +} diff --git a/pkg/auth/server.go b/pkg/auth/server.go index 957c201..a2a85c7 100644 --- a/pkg/auth/server.go +++ b/pkg/auth/server.go @@ -45,7 +45,8 @@ func (a *authV2) Check(ctx context.Context, check *CheckRequestV2) (*CheckRespon // update metrics reason := CerberusReason(response.Response.Header.Get("X-Cerberus-Reason")) - labels := ReasonLabel(reason) + labels := AddReasonLabel(nil, reason) + labels = AddUpstreamAuthLabel(labels, request.Context[HasUpstreamAuth]) labels[CheckRequestVersionLabel] = MetricsCheckRequestVersion2 reqCount.With(labels).Inc() reqLatency.With(labels).Observe(time.Since(reqStartTime).Seconds()) @@ -70,7 +71,8 @@ func (a *authV3) Check(ctx context.Context, check *CheckRequestV3) (*CheckRespon // update metrics reason := CerberusReason(response.Response.Header.Get("X-Cerberus-Reason")) - labels := ReasonLabel(reason) + labels := AddReasonLabel(nil, reason) + labels = AddUpstreamAuthLabel(labels, request.Context[HasUpstreamAuth]) labels[CheckRequestVersionLabel] = MetricsCheckRequestVersion3 reqCount.With(labels).Inc() reqLatency.With(labels).Observe(time.Since(reqStartTime).Seconds())