From 270865966bfa39b02e40dad44e53716a1fc48bf6 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Mon, 13 Mar 2017 13:44:50 +0000 Subject: [PATCH] Use standard prometheus naming Reuse middleware. Add histogram metric. Measure all http calls. --- api/transport.go | 4 +- glide.lock | 6 ++- glide.yaml | 1 + main.go | 16 ++++++- middleware/instrument.go | 94 ++++++++++++++++++++++++++++++++++++++++ middleware/middleware.go | 33 ++++++++++++++ 6 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 middleware/instrument.go create mode 100644 middleware/middleware.go diff --git a/api/transport.go b/api/transport.go index cf59247d..6440d5cd 100644 --- a/api/transport.go +++ b/api/transport.go @@ -9,10 +9,10 @@ import ( "net/http" "strings" + "github.com/go-kit/kit/circuitbreaker" "github.com/go-kit/kit/log" "github.com/go-kit/kit/tracing/opentracing" httptransport "github.com/go-kit/kit/transport/http" - "github.com/go-kit/kit/circuitbreaker" "github.com/gorilla/mux" "github.com/microservices-demo/user/users" stdopentracing "github.com/opentracing/opentracing-go" @@ -25,7 +25,7 @@ var ( ) // MakeHTTPHandler mounts the endpoints into a REST-y HTTP handler. -func MakeHTTPHandler(ctx context.Context, e Endpoints, logger log.Logger, tracer stdopentracing.Tracer) http.Handler { +func MakeHTTPHandler(ctx context.Context, e Endpoints, logger log.Logger, tracer stdopentracing.Tracer) *mux.Router { r := mux.NewRouter().StrictSlash(false) options := []httptransport.ServerOption{ httptransport.ServerErrorLogger(logger), diff --git a/glide.lock b/glide.lock index 166a2d0b..98a60fd5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 32956a03ce39fe62b3bceebd8e22e79651fd3f735597be654e8729ccd731284f -updated: 2017-01-17T17:20:40.444795777-08:00 +hash: b6132ca44a02e5b84ad8f27e24283918ff7c566256ebd76fac73564646aec258 +updated: 2017-03-13T12:37:17.157991208Z imports: - name: github.com/afex/hystrix-go version: 39520ddd07a9d9a071d615f7476798659f5a3b89 @@ -27,6 +27,8 @@ imports: version: bb955e01b9346ac19dc29eb16586c90ded99a98c - name: github.com/eapache/queue version: 44cc805cf13205b55f69e14bcb69867d1ae92f98 +- name: github.com/felixge/httpsnoop + version: 287b56e9e314227d3113c7c6b434d31aec68089d - name: github.com/go-kit/kit version: 988c05d06d8ee3a9c13782f0e49b2c6e4726388d subpackages: diff --git a/glide.yaml b/glide.yaml index 26ed34fa..afa742b5 100644 --- a/glide.yaml +++ b/glide.yaml @@ -26,3 +26,4 @@ import: - package: github.com/afex/hystrix-go subpackages: - hystrix +- package: github.com/felixge/httpsnoop diff --git a/main.go b/main.go index d820e379..7f686317 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/microservices-demo/user/api" "github.com/microservices-demo/user/db" "github.com/microservices-demo/user/db/mongodb" + "github.com/microservices-demo/user/middleware" stdopentracing "github.com/opentracing/opentracing-go" zipkin "github.com/openzipkin/zipkin-go-opentracing" stdprometheus "github.com/prometheus/client_golang/prometheus" @@ -120,10 +121,23 @@ func main() { // Endpoint domain. endpoints := api.MakeEndpoints(service, tracer) + // HTTP router + router := api.MakeHTTPHandler(ctx, endpoints, logger, tracer) + + httpMiddleware := []middleware.Interface{ + middleware.Instrument{ + Duration: middleware.HTTPLatency, + RouteMatcher: router, + Service: ServiceName, + }, + } + + // Handler + handler := middleware.Merge(httpMiddleware...).Wrap(router) + // Create and launch the HTTP server. go func() { logger.Log("transport", "HTTP", "port", port) - handler := api.MakeHTTPHandler(ctx, endpoints, logger, tracer) errc <- http.ListenAndServe(fmt.Sprintf(":%v", port), handler) }() diff --git a/middleware/instrument.go b/middleware/instrument.go new file mode 100644 index 00000000..3ec1b921 --- /dev/null +++ b/middleware/instrument.go @@ -0,0 +1,94 @@ +package middleware + +import ( + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/felixge/httpsnoop" + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + HTTPLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "Time (in seconds) spent serving HTTP requests.", + Buckets: prometheus.DefBuckets, + }, []string{"service", "method", "route", "status_code"}) +) + +func init() { + prometheus.MustRegister(HTTPLatency) +} + +// RouteMatcher matches routes +type RouteMatcher interface { + Match(*http.Request, *mux.RouteMatch) bool +} + +// Instrument is a Middleware which records timings for every HTTP request +type Instrument struct { + RouteMatcher RouteMatcher + Duration *prometheus.HistogramVec + Service string +} + +// Wrap implements middleware.Interface +func (i Instrument) Wrap(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + begin := time.Now() + interceptor := httpsnoop.CaptureMetrics(next, w, r) + route := i.getRouteName(r) + var ( + status = strconv.Itoa(interceptor.Code) + took = time.Since(begin) + ) + i.Duration.WithLabelValues(i.Service, r.Method, route, status).Observe(took.Seconds()) + }) +} + +// Return a name identifier for ths request. There are three options: +// 1. The request matches a gorilla mux route, with a name. Use that. +// 2. The request matches an unamed gorilla mux router. Munge the path +// template such that templates like '/api/{org}/foo' come out as +// 'api_org_foo'. +// 3. The request doesn't match a mux route. Munge the Path in the same +// manner as (2). +// We do all this as we do not wish to emit high cardinality labels to +// prometheus. +func (i Instrument) getRouteName(r *http.Request) string { + var routeMatch mux.RouteMatch + if i.RouteMatcher != nil && i.RouteMatcher.Match(r, &routeMatch) { + if name := routeMatch.Route.GetName(); name != "" { + return name + } + if tmpl, err := routeMatch.Route.GetPathTemplate(); err == nil { + return MakeLabelValue(tmpl) + } + } + return MakeLabelValue(r.URL.Path) +} + +var invalidChars = regexp.MustCompile(`[^a-zA-Z0-9]+`) + +// MakeLabelValue converts a Gorilla mux path to a string suitable for use in +// a Prometheus label value. +func MakeLabelValue(path string) string { + // Convert non-alnums to underscores. + result := invalidChars.ReplaceAllString(path, "_") + + // Trim leading and trailing underscores. + result = strings.Trim(result, "_") + + // Make it all lowercase + result = strings.ToLower(result) + + // Special case. + if result == "" { + result = "root" + } + return result +} diff --git a/middleware/middleware.go b/middleware/middleware.go new file mode 100644 index 00000000..ad8925ac --- /dev/null +++ b/middleware/middleware.go @@ -0,0 +1,33 @@ +package middleware + +import ( + "net/http" +) + +// Interface is the shared contract for all middlesware, and allows middlesware +// to wrap handlers. +type Interface interface { + Wrap(http.Handler) http.Handler +} + +// Func is to Interface as http.HandlerFunc is to http.Handler +type Func func(http.Handler) http.Handler + +// Wrap implements Interface +func (m Func) Wrap(next http.Handler) http.Handler { + return m(next) +} + +// Identity is an Interface which doesn't do anything. +var Identity Interface = Func(func(h http.Handler) http.Handler { return h }) + +// Merge produces a middleware that applies multiple middlesware in turn; +// ie Merge(f,g,h).Wrap(handler) == f.Wrap(g.Wrap(h.Wrap(handler))) +func Merge(middlesware ...Interface) Interface { + return Func(func(next http.Handler) http.Handler { + for i := len(middlesware) - 1; i >= 0; i-- { + next = middlesware[i].Wrap(next) + } + return next + }) +}