Add standard prometheus monitoring

Metrics available under `request_duration_seconds`.
Reuse middleware.

cmd/paymentsvc/main.go

@@ -63,7 +63,7 @@ func main() {
 	errc := make(chan error)
 	ctx := context.Background()
 
-	handler, logger := payment.WireUp(ctx, float32(*declineAmount), tracer)
+	handler, logger := payment.WireUp(ctx, float32(*declineAmount), tracer, ServiceName)
 
 	// Create and launch the HTTP server.
 	go func() {

component_test.go

@@ -16,7 +16,7 @@ func TestComponent(t *testing.T) {
 	// Mechanical stuff.
 	ctx := context.Background()
 
-	handler, logger := WireUp(ctx, float32(99.99), opentracing.GlobalTracer())
+	handler, logger := WireUp(ctx, float32(99.99), opentracing.GlobalTracer(), "test")
 
 	ts := httptest.NewServer(handler)
 	defer ts.Close()

middlewares.go → logging.go

@@ -6,9 +6,6 @@ import (
 	"time"
 )
 
-// Middleware decorates a service.
-type Middleware func(Service) Service
-
 // LoggingMiddleware logs method calls, parameters, results, and elapsed time.
 func LoggingMiddleware(logger log.Logger) Middleware {
 	return func(next Service) Service {

middleware/instrument.go

@@ -0,0 +1,94 @@
+package middleware
+
+import (
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/felixge/httpsnoop"
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	HTTPLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "request_duration_seconds",
+		Help:    "Time (in seconds) spent serving HTTP requests.",
+		Buckets: prometheus.DefBuckets,
+	}, []string{"service", "method", "route", "status_code"})
+)
+
+func init() {
+	prometheus.MustRegister(HTTPLatency)
+}
+
+// RouteMatcher matches routes
+type RouteMatcher interface {
+	Match(*http.Request, *mux.RouteMatch) bool
+}
+
+// Instrument is a Middleware which records timings for every HTTP request
+type Instrument struct {
+	RouteMatcher RouteMatcher
+	Duration     *prometheus.HistogramVec
+	Service      string
+}
+
+// Wrap implements middleware.Interface
+func (i Instrument) Wrap(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		begin := time.Now()
+		interceptor := httpsnoop.CaptureMetrics(next, w, r)
+		route := i.getRouteName(r)
+		var (
+			status = strconv.Itoa(interceptor.Code)
+			took   = time.Since(begin)
+		)
+		i.Duration.WithLabelValues(i.Service, r.Method, route, status).Observe(took.Seconds())
+	})
+}
+
+// Return a name identifier for ths request.  There are three options:
+//   1. The request matches a gorilla mux route, with a name.  Use that.
+//   2. The request matches an unamed gorilla mux router.  Munge the path
+//      template such that templates like '/api/{org}/foo' come out as
+//      'api_org_foo'.
+//   3. The request doesn't match a mux route.  Munge the Path in the same
+//      manner as (2).
+// We do all this as we do not wish to emit high cardinality labels to
+// prometheus.
+func (i Instrument) getRouteName(r *http.Request) string {
+	var routeMatch mux.RouteMatch
+	if i.RouteMatcher != nil && i.RouteMatcher.Match(r, &routeMatch) {
+		if name := routeMatch.Route.GetName(); name != "" {
+			return name
+		}
+		if tmpl, err := routeMatch.Route.GetPathTemplate(); err == nil {
+			return MakeLabelValue(tmpl)
+		}
+	}
+	return MakeLabelValue(r.URL.Path)
+}
+
+var invalidChars = regexp.MustCompile(`[^a-zA-Z0-9]+`)
+
+// MakeLabelValue converts a Gorilla mux path to a string suitable for use in
+// a Prometheus label value.
+func MakeLabelValue(path string) string {
+	// Convert non-alnums to underscores.
+	result := invalidChars.ReplaceAllString(path, "_")
+
+	// Trim leading and trailing underscores.
+	result = strings.Trim(result, "_")
+
+	// Make it all lowercase
+	result = strings.ToLower(result)
+
+	// Special case.
+	if result == "" {
+		result = "root"
+	}
+	return result
+}

middleware/middleware.go

@@ -0,0 +1,33 @@
+package middleware
+
+import (
+	"net/http"
+)
+
+// Interface is the shared contract for all middlesware, and allows middlesware
+// to wrap handlers.
+type Interface interface {
+	Wrap(http.Handler) http.Handler
+}
+
+// Func is to Interface as http.HandlerFunc is to http.Handler
+type Func func(http.Handler) http.Handler
+
+// Wrap implements Interface
+func (m Func) Wrap(next http.Handler) http.Handler {
+	return m(next)
+}
+
+// Identity is an Interface which doesn't do anything.
+var Identity Interface = Func(func(h http.Handler) http.Handler { return h })
+
+// Merge produces a middleware that applies multiple middlesware in turn;
+// ie Merge(f,g,h).Wrap(handler) == f.Wrap(g.Wrap(h.Wrap(handler)))
+func Merge(middlesware ...Interface) Interface {
+	return Func(func(next http.Handler) http.Handler {
+		for i := len(middlesware) - 1; i >= 0; i-- {
+			next = middlesware[i].Wrap(next)
+		}
+		return next
+	})
+}

scripts/build.sh

@@ -23,6 +23,7 @@ CODE_DIR=$(cd $SCRIPT_DIR/..; pwd)
 echo $CODE_DIR
 
 cp -r $CODE_DIR/cmd/ $CODE_DIR/docker/payment/cmd/
+cp -r $CODE_DIR/middleware/ $CODE_DIR/docker/payment/middleware/
 cp $CODE_DIR/*.go $CODE_DIR/docker/payment/
 mkdir $CODE_DIR/docker/payment/vendor && cp $CODE_DIR/vendor/manifest $CODE_DIR/docker/payment/vendor/
 

service.go

@@ -6,6 +6,9 @@ import (
 	"time"
 )
 
+// Middleware decorates a service.
+type Middleware func(Service) Service
+
 type Service interface {
 	Authorise(total float32) (Authorisation, error) // GET /paymentAuth
 	Health() []Health                               // GET /health

transport.go

@@ -19,7 +19,7 @@ import (
 )
 
 // MakeHTTPHandler mounts the endpoints into a REST-y HTTP handler.
-func MakeHTTPHandler(ctx context.Context, e Endpoints, logger log.Logger, tracer stdopentracing.Tracer) http.Handler {
+func MakeHTTPHandler(ctx context.Context, e Endpoints, logger log.Logger, tracer stdopentracing.Tracer) *mux.Router {
 	r := mux.NewRouter().StrictSlash(false)
 	options := []httptransport.ServerOption{
 		httptransport.ServerErrorLogger(logger),

vendor/manifest

@@ -94,6 +94,14 @@
 			"branch": "master",
 			"notests": true
 		},
+		{
+			"importpath": "github.com/felixge/httpsnoop",
+			"repository": "https://github.com/felixge/httpsnoop",
+			"vcs": "git",
+			"revision": "287b56e9e314227d3113c7c6b434d31aec68089d",
+			"branch": "master",
+			"notests": true
+		},
 		{
 			"importpath": "github.com/go-kit/kit/circuitbreaker",
 			"repository": "https://github.com/go-kit/kit",

wiring.go

@@ -10,9 +10,10 @@ import (
 	kitprometheus "github.com/go-kit/kit/metrics/prometheus"
 	stdopentracing "github.com/opentracing/opentracing-go"
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
+	"github.com/microservices-demo/payment/middleware"
 )
 
-func WireUp(ctx context.Context, declineAmount float32, tracer stdopentracing.Tracer) (http.Handler, log.Logger) {
+func WireUp(ctx context.Context, declineAmount float32, tracer stdopentracing.Tracer, serviceName string) (http.Handler, log.Logger) {
 	// Log domain.
 	var logger log.Logger
 	{
@@ -49,6 +50,18 @@ func WireUp(ctx context.Context, declineAmount float32, tracer stdopentracing.Tr
 	// Endpoint domain.
 	endpoints := MakeEndpoints(service, tracer)
 
-	handler := MakeHTTPHandler(ctx, endpoints, logger, tracer)
+	router := MakeHTTPHandler(ctx, endpoints, logger, tracer)
+
+	httpMiddleware := []middleware.Interface{
+		middleware.Instrument{
+			Duration:     middleware.HTTPLatency,
+			RouteMatcher: router,
+			Service:      serviceName,
+		},
+	}
+
+	// Handler
+	handler := middleware.Merge(httpMiddleware...).Wrap(router)
+
 	return handler, logger
 }