Skip to content

Commit

Permalink
added prometheus metrics for api calls (kubernetes#1501)
Browse files Browse the repository at this point in the history
  • Loading branch information
cheld authored and rf232 committed Dec 8, 2016
1 parent ce5f780 commit ebff081
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 0 deletions.
3 changes: 3 additions & 0 deletions src/app/backend/dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/kubernetes/dashboard/src/app/backend/client"
"github.com/kubernetes/dashboard/src/app/backend/handler"
"github.com/spf13/pflag"
"github.com/prometheus/client_golang/prometheus"

)

var (
Expand Down Expand Up @@ -77,6 +79,7 @@ func main() {
http.Handle("/api/", handler.CreateHTTPAPIHandler(apiserverClient, heapsterRESTClient, config))
// TODO(maciaszczykm): Move to /appConfig.json as it was discussed in #640.
http.Handle("/api/appConfig.json", handler.AppHandler(handler.ConfigHandler))
http.Handle("/metrics",prometheus.Handler())
log.Print(http.ListenAndServe(fmt.Sprintf(":%d", *argPort), nil))
}

Expand Down
27 changes: 27 additions & 0 deletions src/app/backend/handler/apihandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import (
clientK8s "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
"k8s.io/kubernetes/pkg/runtime"
utilnet "k8s.io/kubernetes/pkg/util/net"
)

const (
Expand All @@ -79,6 +80,29 @@ type APIHandler struct {
verber common.ResourceVerber
}

func wsMetrics(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
startTime := time.Now()
verb := req.Request.Method
resource := mapUrlToResource(req.SelectedRoutePath())
client := utilnet.GetHTTPClient(req.Request)
chain.ProcessFilter(req, resp)
code := resp.StatusCode()
contentType := resp.Header().Get("Content-Type")
if resource != nil {
Monitor(verb, *resource, client, contentType, code, startTime)
}
}

// Extract the resource from the path
// Third part of URL (/api/v1/<resource>) and ignore potential subresources
func mapUrlToResource(url string) *string {
parts := strings.Split(url, "/")
if len(parts) < 3 {
return nil
}
return &parts[3]
}

// Web-service filter function used for request and response logging.
func wsLogger(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
log.Printf(FormatRequestLog(req))
Expand Down Expand Up @@ -118,6 +142,9 @@ func CreateHTTPAPIHandler(client *clientK8s.Clientset, heapsterClient client.Hea

apiV1Ws := new(restful.WebService)
apiV1Ws.Filter(wsLogger)

RegisterMetrics()
apiV1Ws.Filter(wsMetrics)
apiV1Ws.Path("/api/v1").
Consumes(restful.MIME_JSON).
Produces(restful.MIME_JSON)
Expand Down
50 changes: 50 additions & 0 deletions src/app/backend/handler/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package handler

import (
"github.com/prometheus/client_golang/prometheus"
"strconv"
"time"
)

var (
requestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "apiserver_request_count",
Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response contentType and code.",
},
[]string{"verb", "resource", "client", "contentType", "code"},
)
requestLatencies = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "apiserver_request_latencies",
Help: "Response latency distribution in microseconds for each verb, resource and client.",
// Use buckets ranging from 125 ms to 8 seconds.
Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
},
[]string{"verb", "resource"},
)
requestLatenciesSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "apiserver_request_latencies_summary",
Help: "Response latency summary in microseconds for each verb and resource.",
// Make the sliding window of 1h.
MaxAge: time.Hour,
},
[]string{"verb", "resource"},
)
)

// Initialize all metrics in prometheus
func RegisterMetrics() {
prometheus.MustRegister(requestCounter)
prometheus.MustRegister(requestLatencies)
prometheus.MustRegister(requestLatenciesSummary)
}

// Track API call in prometheus
func Monitor(verb, resource string, client, contentType string, httpCode int, reqStart time.Time) {
elapsed := float64((time.Since(reqStart)) / time.Microsecond)
requestCounter.WithLabelValues(verb, resource, client, contentType, strconv.Itoa(httpCode)).Inc()
requestLatencies.WithLabelValues(verb, resource).Observe(elapsed)
requestLatenciesSummary.WithLabelValues(verb, resource).Observe(elapsed)
}

0 comments on commit ebff081

Please sign in to comment.