Introduce new unauthenticated endpoint /metric (minio#5723) (minio#5829)

/metric exposes Promethus compatible data for scraping metrics

Fixes: minio#5723

cmd/generic-handlers.go

@@ -205,6 +205,17 @@ func guessIsHealthCheckReq(req *http.Request) bool {
 			req.URL.Path == healthCheckPathPrefix+healthCheckReadinessPath)
 }
 
+// guessIsMetricsReq - returns true if incoming request looks
+// like metrics request
+func guessIsMetricsReq(req *http.Request) bool {
+	if req == nil {
+		return false
+	}
+	aType := getRequestAuthType(req)
+	return aType == authTypeAnonymous &&
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsPath
+}
+
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.
 func guessIsRPCReq(req *http.Request) bool {
 	if req == nil {
@@ -275,7 +286,7 @@ func setReservedBucketHandler(h http.Handler) http.Handler {
 
 func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch {
-	case guessIsRPCReq(r), guessIsBrowserReq(r), guessIsHealthCheckReq(r), isAdminReq(r):
+	case guessIsRPCReq(r), guessIsBrowserReq(r), guessIsHealthCheckReq(r), guessIsMetricsReq(r), isAdminReq(r):
 		// Allow access to reserved buckets
 	default:
 		// For all other requests reject access to reserved

cmd/http-stats.go

@@ -21,6 +21,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"go.uber.org/atomic"
 )
 
@@ -188,6 +190,10 @@ func (st *HTTPStats) updateStats(r *http.Request, w *httpResponseRecorder, durat
 			st.successDELETEs.Duration.Add(durationSecs)
 		}
 	}
+	// Increment the prometheus http request count with appropriate label
+	httpRequests.With(prometheus.Labels{"request_type": r.Method}).Inc()
+	// Increment the prometheus http request response histogram with appropriate label
+	httpRequestsDuration.With(prometheus.Labels{"request_type": r.Method}).Observe(durationSecs)
 }
 
 // Prepare new HTTPStats structure

cmd/metrics-router.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	router "github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	prometheusMetricsPath = "/prometheus/metrics"
+)
+
+// registerMetricsRouter - add handler functions for metrics.
+func registerMetricsRouter(mux *router.Router) {
+	// metrics router
+	metricsRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
+	metricsRouter.Handle(prometheusMetricsPath, metricsHandler(prometheus.Handler()))
+}

cmd/metrics.go

@@ -0,0 +1,129 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	httpRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "minio_http_requests_total",
+			Help: "Total number of requests served by current Minio server instance",
+		},
+		[]string{"request_type"},
+	)
+	httpRequestsDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "minio_http_requests_duration_seconds",
+			Help:    "Time taken by requests served by current Minio server instance",
+			Buckets: []float64{.001, .003, .005, .1, .5, 1},
+		},
+		[]string{"request_type"},
+	)
+	networkSentBytes = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_network_sent_bytes",
+			Help: "Total number of bytes sent by current Minio server instance",
+		},
+	)
+	networkReceivedBytes = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_network_received_bytes",
+			Help: "Total number of bytes received by current Minio server instance",
+		},
+	)
+	onlineMinioDisks = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_online_disks_total",
+			Help: "Total number of online disks for current Minio server instance",
+		},
+	)
+	offlineMinioDisks = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_offline_disks_total",
+			Help: "Total number of offline disks for current Minio server instance",
+		},
+	)
+	serverUptime = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_server_uptime_seconds",
+			Help: "Time elapsed since current Minio server instance started",
+		},
+	)
+	minioStorageTotal = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_disk_storage_bytes",
+			Help: "Total disk storage available to current Minio server instance",
+		},
+	)
+	minioStorageFree = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "minio_disk_storage_free_bytes",
+			Help: "Total free disk storage available to current Minio server instance",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(httpRequests)
+	prometheus.MustRegister(httpRequestsDuration)
+	prometheus.MustRegister(networkSentBytes)
+	prometheus.MustRegister(networkReceivedBytes)
+	prometheus.MustRegister(onlineMinioDisks)
+	prometheus.MustRegister(offlineMinioDisks)
+	prometheus.MustRegister(serverUptime)
+	prometheus.MustRegister(minioStorageTotal)
+	prometheus.MustRegister(minioStorageFree)
+}
+
+func updateGeneralMetrics() {
+	// Increment server uptime
+	serverUptime.Set(UTCNow().Sub(globalBootTime).Seconds())
+
+	// Fetch disk space info
+	objLayer := newObjectLayerFn()
+	// Service not initialized yet
+	if objLayer == nil {
+		return
+	}
+
+	// Update total/free disk space
+	s := objLayer.StorageInfo(context.Background())
+	minioStorageTotal.Set(float64(s.Total))
+	minioStorageFree.Set(float64(s.Free))
+
+	// Update online/offline disks
+	onlineMinioDisks.Set(float64(s.Backend.OnlineDisks))
+	offlineMinioDisks.Set(float64(s.Backend.OfflineDisks))
+
+	// Update prometheus metric
+	networkSentBytes.Set(float64(globalConnStats.getTotalOutputBytes()))
+	networkReceivedBytes.Set(float64(globalConnStats.getTotalInputBytes()))
+}
+
+func metricsHandler(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Update generic metrics before handling the prometheus scrape request
+		updateGeneralMetrics()
+		prometheus.Handler().ServeHTTP(w, r)
+	})
+}

cmd/routers.go

@@ -119,6 +119,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 	// Add healthcheck router
 	registerHealthCheckRouter(mux)
 
+	// Add server metrics router
+	registerMetricsRouter(mux)
+
 	// Register web router when its enabled.
 	if globalIsBrowserEnabled {
 		if err := registerWebRouter(mux); err != nil {

docs/metric/README.md

@@ -0,0 +1,38 @@
+## Minio Prometheus Metric
+
+Minio server exposes an endpoint for Promethueus to scrape server data at `/minio/metric`.
+
+### Prometheus probe
+Prometheus is used to monitor Minio server information like http request, disk storage, network stats etc.. It uses a config file named `prometheus.yaml` to scrape data from server. The value for `metrics_path` and `targets` need to be configured in the config yaml to specify the endpoint and url as shown:
+```
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: minio
+    metrics_path: /minio/prometheus/metrics
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9000']
+```
+ Prometheus can be run by executing :
+```
+./prometheus --config.file=prometheus.yml
+```
+
+### List of Minio metric exposed
+Minio exposes the following list of metric to Prometheus
+```
+minio_disk_storage_bytes
+minio_disk_storage_free_bytes
+minio_http_requests_duration_seconds_bucket
+minio_http_requests_duration_seconds_count
+minio_http_requests_duration_seconds_sum
+minio_http_requests_total
+minio_network_received_bytes
+minio_network_sent_bytes
+minio_offline_disks_total
+minio_online_disks_total
+minio_server_uptime_seconds
+```