store/tikv: Distinguish req type in forwarding ops metrics (pingcap#2…

…3438)
makemyownlife · Mar 19, 2021 · 5d7bc9f · 5d7bc9f
1 parent 514a4ee
commit 5d7bc9f
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 2 deletions.
diff --git a/metrics/grafana/tidb.json b/metrics/grafana/tidb.json
@@ -7394,6 +7394,99 @@
             "align": false,
             "alignLevel": null
           }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "kv requests that's forwarded by different stores, grouped by request type",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 8,
+            "y": 15
+          },
+          "id": 220,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(tidb_tikvclient_forward_request_counter{tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, result)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}-{{result}}",
+              "refId": "A",
+              "step": 40
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "KV Request Forwarding OPS by Type",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "repeat": null,

diff --git a/store/tikv/metrics/metrics.go b/store/tikv/metrics/metrics.go
@@ -382,7 +382,7 @@ func initMetrics(namespace, subsystem string) {
 			Subsystem: subsystem,
 			Name:      "forward_request_counter",
 			Help:      "Counter of tikv request being forwarded through another node",
-		}, []string{LblFromStore, LblToStore, LblResult})
+		}, []string{LblFromStore, LblToStore, LblType, LblResult})
 
 	initShortcuts()
 }

diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go
@@ -496,7 +496,7 @@ func (s *RegionRequestSender) sendReqToRegion(bo *Backoffer, rpcCtx *RPCContext,
 		if err != nil {
 			result = "fail"
 		}
-		metrics.TiKVForwardRequestCounter.WithLabelValues(fromStore, toStore, result).Inc()
+		metrics.TiKVForwardRequestCounter.WithLabelValues(fromStore, toStore, req.Type.String(), result).Inc()
 	}
 
 	if err != nil {