Skip to content

Commit

Permalink
add more go runtime metrics (matrixorigin#12800)
Browse files Browse the repository at this point in the history
Add more go runtime metrics

Approved by: @aptend
  • Loading branch information
zhangxu19830126 authored Nov 17, 2023
1 parent 0c38cbf commit 6ac2f95
Show file tree
Hide file tree
Showing 6 changed files with 192 additions and 28 deletions.
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ require (
github.com/pkg/errors v0.9.1
github.com/plar/go-adaptive-radix-tree v1.0.5
github.com/prashantv/gostub v1.1.0
github.com/prometheus/client_golang v1.15.1
github.com/prometheus/client_model v0.4.0
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.38.1
github.com/shirou/gopsutil/v3 v3.22.4
Expand Down Expand Up @@ -137,9 +137,9 @@ require (
github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/smartystreets/assertions v1.13.1 // indirect
Expand Down
19 changes: 10 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1678,15 +1678,15 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
Expand All @@ -1695,8 +1695,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
Expand All @@ -1709,8 +1709,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
Expand All @@ -1722,8 +1722,9 @@ github.com/rogpeppe/go-internal v1.0.1-alpha.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/q
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
Expand Down
22 changes: 10 additions & 12 deletions pkg/util/metric/mometric/metric.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,22 @@ package mometric
import (
"context"
"fmt"
"github.com/matrixorigin/matrixone/pkg/common/mpool"
v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"

"github.com/matrixorigin/matrixone/pkg/common/runtime"
"github.com/matrixorigin/matrixone/pkg/util/metric"
"github.com/matrixorigin/matrixone/pkg/util/metric/stats"

"github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/common/mpool"
"github.com/matrixorigin/matrixone/pkg/common/runtime"
"github.com/matrixorigin/matrixone/pkg/config"
"github.com/matrixorigin/matrixone/pkg/logutil"
"github.com/matrixorigin/matrixone/pkg/util/export/table"
ie "github.com/matrixorigin/matrixone/pkg/util/internalExecutor"

"github.com/matrixorigin/matrixone/pkg/util/metric"
"github.com/matrixorigin/matrixone/pkg/util/metric/stats"
v2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
Expand Down Expand Up @@ -117,7 +115,7 @@ func InitMetric(ctx context.Context, ieFactory func() ie.InternalExecutor, SV *c
if metric.EnableExportToProm() {
// http.HandleFunc("/query", makeDebugHandleFunc(ieFactory))
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{}))
mux.Handle("/metrics", promhttp.HandlerFor(v2.GetPrometheusGatherer(), promhttp.HandlerOpts{}))
addr := fmt.Sprintf(":%d", SV.StatusPort)
statusSvr = &statusServer{Server: &http.Server{Addr: addr, Handler: mux}}
statusSvr.Add(1)
Expand Down Expand Up @@ -206,7 +204,7 @@ func StopMetricSync() {
}

func mustRegiterToProm(collector prom.Collector) {
if err := prom.Register(collector); err != nil {
if err := v2.GetPrometheusRegistry().Register(collector); err != nil {
// err is either registering a collector more than once or metrics have duplicate description.
// in any case, we respect the existing collectors in the prom registry
logutil.Debugf("[Metric] register to prom register: %v", err)
Expand Down Expand Up @@ -304,9 +302,9 @@ type descExtra struct {
func newDescExtra(desc *prom.Desc) *descExtra {
str := desc.String()[14:] // strip Desc{fqName: "
fqName := str[:strings.Index(str, "\"")]
str = str[strings.Index(str, "variableLabels: [")+17:] // spot varlbl list
str = str[:strings.Index(str, "]")]
varLblCnt := len(strings.Split(str, " "))
str = str[strings.Index(str, "variableLabels: {")+17:] // spot varlbl list
str = str[:strings.Index(str, "}")]
varLblCnt := len(strings.Split(str, ","))
labels := prom.MakeLabelPairs(desc, make([]string, varLblCnt))
return &descExtra{orig: desc, fqName: fqName, labels: labels}
}
Expand Down
6 changes: 5 additions & 1 deletion pkg/util/metric/v2/dashboard/grafana_dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ func (c *DashboardCreator) Create() error {
return err
}

if err := c.initRuntimeDashboard(); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -218,7 +222,7 @@ func (c *DashboardCreator) getHistogramWithExtraBy(
legend := fmt.Sprintf("P%.2f%%", percent*100)
if len(extraBy) > 0 {
query = fmt.Sprintf("histogram_quantile(%f, sum(rate(%s[$interval])) by (le, %s))", percent, metric, extraBy)
legend = fmt.Sprintf("{{ name }}(P%.2f%%)", percent*100)
legend = fmt.Sprintf("{{ "+extraBy+" }}(P%.2f%%)", percent*100)
}
queries = append(queries, query)
legends = append(legends, legend)
Expand Down
147 changes: 147 additions & 0 deletions pkg/util/metric/v2/dashboard/grafana_dashboard_runtime.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
// Copyright 2023 Matrix Origin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dashboard

import (
"context"

"github.com/K-Phoen/grabana/axis"
"github.com/K-Phoen/grabana/dashboard"
)

func (c *DashboardCreator) initRuntimeDashboard() error {
folder, err := c.createFolder(moFolderName)
if err != nil {
return err
}

build, err := dashboard.New(
"Go Runtime Metrics",
c.withRowOptions(
c.initMemoryRow(),
c.initGCRow(),
c.initGoroutineRow(),
)...)
if err != nil {
return err
}
_, err = c.cli.UpsertDashboard(context.Background(), folder, build)
return err
}

func (c *DashboardCreator) initGCRow() dashboard.Option {
return dashboard.Row(
"Go GC Status",
c.getHistogramWithExtraBy(
"STW duration",
c.getMetricWithFilter(`go_gc_pauses_seconds_bucket`, ``),
[]float64{0.8, 0.90, 0.95, 0.99},
12,
c.by,
axis.Unit("s"),
axis.Min(0)),
)
}

func (c *DashboardCreator) initGoroutineRow() dashboard.Option {
return dashboard.Row(
"Goroutine Status",
c.withGraph(
"Goroutine count",
6,
`sum(`+c.getMetricWithFilter("go_goroutines", "")+`) by (`+c.by+`)`,
"{{ "+c.by+" }}"),

c.getHistogramWithExtraBy(
"Schedule latency duration",
c.getMetricWithFilter(`go_sched_latencies_seconds_bucket`, ``),
[]float64{0.8, 0.90, 0.95, 0.99},
6,
c.by,
axis.Unit("s"),
axis.Min(0)),
)
}

func (c *DashboardCreator) initMemoryRow() dashboard.Option {
return dashboard.Row(
"Memory Status",
c.withGraph(
"Live Objects",
3,
`sum(`+c.getMetricWithFilter("go_gc_heap_objects_objects", "")+`) by (`+c.by+`)`,
"{{ "+c.by+" }}"),

c.withGraph(
"Free and ready to return system",
3,
`sum(`+c.getMetricWithFilter("go_memory_classes_heap_free_bytes", "")+`) by (`+c.by+`)`,
"{{ "+c.by+" }}",
axis.Unit("bytes"),
axis.Min(0)),

c.withGraph(
"Dead objects and not marked free live objects",
3,
`sum(`+c.getMetricWithFilter("go_memory_classes_heap_objects_bytes", "")+`) by (`+c.by+`)`,
"{{ "+c.by+" }}",
axis.Unit("bytes"),
axis.Min(0)),

c.withGraph(
"Released to system",
3,
`sum(`+c.getMetricWithFilter("go_memory_classes_heap_released_bytes", "")+`) by (`+c.by+`)`,
"{{ "+c.by+" }}",
axis.Unit("bytes"),
axis.Min(0)),

c.withGraph(
"Heap Allocation Bytes/s",
3,
`sum(rate(`+c.getMetricWithFilter("go_gc_heap_allocs_bytes_total", "")+`[$interval])) by (`+c.by+`)`,
"{{ "+c.by+" }}",
axis.Unit("bytes"),
axis.Min(0)),

c.withGraph(
"Heap Free Bytes/s",
3,
`sum(rate(`+c.getMetricWithFilter("go_gc_heap_frees_bytes_total", "")+`[$interval])) by (`+c.by+`)`,
"{{ "+c.by+" }}",
axis.Unit("bytes"),
axis.Min(0)),

c.withGraph(
"Heap Allocation Object/s",
3,
`sum(rate(`+c.getMetricWithFilter("go_gc_heap_allocs_objects_total", "")+`[$interval])) by (`+c.by+`)`,
"{{ "+c.by+" }}"),

c.withGraph(
"Heap Free Object/s",
3,
`sum(rate(`+c.getMetricWithFilter("go_gc_heap_frees_objects_total", "")+`[$interval])) by (`+c.by+`)`,
"{{ "+c.by+" }}"),

c.getHistogram(
"Allocation bytes size",
c.getMetricWithFilter(`go_gc_heap_allocs_by_size_bytes_bucket`, ``),
[]float64{0.8, 0.90, 0.95, 0.99},
12,
axis.Unit("bytes"),
axis.Min(0)),
)
}
16 changes: 15 additions & 1 deletion pkg/util/metric/v2/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,27 @@ package v2

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
)

var (
registry = prometheus.DefaultRegisterer
registry = prometheus.NewRegistry()
)

func GetPrometheusRegistry() prometheus.Registerer {
return registry
}

func GetPrometheusGatherer() prometheus.Gatherer {
return registry
}

func init() {
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
registry.MustRegister(collectors.NewGoCollector(
collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll),
))

initFileServiceMetrics()
initLogtailMetrics()
initTxnMetrics()
Expand Down

0 comments on commit 6ac2f95

Please sign in to comment.