diff --git a/pkg/query-service/app/integrations/builtin.go b/pkg/query-service/app/integrations/builtin.go new file mode 100644 index 00000000000..7b6c98c4530 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin.go @@ -0,0 +1,197 @@ +package integrations + +import ( + "context" + "embed" + "strings" + + "encoding/base64" + "encoding/json" + "fmt" + "io/fs" + "path" + + koanfJson "github.com/knadh/koanf/parsers/json" + "go.signoz.io/signoz/pkg/query-service/model" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +type BuiltInIntegrations struct{} + +var builtInIntegrations map[string]IntegrationDetails + +func (bi *BuiltInIntegrations) list(ctx context.Context) ( + []IntegrationDetails, *model.ApiError, +) { + integrations := maps.Values(builtInIntegrations) + slices.SortFunc(integrations, func(i1, i2 IntegrationDetails) bool { + return i1.Id < i2.Id + }) + return integrations, nil +} + +func (bi *BuiltInIntegrations) get( + ctx context.Context, integrationIds []string, +) ( + map[string]IntegrationDetails, *model.ApiError, +) { + result := map[string]IntegrationDetails{} + for _, iid := range integrationIds { + i, exists := builtInIntegrations[iid] + if exists { + result[iid] = i + } + } + return result, nil +} + +//go:embed builtin_integrations/* +var integrationFiles embed.FS + +func init() { + err := readBuiltIns() + if err != nil { + panic(fmt.Errorf("couldn't read builtin integrations: %w", err)) + } +} + +func readBuiltIns() error { + rootDirName := "builtin_integrations" + builtinDirs, err := fs.ReadDir(integrationFiles, rootDirName) + if err != nil { + return fmt.Errorf("couldn't list integrations dirs: %w", err) + } + + builtInIntegrations = map[string]IntegrationDetails{} + for _, d := range builtinDirs { + if !d.IsDir() { + continue + } + + integrationDir := path.Join(rootDirName, d.Name()) + i, err := readBuiltInIntegration(integrationDir) + if err != nil { + return fmt.Errorf("couldn't parse integration %s from files: %w", d.Name(), err) + } + + _, exists := builtInIntegrations[i.Id] + if exists { + return fmt.Errorf( + "duplicate integration for id %s at %s", i.Id, d.Name(), + ) + } + builtInIntegrations[i.Id] = *i + } + return nil +} + +func readBuiltInIntegration(dirpath string) ( + *IntegrationDetails, error, +) { + integrationJsonPath := path.Join(dirpath, "integration.json") + + serializedSpec, err := integrationFiles.ReadFile(integrationJsonPath) + if err != nil { + return nil, fmt.Errorf("couldn't find integration.json in %s: %w", dirpath, err) + } + + integrationSpec, err := koanfJson.Parser().Unmarshal(serializedSpec) + if err != nil { + return nil, fmt.Errorf( + "couldn't parse integration json from %s: %w", integrationJsonPath, err, + ) + } + + hydrated, err := hydrateFileUris(integrationSpec, dirpath) + if err != nil { + return nil, fmt.Errorf( + "couldn't hydrate files referenced in integration %s: %w", integrationJsonPath, err, + ) + } + + hydratedSpec := hydrated.(map[string]interface{}) + hydratedSpecJson, err := koanfJson.Parser().Marshal(hydratedSpec) + if err != nil { + return nil, fmt.Errorf( + "couldn't serialize hydrated integration spec back to JSON %s: %w", integrationJsonPath, err, + ) + } + + var integration IntegrationDetails + err = json.Unmarshal(hydratedSpecJson, &integration) + if err != nil { + return nil, fmt.Errorf( + "couldn't parse hydrated JSON spec read from %s: %w", + integrationJsonPath, err, + ) + } + + integration.Id = "builtin::" + integration.Id + + return &integration, nil +} + +func hydrateFileUris(spec interface{}, basedir string) (interface{}, error) { + if specMap, ok := spec.(map[string]interface{}); ok { + result := map[string]interface{}{} + for k, v := range specMap { + hydrated, err := hydrateFileUris(v, basedir) + if err != nil { + return nil, err + } + result[k] = hydrated + } + return result, nil + + } else if specSlice, ok := spec.([]interface{}); ok { + result := []interface{}{} + for _, v := range specSlice { + hydrated, err := hydrateFileUris(v, basedir) + if err != nil { + return nil, err + } + result = append(result, hydrated) + } + return result, nil + + } else if maybeFileUri, ok := spec.(string); ok { + return readFileIfUri(maybeFileUri, basedir) + } + + return spec, nil + +} + +func readFileIfUri(maybeFileUri string, basedir string) (interface{}, error) { + fileUriPrefix := "file://" + if !strings.HasPrefix(maybeFileUri, fileUriPrefix) { + return maybeFileUri, nil + } + + relativePath := maybeFileUri[len(fileUriPrefix):] + fullPath := path.Join(basedir, relativePath) + + fileContents, err := integrationFiles.ReadFile(fullPath) + if err != nil { + return nil, fmt.Errorf("couldn't read referenced file: %w", err) + } + if strings.HasSuffix(maybeFileUri, ".md") { + return string(fileContents), nil + + } else if strings.HasSuffix(maybeFileUri, ".json") { + parsed, err := koanfJson.Parser().Unmarshal(fileContents) + if err != nil { + return nil, fmt.Errorf("couldn't parse referenced JSON file: %w", err) + } + return parsed, nil + + } else if strings.HasSuffix(maybeFileUri, ".svg") { + base64Svg := base64.StdEncoding.EncodeToString(fileContents) + dataUri := fmt.Sprintf("data:image/svg+xml;base64,%s", base64Svg) + return dataUri, nil + + } + + return nil, fmt.Errorf("unsupported file type %s", maybeFileUri) +} diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json new file mode 100644 index 00000000000..5b5d93ad9e8 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json @@ -0,0 +1,796 @@ +{ + "description": "This dashboard provides a high-level overview of your MongoDB. It includes read/write performance, most-used replicas, collection metrics etc...", + "layout": [ + { + "h": 3, + "i": "0c3d2b15-89be-4d62-a821-b26d93332ed3", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 3 + }, + { + "h": 3, + "i": "14504a3c-4a05-4d22-bab3-e22e94f51380", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 6 + }, + { + "h": 3, + "i": "dcfb3829-c3f2-44bb-907d-8dc8a6dc4aab", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 3 + }, + { + "h": 3, + "i": "bfc9e80b-02bf-4122-b3da-3dd943d35012", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 0 + }, + { + "h": 3, + "i": "4c07a7d2-893a-46c2-bcdb-a19b6efeac3a", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 0 + }, + { + "h": 3, + "i": "a5a64eec-1034-4aa6-8cb1-05673c4426c6", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 6 + }, + { + "h": 3, + "i": "503af589-ef4d-4fe3-8934-c8f7eb480d9a", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 9 + } + ], + "name": "", + "tags": [ + "mongo", + "database" + ], + "title": "Mongo overview", + "variables": { + "a2c21714-a814-4d31-9b56-7367c3208801": { + "allSelected": true, + "customValue": "", + "description": "List of hosts sending mongo metrics", + "id": "a2c21714-a814-4d31-9b56-7367c3208801", + "modificationUUID": "448e675a-4531-45b1-b434-a9ee809470d6", + "multiSelect": true, + "name": "host_name", + "order": 0, + "queryValue": "SELECT JSONExtractString(labels, 'host_name') AS host_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'mongodb_memory_usage'\nGROUP BY host_name", + "selectedValue": [ + "Srikanths-MacBook-Pro.local" + ], + "showALLOption": true, + "sort": "ASC", + "textboxValue": "", + "type": "QUERY" + } + }, + "widgets": [ + { + "description": "Total number of operations", + "fillSpans": false, + "id": "4c07a7d2-893a-46c2-bcdb-a19b6efeac3a", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_operation_count--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_operation_count", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "a468a30b", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + } + ], + "having": [], + "legend": "{{operation}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "7da5d899-8b06-4139-9a89-47baf9551ff8", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Operations count", + "yAxisUnit": "none" + }, + { + "description": "The total time spent performing operations.", + "fillSpans": false, + "id": "bfc9e80b-02bf-4122-b3da-3dd943d35012", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_operation_time--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_operation_time", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "31be3166", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + } + ], + "having": [], + "legend": "{{operation}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "2ca35957-894a-46ae-a2a6-95d7e400d8e1", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Total operations time", + "yAxisUnit": "ms" + }, + { + "description": "The number of cache operations", + "fillSpans": false, + "id": "dcfb3829-c3f2-44bb-907d-8dc8a6dc4aab", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_cache_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_cache_operations", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "01b45814", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "type--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "type", + "type": "tag" + } + ], + "having": [], + "legend": "{{type}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "bb439198-dcf5-4767-b0d0-ab5785159b8d", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Cache operations", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "14504a3c-4a05-4d22-bab3-e22e94f51380", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_operation_latency_time--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_operation_latency_time", + "type": "Gauge" + }, + "aggregateOperator": "max", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "2e165319", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "read" + }, + { + "id": "888e920b", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Latency", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "4a9cafe8-778b-476c-b825-c04e165bf285", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Read latency", + "yAxisUnit": "µs" + }, + { + "description": "", + "fillSpans": false, + "id": "a5a64eec-1034-4aa6-8cb1-05673c4426c6", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_operation_latency_time--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_operation_latency_time", + "type": "Gauge" + }, + "aggregateOperator": "max", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "53b37ca7", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "9862c46c", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "write" + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Latency", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "446827eb-a4f2-4ff3-966b-fb65288c983b", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Write latency", + "yAxisUnit": "µs" + }, + { + "description": "", + "fillSpans": false, + "id": "503af589-ef4d-4fe3-8934-c8f7eb480d9a", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_operation_latency_time--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_operation_latency_time", + "type": "Gauge" + }, + "aggregateOperator": "max", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "c33ad4b6", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "c70ecfd0", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "command" + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Latency", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "7b7b977d-0921-4552-8cfe-d82dfde63ef4", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Command latency", + "yAxisUnit": "µs" + }, + { + "description": "", + "fillSpans": false, + "id": "0c3d2b15-89be-4d62-a821-b26d93332ed3", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_network_io_receive--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_network_io_receive", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "5c9d7fe3", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Bytes received :: {{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "mongodb_network_io_transmit--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "mongodb_network_io_transmit", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "96520885", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Bytes transmitted :: {{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "41eea5bc-f9cf-45c2-92fb-ef226d6b540b", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Network IO", + "yAxisUnit": "bytes" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/pipelines/log-parser.json b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/pipelines/log-parser.json new file mode 100644 index 00000000000..e75db3ec5df --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/pipelines/log-parser.json @@ -0,0 +1,33 @@ +{ + "id": "parse-default-mongo-access-log", + "name": "Parse default mongo access log", + "alias": "parse-default-mongo-access-log", + "description": "Parse standard mongo access log", + "enabled": true, + "filter": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "mongo" + } + ] + }, + "config": [ + { + "type": "grok_parser", + "id": "parse-body-grok", + "enabled": true, + "orderId": 1, + "name": "Parse Body", + "parse_to": "attributes", + "pattern": "%{GREEDYDATA}", + "parse_from": "body" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/config/configure-otel-collector.md b/pkg/query-service/app/integrations/builtin_integrations/mongo/config/configure-otel-collector.md new file mode 100644 index 00000000000..abb9bd8f255 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/config/configure-otel-collector.md @@ -0,0 +1,74 @@ +### Configure otel collector + +#### Save collector config file + +Save the following collector config in a file named `mongo-collector-config.yaml` + +``` +receivers: + mongodb: + # - For standalone MongoDB deployments this is the hostname and port of the mongod instance + # - For replica sets specify the hostnames and ports of the mongod instances that are in the replica set configuration. If the replica_set field is specified, nodes will be autodiscovered. + # - For a sharded MongoDB deployment, please specify a list of the mongos hosts. + hosts: + - endpoint: 127.0.0.1:27017 + # If authentication is required, the user can with clusterMonitor permissions can be provided here + username: monitoring + # If authentication is required, the password can be provided here. + password: ${env:MONGODB_PASSWORD} + collection_interval: 60s + # If TLS is enabled, the following fields can be used to configure the connection + tls: + insecure: true + insecure_skip_verify: true + # ca_file: /etc/ssl/certs/ca-certificates.crt + # cert_file: /etc/ssl/certs/mongodb.crt + # key_file: /etc/ssl/certs/mongodb.key + metrics: + mongodb.lock.acquire.count: + enabled: true + mongodb.lock.acquire.time: + enabled: true + mongodb.lock.acquire.wait_count: + enabled: true + mongodb.lock.deadlock.count: + enabled: true + mongodb.operation.latency.time: + enabled: true + +processors: + # enriches the data with additional host information + # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor + resourcedetection/system: + # add additional detectors if needed + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # export to local collector + otlp/local: + endpoint: "localhost:4317" + tls: + insecure: true + # export to SigNoz cloud + otlp/signoz: + endpoint: "ingest.{region}.signoz.cloud:443" + tls: + insecure: false + headers: + "signoz-access-token": "" + +service: + pipelines: + metrics/mongodb: + receivers: [mongodb] + # note: remove this processor if the collector host is not running on the same host as the mongo instance + processors: [resourcedetection/system] + exporters: [otlp/local] + +``` + +#### Use collector config file + +Run your collector with the added flag `--config mongo-collector-config.yaml` diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/config/prerequisites.md b/pkg/query-service/app/integrations/builtin_integrations/mongo/config/prerequisites.md new file mode 100644 index 00000000000..2914a64b860 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/config/prerequisites.md @@ -0,0 +1,22 @@ +### Prepare mongo for monitoring + +- Have a running mongodb instance +- Have the monitoring user created +- Have the monitoring user granted the necessary permissions + +Mongodb recommends to set up a least privilege user (LPU) with a `clusterMonitor` role in order to collect. + +Run the following command to create a user with the necessary permissions. + +``` +use admin +db.createUser( + { + user: "monitoring", + pwd: "", + roles: ["clusterMonitor"] + } +); +``` + +Replace `` with a strong password and set is as env var `MONGODB_PASSWORD`. diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/icon.svg b/pkg/query-service/app/integrations/builtin_integrations/mongo/icon.svg new file mode 100644 index 00000000000..4ffedc63390 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/icon.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json b/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json new file mode 100644 index 00000000000..c5d0fcefcd3 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json @@ -0,0 +1,88 @@ +{ + "id": "mongo", + "title": "Mongo", + "description": "Monitor mongo using logs and metrics.", + "author": { + "name": "SigNoz", + "email": "integrations@signoz.io", + "homepage": "https://signoz.io" + }, + "icon": "file://icon.svg", + "categories": [ + "Database" + ], + "overview": "file://overview.md", + "configuration": [ + { + "title": "Prerequisites", + "instructions": "file://config/prerequisites.md" + }, + { + "title": "Configure Otel Collector", + "instructions": "file://config/configure-otel-collector.md" + } + ], + "assets": { + "logs": { + "pipelines": [ + "file://assets/pipelines/log-parser.json" + ] + }, + "dashboards": [ + "file://assets/dashboards/overview.json" + ], + "alerts": [] + }, + "connection_tests": { + "logs": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "mongo" + } + ] + } + }, + "data_collected": { + "logs": [ + { + "name": "Request Method", + "path": "attributes[\"http.request.method\"]", + "type": "string", + "description": "HTTP method" + }, + { + "name": "Request Path", + "path": "attributes[\"url.path\"]", + "type": "string", + "description": "path requested" + }, + { + "name": "Response Status Code", + "path": "attributes[\"http.response.status_code\"]", + "type": "int", + "description": "HTTP response code" + } + ], + "metrics": [ + { + "name": "http.server.request.duration", + "type": "Histogram", + "unit": "s", + "description": "Duration of HTTP server requests" + }, + { + "name": "http.server.active_requests", + "type": "UpDownCounter", + "unit": "{ request }", + "description": "Number of active HTTP server requests" + } + ] + } +} diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/overview.md b/pkg/query-service/app/integrations/builtin_integrations/mongo/overview.md new file mode 100644 index 00000000000..c7a84541ad9 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/overview.md @@ -0,0 +1,3 @@ +### Monitor MongoDB with SigNoz + +Collect key MongoDB metrics and parse your MongoDB logs diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/assets/pipelines/log-parser.json b/pkg/query-service/app/integrations/builtin_integrations/nginx/assets/pipelines/log-parser.json new file mode 100644 index 00000000000..e9521e45ff1 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/assets/pipelines/log-parser.json @@ -0,0 +1,62 @@ +{ + "id": "parse-default-nginx-access-log", + "name": "Parse default nginx access log", + "alias": "parse-default-nginx-access-log", + "description": "Parse standard nginx access log", + "enabled": true, + "filter": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "nginx" + } + ] + }, + "config": [ + { + "type": "grok_parser", + "id": "parse-body-grok", + "enabled": true, + "orderId": 1, + "name": "Parse Body", + "parse_to": "attributes", + "pattern": "%{IP:client.address} - %{USERNAME:enduser.id} \\[%{HTTPDATE:time.local}\\] \"((%{WORD:http.method} %{DATA:http.path}(\\?%{DATA:http.query})? %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version})|%{DATA})\" %{INT:http.response.status_code:int} %{INT:http.request.body.bytes:int} \"%{NOTSPACE:http.referer}\" \"%{DATA:http.user.agent}\" %{INT:http.request.bytes:int} %{NUMBER:http.request.time:float} \\[%{DATA:proxy.upstream.name}?\\] \\[%{DATA:proxy.alternative.upstream.name}?\\] ((%{IP:network.peer.address}:%{INT:network.peer.port:int})|%{DATA})? (%{INT:http.response.bytes:int}|-)? (%{NUMBER:http.response.time:float}|-)? (%{NUMBER:network.peer.status.code:int}|-)? %{NOTSPACE:request.id}", + "parse_from": "body" + }, + { + "type": "severity_parser", + "id": "parse-sev", + "enabled": true, + "orderId": 2, + "name": "Set Severity", + "parse_from": "attributes[\"http.response.status_code\"]", + "mapping": { + "debug": [ + "1xx" + ], + "error": [ + "4xx" + ], + "fatal": [ + "5xx" + ], + "info": [ + "2xx" + ], + "trace": [ + "trace" + ], + "warn": [ + "3xx" + ] + }, + "overwrite_text": true + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/config/configure-otel-collector.md b/pkg/query-service/app/integrations/builtin_integrations/nginx/config/configure-otel-collector.md new file mode 100644 index 00000000000..f5c22e16cb6 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/config/configure-otel-collector.md @@ -0,0 +1 @@ +### Configure otel collector diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/config/prepare-nginx.md b/pkg/query-service/app/integrations/builtin_integrations/nginx/config/prepare-nginx.md new file mode 100644 index 00000000000..2677d09b6be --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/config/prepare-nginx.md @@ -0,0 +1 @@ +### Prepare nginx for observability diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/icon.svg b/pkg/query-service/app/integrations/builtin_integrations/nginx/icon.svg new file mode 100644 index 00000000000..56876150207 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/icon.svg @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json b/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json new file mode 100644 index 00000000000..558f9780d09 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json @@ -0,0 +1,87 @@ +{ + "id": "nginx", + "title": "Nginx", + "description": "Monitor nginx using logs and metrics.", + "author": { + "name": "SigNoz", + "email": "integrations@signoz.io", + "homepage": "https://signoz.io" + }, + "icon": "file://icon.svg", + "categories": [ + "Ingress", + "HTTP" + ], + "overview": "file://overview.md", + "configuration": [ + { + "title": "Prepare Nginx", + "instructions": "file://config/prepare-nginx.md" + }, + { + "title": "Configure Otel Collector", + "instructions": "file://config/configure-otel-collector.md" + } + ], + "assets": { + "logs": { + "pipelines": [ + "file://assets/pipelines/log-parser.json" + ] + }, + "dashboards": null, + "alerts": null + }, + "connection_tests": { + "logs": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "nginx" + } + ] + } + }, + "data_collected": { + "logs": [ + { + "name": "Request Method", + "path": "attributes[\"http.request.method\"]", + "type": "string", + "description": "HTTP method" + }, + { + "name": "Request Path", + "path": "attributes[\"url.path\"]", + "type": "string", + "description": "path requested" + }, + { + "name": "Response Status Code", + "path": "attributes[\"http.response.status_code\"]", + "type": "int", + "description": "HTTP response code" + } + ], + "metrics": [ + { + "name": "http.server.request.duration", + "type": "Histogram", + "unit": "s", + "description": "Duration of HTTP server requests" + }, + { + "name": "http.server.active_requests", + "type": "UpDownCounter", + "unit": "{ request }", + "description": "Number of active HTTP server requests" + } + ] + } +} diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/overview.md b/pkg/query-service/app/integrations/builtin_integrations/nginx/overview.md new file mode 100644 index 00000000000..dac6354fc09 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/overview.md @@ -0,0 +1,3 @@ +### Monitor Nginx with SigNoz + +Parse your Nginx logs and collect key metrics. diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json new file mode 100644 index 00000000000..9799ad66e32 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json @@ -0,0 +1,1868 @@ +{ + "description": "This dashboard provides a high-level overview of your PostgreSQL databases. It includes replication, locks, and throughput etc...", + "layout": [ + { + "h": 3, + "i": "9552123d-6265-48a7-8624-3f4a3fc3c9c0", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 18 + }, + { + "h": 3, + "i": "d7838815-4f5b-4454-86fd-f658b201f3a9", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 15 + }, + { + "h": 3, + "i": "f9a6f683-7455-4643-acc8-467cc5ea52cf", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 15 + }, + { + "h": 3, + "i": "8638a199-20a0-4255-b0a2-3b1ba06c485b", + "moved": false, + "static": false, + "w": 6, + "x": 3, + "y": 12 + }, + { + "h": 3, + "i": "e9341e70-ccb3-47fc-af95-56ba8942c4f2", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 9 + }, + { + "h": 3, + "i": "6b700035-e3c2-4c48-99fa-ebfd6202eed3", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 9 + }, + { + "h": 3, + "i": "bada7864-1d23-4d49-a868-c6b8a93c738f", + "moved": false, + "static": false, + "w": 6, + "x": 3, + "y": 6 + }, + { + "h": 3, + "i": "191d09a6-40b0-4de8-a5b0-aa4254454b99", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 0 + }, + { + "h": 3, + "i": "fa941c00-ce19-49cc-baf2-c38598767dee", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 0 + }, + { + "h": 3, + "i": "114fcf80-e1de-4716-b1aa-0e0738dba10e", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 3 + }, + { + "h": 3, + "i": "667428ef-9b9a-4e91-bd1e-938e0dc1ff32", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 3 + } + ], + "name": "", + "tags": [ + "postgres", + "database" + ], + "title": "Postgres overview", + "variables": { + "4250ef7b-8f42-4a24-902a-a764d070b92d": { + "allSelected": true, + "customValue": "", + "description": "List of hosts sending Postgres metrics", + "id": "4250ef7b-8f42-4a24-902a-a764d070b92d", + "key": "4250ef7b-8f42-4a24-902a-a764d070b92d", + "modificationUUID": "4427b655-c8d2-40ce-84ed-7cb058bd3041", + "multiSelect": true, + "name": "host_name", + "order": 0, + "queryValue": "SELECT JSONExtractString(labels, 'host_name') AS host_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'postgresql_operations'\nGROUP BY host_name", + "selectedValue": [ + "Srikanths-MacBook-Pro.local" + ], + "showALLOption": true, + "sort": "ASC", + "textboxValue": "", + "type": "QUERY" + }, + "8ecaee70-640f-46fd-83d9-a4fd18bc66e6": { + "customValue": "", + "description": "List of tables", + "id": "8ecaee70-640f-46fd-83d9-a4fd18bc66e6", + "modificationUUID": "a51321cd-47a2-470a-8df4-372e5bb36f2c", + "multiSelect": true, + "name": "table_name", + "order": 0, + "queryValue": "SELECT JSONExtractString(labels, 'postgresql_table_name') AS table_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'postgresql_operations' AND JSONExtractString(labels, 'postgresql_database_name') IN {{.db_name}}\nGROUP BY table_name", + "showALLOption": true, + "sort": "ASC", + "textboxValue": "", + "type": "QUERY", + "selectedValue": [ + "public.activations", + "public.licenses", + "public.plans", + "public.subscription_items", + "public.subscriptions", + "public.trials", + "public.usage" + ], + "allSelected": true + }, + "c66d1581-e5e1-440d-8ff6-ebcf078ab6dd": { + "allSelected": true, + "customValue": "", + "description": "List of databases", + "id": "c66d1581-e5e1-440d-8ff6-ebcf078ab6dd", + "key": "c66d1581-e5e1-440d-8ff6-ebcf078ab6dd", + "modificationUUID": "564a3f43-98f8-4189-b5e4-dcb518d73852", + "multiSelect": true, + "name": "db_name", + "order": 0, + "queryValue": "SELECT JSONExtractString(labels, 'postgresql_database_name') AS db_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'postgresql_operations'\nGROUP BY db_name", + "selectedValue": [ + "postgres" + ], + "showALLOption": true, + "sort": "DISABLED", + "textboxValue": "", + "type": "QUERY" + } + }, + "widgets": [ + { + "description": "The average number of db insert operations.", + "fillSpans": false, + "id": "191d09a6-40b0-4de8-a5b0-aa4254454b99", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "c1dff946", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "ins" + }, + { + "id": "0cd6dc8f", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "2e60e171", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "bf48ac4c-bc0c-41a0-87f4-6f8ae7888d1f", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Inserts", + "yAxisUnit": "none" + }, + { + "description": "The average number of db update operations.", + "fillSpans": false, + "id": "fa941c00-ce19-49cc-baf2-c38598767dee", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "98463ec9", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "upd" + }, + { + "id": "47db4e8e", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "64020332", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "34a6ac3a-b7f6-4b5f-a084-a44378033d82", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Updates", + "yAxisUnit": "none" + }, + { + "description": "The average number of db delete operations.", + "fillSpans": false, + "id": "114fcf80-e1de-4716-b1aa-0e0738dba10e", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "62738de4", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "del" + }, + { + "id": "d18471e2", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "9d153899", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "734393d1-76ed-4f4f-bef8-0a91d27ebec4", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Deleted", + "yAxisUnit": "none" + }, + { + "description": "The average number of db heap-only update operations.", + "fillSpans": false, + "id": "667428ef-9b9a-4e91-bd1e-938e0dc1ff32", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "a91e35c4", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "hot_upd" + }, + { + "id": "2b419378", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "7b4a29a2", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "f43c2d19-4abc-4f5e-881b-db7add4a870a", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Heap updates", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "bada7864-1d23-4d49-a868-c6b8a93c738f", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "d6aeccf7", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "ins" + }, + { + "id": "4004a127", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "ee4e9344", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "Inserted", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "a12cceed", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "upd" + }, + { + "id": "11735104", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "2d542482", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "Updated", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_operations--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_operations", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "C", + "filters": { + "items": [ + { + "id": "1bca3e46", + "key": { + "dataType": "string", + "id": "operation--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "operation", + "type": "tag" + }, + "op": "=", + "value": "del" + }, + { + "id": "3631755d", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "44ffc874", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "Deleted", + "limit": null, + "orderBy": [], + "queryName": "C", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "5056105b-1c30-4d27-8187-64457f2a1ec6", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Operation by database", + "yAxisUnit": "none" + }, + { + "description": "The number of database locks.", + "fillSpans": false, + "id": "6b700035-e3c2-4c48-99fa-ebfd6202eed3", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_database_locks--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_database_locks", + "type": "Gauge" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "527a3124", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "mode--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "mode", + "type": "tag" + } + ], + "having": [], + "legend": "{{mode}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "877b0df3-9ae3-455e-ad27-bc3aa40b3f4c", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Locks by lock mode", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "e9341e70-ccb3-47fc-af95-56ba8942c4f2", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_deadlocks--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_deadlocks", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "ff14f172", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "efb83717", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "5056105b-1c30-4d27-8187-64457f2a1ec6", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Deadlocks count", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "8638a199-20a0-4255-b0a2-3b1ba06c485b", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_backends--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_backends", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "ed335b00", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "20d2a4c5", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_database_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "205b99a0-2f1c-4bd2-9ba0-cc2da6ef247a", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Connections per db", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "f9a6f683-7455-4643-acc8-467cc5ea52cf", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_rows--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_rows", + "type": "Sum" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "70786905", + "key": { + "dataType": "string", + "id": "state--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "state", + "type": "tag" + }, + "op": "=", + "value": "dead" + }, + { + "id": "810e39a9", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "3e5ef839", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "9e913563", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Dead rows", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "cc7452c8-118b-4676-959e-7062bafc41ee", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Dead rows", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "d7838815-4f5b-4454-86fd-f658b201f3a9", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_index_scans--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_index_scans", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "da04d826", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "590332a7", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "171b9516", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_index_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_index_name", + "type": "tag" + } + ], + "having": [], + "legend": "{{postgresql_index_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "2c6b630b-8bd9-4001-815b-f2b1f439a9dd", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Index scans by index", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "9552123d-6265-48a7-8624-3f4a3fc3c9c0", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_rows--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_rows", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "83f9cab9", + "key": { + "dataType": "string", + "id": "state--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "state", + "type": "tag" + }, + "op": "=", + "value": "dead" + }, + { + "id": "2a0284c2", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "c2aaf758", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "a603fda9", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + } + ], + "having": [], + "legend": "Dead rows", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_rows--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_rows", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "82f1f0f5", + "key": { + "dataType": "string", + "id": "state--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "state", + "type": "tag" + }, + "op": "=", + "value": "live" + }, + { + "id": "14de7a06", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "0a88a27a", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "4417218d", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + } + ], + "having": [], + "legend": "Live rows", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_index_scans--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_index_scans", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "C", + "filters": { + "items": [ + { + "id": "22795c15", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "d7e7c193", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "d3ae1dbe", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + } + ], + "having": [], + "legend": "Index scans", + "limit": null, + "orderBy": [], + "queryName": "C", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "postgresql_table_size--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "postgresql_table_size", + "type": "Sum" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "D", + "filters": { + "items": [ + { + "id": "48c436ab", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + }, + { + "id": "cc617789", + "key": { + "dataType": "string", + "id": "postgresql_database_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_database_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.db_name}}" + ] + }, + { + "id": "b4029d50", + "key": { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.table_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "postgresql_table_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "postgresql_table_name", + "type": "tag" + } + ], + "having": [], + "legend": "Table size", + "limit": null, + "orderBy": [], + "queryName": "D", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "26a9dcbf-4fc7-4ddd-b786-2078def1f462", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Table stats", + "yAxisUnit": "none" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/pipelines/log-parser.json b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/pipelines/log-parser.json new file mode 100644 index 00000000000..776565861c3 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/pipelines/log-parser.json @@ -0,0 +1,33 @@ +{ + "id": "parse-default-postgres-access-log", + "name": "Parse default postgres access log", + "alias": "parse-default-postgres-access-log", + "description": "Parse standard postgres access log", + "enabled": true, + "filter": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "postgres" + } + ] + }, + "config": [ + { + "type": "grok_parser", + "id": "parse-body-grok", + "enabled": true, + "orderId": 1, + "name": "Parse Body", + "parse_to": "attributes", + "pattern": "%{GREEDYDATA}", + "parse_from": "body" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/config/configure-otel-collector.md b/pkg/query-service/app/integrations/builtin_integrations/postgres/config/configure-otel-collector.md new file mode 100644 index 00000000000..3eb3a39b032 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/config/configure-otel-collector.md @@ -0,0 +1,72 @@ +### Configure otel collector + +#### Save collector config file + +Save the following collector config in a file named `postgres-collector-config.yaml` + +``` +receivers: + postgresql: + # The endpoint of the postgresql server. Whether using TCP or Unix sockets, this value should be host:port. If transport is set to unix, the endpoint will internally be translated from host:port to /host.s.PGSQL.port + endpoint: "localhost:5432" + # The frequency at which to collect metrics from the Postgres instance. + collection_interval: 60s + # The username used to access the postgres instance + username: monitoring + # The password used to access the postgres instance + password: ${env:POSTGRESQL_PASSWORD} + # The list of databases for which the receiver will attempt to collect statistics. If an empty list is provided, the receiver will attempt to collect statistics for all non-template databases + databases: [] + # List of databases which will be excluded when collecting statistics. + exclude_databases: [] + # # Defines the network to use for connecting to the server. Valid Values are `tcp` or `unix` + # transport: tcp + tls: + # set to false if SSL is enabled on the server + insecure: true + # ca_file: /etc/ssl/certs/ca-certificates.crt + # cert_file: /etc/ssl/certs/postgres.crt + # key_file: /etc/ssl/certs/postgres.key + metrics: + postgresql.database.locks: + enabled: true + postgresql.deadlocks: + enabled: true + postgresql.sequential_scans: + enabled: true + +processors: + # enriches the data with additional host information + # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor + resourcedetection/system: + # add additional detectors if needed + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # export to local collector + otlp/local: + endpoint: "localhost:4317" + tls: + insecure: true + # export to SigNoz cloud + otlp/signoz: + endpoint: "ingest.{region}.signoz.cloud:443" + tls: + insecure: false + headers: + "signoz-access-token": "" + +service: + pipelines: + metrics/postgresql: + receivers: [postgresql] + # note: remove this processor if the collector host is not running on the same host as the postgres instance + processors: [resourcedetection/system] + exporters: [otlp/local] +``` + +#### Use collector config file + +Run your collector with the added flag `--config postgres-collector-config.yaml` diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/config/prerequisites.md b/pkg/query-service/app/integrations/builtin_integrations/postgres/config/prerequisites.md new file mode 100644 index 00000000000..004692ba53c --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/config/prerequisites.md @@ -0,0 +1,26 @@ +### Prepare postgres for monitoring + +- Have a running postgresql instance +- Have the monitoring user created +- Have the monitoring user granted the necessary permissions + +This receiver supports PostgreSQL versions 9.6+ + +For PostgreSQL versions 10+, run: + +``` +create user monitoring with password ''; +grant pg_monitor to monitoring; +grant SELECT ON pg_stat_database to monitoring; +``` + +For PostgreSQL versions >= 9.6 and <10, run: + +``` +create user monitoring with password ''; +grant SELECT ON pg_stat_database to monitoring; +``` + +Set the following environment variables: + +- POSTGRESQL_PASSWORD diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/icon.svg b/pkg/query-service/app/integrations/builtin_integrations/postgres/icon.svg new file mode 100644 index 00000000000..32bcf493ef0 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/icon.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json b/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json new file mode 100644 index 00000000000..9b3da798bc8 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json @@ -0,0 +1,88 @@ +{ + "id": "postgres", + "title": "PostgreSQL", + "description": "Monitor postgres using logs and metrics.", + "author": { + "name": "SigNoz", + "email": "integrations@signoz.io", + "homepage": "https://signoz.io" + }, + "icon": "file://icon.svg", + "categories": [ + "Database" + ], + "overview": "file://overview.md", + "configuration": [ + { + "title": "Prerequisites", + "instructions": "file://config/prerequisites.md" + }, + { + "title": "Configure Otel Collector", + "instructions": "file://config/configure-otel-collector.md" + } + ], + "assets": { + "logs": { + "pipelines": [ + "file://assets/pipelines/log-parser.json" + ] + }, + "dashboards": [ + "file://assets/dashboards/overview.json" + ], + "alerts": [] + }, + "connection_tests": { + "logs": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "postgres" + } + ] + } + }, + "data_collected": { + "logs": [ + { + "name": "Request Method", + "path": "attributes[\"http.request.method\"]", + "type": "string", + "description": "HTTP method" + }, + { + "name": "Request Path", + "path": "attributes[\"url.path\"]", + "type": "string", + "description": "path requested" + }, + { + "name": "Response Status Code", + "path": "attributes[\"http.response.status_code\"]", + "type": "int", + "description": "HTTP response code" + } + ], + "metrics": [ + { + "name": "http.server.request.duration", + "type": "Histogram", + "unit": "s", + "description": "Duration of HTTP server requests" + }, + { + "name": "http.server.active_requests", + "type": "UpDownCounter", + "unit": "{ request }", + "description": "Number of active HTTP server requests" + } + ] + } +} diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/overview.md b/pkg/query-service/app/integrations/builtin_integrations/postgres/overview.md new file mode 100644 index 00000000000..4af57e6b20c --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/overview.md @@ -0,0 +1,3 @@ +### Monitor Postgres with SigNoz + +Parse your Postgres logs and collect key metrics. diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json new file mode 100644 index 00000000000..58e964af737 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json @@ -0,0 +1,923 @@ +{ + "description": "This dashboard shows the Redis instance overview. It includes latency, hit/miss rate, connections, and memory information.\n", + "layout": [ + { + "h": 3, + "i": "d4c164bc-8fc2-4dbc-aadd-8d17479ca649", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 9 + }, + { + "h": 3, + "i": "2fbaef0d-3cdb-4ce3-aa3c-9bbbb41786d9", + "moved": false, + "static": false, + "w": 6, + "x": 3, + "y": 6 + }, + { + "h": 3, + "i": "f5ee1511-0d2b-4404-9ce0-e991837decc2", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 3 + }, + { + "h": 3, + "i": "b19c7058-b806-4ea2-974a-ca555b168991", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 3 + }, + { + "h": 3, + "i": "bf0deeeb-e926-4234-944c-82bacd96af47", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 0 + }, + { + "h": 3, + "i": "a77227c7-16f5-4353-952e-b183c715a61c", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 0 + }, + { + "h": 3, + "i": "9698cee2-b1f3-4c0b-8c9f-3da4f0e05f17", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 9 + }, + { + "h": 3, + "i": "64a5f303-d7db-44ff-9a0e-948e5c653320", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 12 + }, + { + "h": 3, + "i": "3e80a918-69af-4c9a-bc57-a94e1d41b05c", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 12 + } + ], + "name": "", + "tags": [ + "redis", + "database" + ], + "title": "Redis overview", + "variables": { + "94f19b3c-ad9f-4b47-a9b2-f312c09fa965": { + "allSelected": true, + "customValue": "", + "description": "List of hosts sending Redis metrics", + "id": "94f19b3c-ad9f-4b47-a9b2-f312c09fa965", + "key": "94f19b3c-ad9f-4b47-a9b2-f312c09fa965", + "modificationUUID": "4c5b0c03-9cbc-425b-8d8e-7152e5c39ba8", + "multiSelect": true, + "name": "host_name", + "order": 0, + "queryValue": "SELECT JSONExtractString(labels, 'host_name') AS host_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'redis_cpu_time'\nGROUP BY host_name", + "selectedValue": [ + "Srikanths-MacBook-Pro.local" + ], + "showALLOption": true, + "sort": "ASC", + "textboxValue": "", + "type": "QUERY" + } + }, + "widgets": [ + { + "description": "Rate successful lookup of keys in the main dictionary", + "fillSpans": false, + "id": "a77227c7-16f5-4353-952e-b183c715a61c", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_keyspace_hits--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "redis_keyspace_hits", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "e99669ea", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Hit/s across all hosts", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "42c9c117-bfaf-49f7-b528-aad099392295", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Hits/s", + "yAxisUnit": "none" + }, + { + "description": "Number of clients pending on a blocking call", + "fillSpans": false, + "id": "bf0deeeb-e926-4234-944c-82bacd96af47", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_clients_blocked--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "redis_clients_blocked", + "type": "Sum" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "97247f25", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Blocked clients across all hosts", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "b77a9e11-fb98-4a95-88a8-c3ad25c14369", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Clients blocked", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "b19c7058-b806-4ea2-974a-ca555b168991", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "redis_db_keys------false", + "isColumn": false, + "key": "redis_db_keys", + "type": "" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "b77a9e11-fb98-4a95-88a8-c3ad25c14369", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Keyspace Keys", + "yAxisUnit": "none" + }, + { + "description": "Number of changes since the last dump", + "fillSpans": false, + "id": "f5ee1511-0d2b-4404-9ce0-e991837decc2", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_rdb_changes_since_last_save--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "redis_rdb_changes_since_last_save", + "type": "Sum" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "d4aef346", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "Number of unsaved changes", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "32cedddf-606d-4de1-8c1d-4b7049e6430c", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Unsaved changes", + "yAxisUnit": "none" + }, + { + "description": "", + "fillSpans": false, + "id": "2fbaef0d-3cdb-4ce3-aa3c-9bbbb41786d9", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_commands--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "redis_commands", + "type": "Gauge" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "458dc402", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [], + "having": [], + "legend": "ops/s", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "c70de4dd-a68a-42df-a249-6610c296709c", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Command/s", + "yAxisUnit": "ops" + }, + { + "description": "", + "fillSpans": false, + "id": "d4c164bc-8fc2-4dbc-aadd-8d17479ca649", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_memory_used--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "redis_memory_used", + "type": "Gauge" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "394a537e", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Used::{{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_maxmemory--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "redis_maxmemory", + "type": "Gauge" + }, + "aggregateOperator": "max", + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "0c0754da", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Max::{{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "2f47df76-f09e-4152-8623-971f0fe66bfe", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Memory usage", + "yAxisUnit": "bytes" + }, + { + "description": "", + "fillSpans": false, + "id": "9698cee2-b1f3-4c0b-8c9f-3da4f0e05f17", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_memory_rss--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "redis_memory_rss", + "type": "Gauge" + }, + "aggregateOperator": "sum", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "4dc9ae49", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Rss::{{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "fddd043c-1385-481c-9f4c-381f261e1dd9", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "RSS Memory", + "yAxisUnit": "bytes" + }, + { + "description": "", + "fillSpans": false, + "id": "64a5f303-d7db-44ff-9a0e-948e5c653320", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_memory_fragmentation_ratio--float64--Gauge--true", + "isColumn": true, + "isJSON": false, + "key": "redis_memory_fragmentation_ratio", + "type": "Gauge" + }, + "aggregateOperator": "avg", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "79dc25f3", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Rss::{{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "3e802b07-0249-4d79-a5c7-6580ab535ad0", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Fragmentation ratio", + "yAxisUnit": "short" + }, + { + "description": "Number of evicted keys due to maxmemory limit", + "fillSpans": false, + "id": "3e80a918-69af-4c9a-bc57-a94e1d41b05c", + "isStacked": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "redis_keys_evicted--float64--Sum--true", + "isColumn": true, + "isJSON": false, + "key": "redis_keys_evicted", + "type": "Sum" + }, + "aggregateOperator": "sum_rate", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "53d189ac", + "key": { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + }, + "op": "in", + "value": [ + "{{.host_name}}" + ] + } + ], + "op": "AND" + }, + "groupBy": [ + { + "dataType": "string", + "id": "host_name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "host_name", + "type": "tag" + } + ], + "having": [], + "legend": "Rss::{{host_name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "15d1d9d7-eb10-464b-aa7b-33ff211996f7", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "softMax": null, + "softMin": null, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Eviction rate", + "yAxisUnit": "short" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/assets/pipelines/log-parser.json b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/pipelines/log-parser.json new file mode 100644 index 00000000000..d06760e0b8a --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/pipelines/log-parser.json @@ -0,0 +1,33 @@ +{ + "id": "parse-default-redis-access-log", + "name": "Parse default redis access log", + "alias": "parse-default-redis-access-log", + "description": "Parse standard redis access log", + "enabled": true, + "filter": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "redis" + } + ] + }, + "config": [ + { + "type": "grok_parser", + "id": "parse-body-grok", + "enabled": true, + "orderId": 1, + "name": "Parse Body", + "parse_to": "attributes", + "pattern": "%{GREEDYDATA}", + "parse_from": "body" + } + ] +} \ No newline at end of file diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/config/configure-otel-collector.md b/pkg/query-service/app/integrations/builtin_integrations/redis/config/configure-otel-collector.md new file mode 100644 index 00000000000..21d586a771d --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/config/configure-otel-collector.md @@ -0,0 +1,63 @@ +### Configure otel collector + +#### Save collector config file + +Save the following collector config in a file named `redis-collector-config.yaml` + +``` +receivers: + redis: + # The hostname and port of the Redis instance, separated by a colon. + endpoint: "localhost:6379" + # The frequency at which to collect metrics from the Redis instance. + collection_interval: 60s + # # The password used to access the Redis instance; must match the password specified in the requirepass server configuration option. + # password: ${env:REDIS_PASSWORD} + # # Defines the network to use for connecting to the server. Valid Values are `tcp` or `Unix` + # transport: tcp + # tls: + # insecure: false + # ca_file: /etc/ssl/certs/ca-certificates.crt + # cert_file: /etc/ssl/certs/redis.crt + # key_file: /etc/ssl/certs/redis.key + metrics: + redis.maxmemory: + enabled: true + redis.cmd.latency: + enabled: true + +processors: + # enriches the data with additional host information + # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor + resourcedetection/system: + # add additional detectors if needed + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # export to local collector + otlp/local: + endpoint: "localhost:4317" + tls: + insecure: true + # export to SigNoz cloud + otlp/signoz: + endpoint: "ingest.{region}.signoz.cloud:443" + tls: + insecure: false + headers: + "signoz-access-token": "" + +service: + pipelines: + metrics/redis: + receivers: [redis] + # note: remove this processor if the collector host is not running on the same host as the redis instance + processors: [resourcedetection/system] + exporters: [otlp/local] +``` + +#### Use collector config file + +Run your collector with the added flag `--config redis-collector-config.yaml` diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/config/prerequisites.md b/pkg/query-service/app/integrations/builtin_integrations/redis/config/prerequisites.md new file mode 100644 index 00000000000..4e98933b69f --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/config/prerequisites.md @@ -0,0 +1,5 @@ +### Prepare redis for monitoring + +- Have a running redis instance +- Have the monitoring user created +- Have the monitoring user granted the necessary permissions diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/icon.svg b/pkg/query-service/app/integrations/builtin_integrations/redis/icon.svg new file mode 100644 index 00000000000..63e5dfd2e40 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json b/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json new file mode 100644 index 00000000000..862a98b3061 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json @@ -0,0 +1,88 @@ +{ + "id": "redis", + "title": "Redis", + "description": "Monitor redis using logs and metrics.", + "author": { + "name": "SigNoz", + "email": "integrations@signoz.io", + "homepage": "https://signoz.io" + }, + "icon": "file://icon.svg", + "categories": [ + "Database" + ], + "overview": "file://overview.md", + "configuration": [ + { + "title": "Prerequisites", + "instructions": "file://config/prerequisites.md" + }, + { + "title": "Configure Otel Collector", + "instructions": "file://config/configure-otel-collector.md" + } + ], + "assets": { + "logs": { + "pipelines": [ + "file://assets/pipelines/log-parser.json" + ] + }, + "dashboards": [ + "file://assets/dashboards/overview.json" + ], + "alerts": [] + }, + "connection_tests": { + "logs": { + "op": "AND", + "items": [ + { + "key": { + "type": "tag", + "key": "source", + "dataType": "string" + }, + "op": "=", + "value": "redis" + } + ] + } + }, + "data_collected": { + "logs": [ + { + "name": "Request Method", + "path": "attributes[\"http.request.method\"]", + "type": "string", + "description": "HTTP method" + }, + { + "name": "Request Path", + "path": "attributes[\"url.path\"]", + "type": "string", + "description": "path requested" + }, + { + "name": "Response Status Code", + "path": "attributes[\"http.response.status_code\"]", + "type": "int", + "description": "HTTP response code" + } + ], + "metrics": [ + { + "name": "http.server.request.duration", + "type": "Histogram", + "unit": "s", + "description": "Duration of HTTP server requests" + }, + { + "name": "http.server.active_requests", + "type": "UpDownCounter", + "unit": "{ request }", + "description": "Number of active HTTP server requests" + } + ] + } +} diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/overview.md b/pkg/query-service/app/integrations/builtin_integrations/redis/overview.md new file mode 100644 index 00000000000..60ce2337b69 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/overview.md @@ -0,0 +1,3 @@ +### Monitor Redis with SigNoz + +Parse your Redis logs and collect key metrics. diff --git a/pkg/query-service/app/integrations/builtin_test.go b/pkg/query-service/app/integrations/builtin_test.go new file mode 100644 index 00000000000..66d7cea0fa4 --- /dev/null +++ b/pkg/query-service/app/integrations/builtin_test.go @@ -0,0 +1,32 @@ +package integrations + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuiltinIntegrations(t *testing.T) { + require := require.New(t) + + repo := BuiltInIntegrations{} + + builtins, apiErr := repo.list(context.Background()) + require.Nil(apiErr) + require.Greater( + len(builtins), 0, + "some built in integrations are expected to be bundled.", + ) + + nginxIntegrationId := "builtin::nginx" + res, apiErr := repo.get(context.Background(), []string{ + nginxIntegrationId, + }) + require.Nil(apiErr) + + nginxIntegration, exists := res[nginxIntegrationId] + require.True(exists) + require.False(strings.HasPrefix(nginxIntegration.Overview, "file://")) +} diff --git a/pkg/query-service/app/integrations/controller.go b/pkg/query-service/app/integrations/controller.go index 72badb0c86c..e047333bc57 100644 --- a/pkg/query-service/app/integrations/controller.go +++ b/pkg/query-service/app/integrations/controller.go @@ -86,6 +86,12 @@ type UninstallIntegrationRequest struct { func (c *Controller) Uninstall( ctx context.Context, req *UninstallIntegrationRequest, ) *model.ApiError { + if len(req.IntegrationId) < 1 { + return model.BadRequest(fmt.Errorf( + "integration_id is required.", + )) + } + return c.mgr.UninstallIntegration( ctx, req.IntegrationId, ) diff --git a/pkg/query-service/app/integrations/manager.go b/pkg/query-service/app/integrations/manager.go index fa1c4b3ce6f..1685afff656 100644 --- a/pkg/query-service/app/integrations/manager.go +++ b/pkg/query-service/app/integrations/manager.go @@ -122,8 +122,7 @@ func NewManager(db *sqlx.DB) (*Manager, error) { } return &Manager{ - // TODO(Raj): Hook up a real available integrations provider. - availableIntegrationsRepo: &TestAvailableIntegrationsRepo{}, + availableIntegrationsRepo: &BuiltInIntegrations{}, installedIntegrationsRepo: iiRepo, }, nil }