expand the match_all to also optionally allow counting bytes per stre…

…am (grafana#2066)

Signed-off-by: Ed Welch <[email protected]>

docs/clients/promtail/stages/metrics.md

@@ -38,6 +38,11 @@ config:
   # attempting to match the source to the extract map.
   # It is an error to specify `match_all: true` and also specify a `value`
   [match_all: <bool>]
+
+  # If present and true all log line bytes will be counted.
+  # It is an error to specify `count_entry_bytes: true` without specifying `match_all: true`
+  # It is an error to specify `count_entry_bytes: true` without specifying `action: add`
+  [count_entry_bytes: <bool>]
 
   # Filters down source data and only changes the metric
   # if the targeted value exactly matches the provided string.
@@ -136,17 +141,30 @@ config:
       type: Counter
       description: "total number of log lines"
       prefix: my_promtail_custom_
-      source: time
+      match_all: true
       config:
         action: inc
+    log_bytes_total:
+      type: Counter
+      description: "total bytes of log lines"
+      prefix: my_promtail_custom_
+      match_all: true
+      count_entry_bytes: true
+      config:
+        action: add
 ```
 
-This pipeline creates a `log_lines_total` counter that increments whenever the
-extracted map contains a key for `time`. Since every log entry has a timestamp,
-this is a good field to use to count every line. Notice that `value` is not
-defined in the `config` section as we want to count every line and don't need to
-filter the value. Similarly, `inc` is used as the action because we want to
-increment the counter by one rather than by using the value of `time`.
+This pipeline creates a `log_lines_total` counter which increments for every log line received 
+by using the `match_all: true` parameter.
+
+It also creates a `log_bytes_total` counter which adds the byte size of every log line received 
+to the counter by using the `count_entry_bytes: true` parameter.
+
+The combination of these two metric stages will give you two counters to track the volume of 
+every log stream in both number of lines and bytes, which can be useful in identifying sources
+of very high volume, as well as helping understand why you may have too much cardinality.
+
+These stages should be placed towards the end of your pipeline after any `labels` stages
 
 ```yaml
 - regex:

pkg/logentry/metric/counters.go

@@ -14,15 +14,18 @@ const (
 	CounterInc = "inc"
 	CounterAdd = "add"
 
-	ErrCounterActionRequired  = "counter action must be defined as either `inc` or `add`"
-	ErrCounterInvalidAction   = "action %s is not valid, action must be either `inc` or `add`"
-	ErrCounterInvalidMatchAll = "`match_all: true` cannot be combined with `value`, please remove `match_all` or `value`"
+	ErrCounterActionRequired          = "counter action must be defined as either `inc` or `add`"
+	ErrCounterInvalidAction           = "action %s is not valid, action must be either `inc` or `add`"
+	ErrCounterInvalidMatchAll         = "`match_all: true` cannot be combined with `value`, please remove `match_all` or `value`"
+	ErrCounterInvalidCountBytes       = "`count_entry_bytes: true` can only be set with `match_all: true`"
+	ErrCounterInvalidCountBytesAction = "`count_entry_bytes: true` can only be used with `action: add`"
 )
 
 type CounterConfig struct {
-	MatchAll *bool   `mapstructure:"match_all"`
-	Value    *string `mapstructure:"value"`
-	Action   string  `mapstructure:"action"`
+	MatchAll   *bool   `mapstructure:"match_all"`
+	CountBytes *bool   `mapstructure:"count_entry_bytes"`
+	Value      *string `mapstructure:"value"`
+	Action     string  `mapstructure:"action"`
 }
 
 func validateCounterConfig(config *CounterConfig) error {
@@ -36,6 +39,12 @@ func validateCounterConfig(config *CounterConfig) error {
 	if config.MatchAll != nil && *config.MatchAll && config.Value != nil {
 		return errors.Errorf(ErrCounterInvalidMatchAll)
 	}
+	if config.CountBytes != nil && *config.CountBytes && (config.MatchAll == nil || !*config.MatchAll) {
+		return errors.New(ErrCounterInvalidCountBytes)
+	}
+	if config.CountBytes != nil && *config.CountBytes && config.Action != CounterAdd {
+		return errors.New(ErrCounterInvalidCountBytesAction)
+	}
 	return nil
 }
 

pkg/logentry/metric/counters_test.go

@@ -37,6 +37,30 @@ func Test_validateCounterConfig(t *testing.T) {
 			},
 			errors.New(ErrCounterInvalidMatchAll),
 		},
+		{"invalid counter match bytes",
+			CounterConfig{
+				MatchAll:   nil,
+				CountBytes: &counterTestTrue,
+				Action:     "add",
+			},
+			errors.New(ErrCounterInvalidCountBytes),
+		},
+		{"invalid counter match bytes action",
+			CounterConfig{
+				MatchAll:   &counterTestTrue,
+				CountBytes: &counterTestTrue,
+				Action:     "inc",
+			},
+			errors.New(ErrCounterInvalidCountBytesAction),
+		},
+		{"valid counter match bytes",
+			CounterConfig{
+				MatchAll:   &counterTestTrue,
+				CountBytes: &counterTestTrue,
+				Action:     "add",
+			},
+			nil,
+		},
 		{"valid",
 			CounterConfig{
 				Value:  &counterTestVal,

pkg/logentry/stages/metrics.go

@@ -151,7 +151,13 @@ func (m *metricStage) Process(labels model.LabelSet, extracted map[string]interf
 		// There is a special case for counters where we count even if there is no match in the extracted map.
 		if c, ok := collector.(*metric.Counters); ok {
 			if c != nil && c.Cfg.MatchAll != nil && *c.Cfg.MatchAll {
-				m.recordCounter(name, c, labels, nil)
+				if c.Cfg.CountBytes != nil && *c.Cfg.CountBytes {
+					if entry != nil {
+						m.recordCounter(name, c, labels, len(*entry))
+					}
+				} else {
+					m.recordCounter(name, c, labels, nil)
+				}
 				continue
 			}
 		}

pkg/logentry/stages/metrics_test.go

@@ -45,6 +45,13 @@ pipeline_stages:
       config:
         match_all: true
         action: inc
+    total_bytes_count:
+      type: Counter
+      description: nothing to see here...
+      config:
+        match_all: true
+        count_entry_bytes: true
+        action: add
     payload_size_bytes:
       type: Histogram
       description: grrrragh
@@ -93,6 +100,9 @@ promtail_custom_payload_size_bytes_bucket{test="app",le="20"} 2
 promtail_custom_payload_size_bytes_bucket{test="app",le="+Inf"} 2
 promtail_custom_payload_size_bytes_sum{test="app"} 30
 promtail_custom_payload_size_bytes_count{test="app"} 2
+# HELP promtail_custom_total_bytes_count nothing to see here...
+# TYPE promtail_custom_total_bytes_count counter
+promtail_custom_total_bytes_count{test="app"} 255
 # HELP promtail_custom_total_lines_count nothing to see here...
 # TYPE promtail_custom_total_lines_count counter
 promtail_custom_total_lines_count{test="app"} 2