pkg/distributor: fix metric loki_discarded_samples_total (grafana#1279)

loki_discarded_samples_total is supposed to be the count of samples that
were discarded, but was instead being reported as the total bytes
discarded. The metric has been fixed and a new metric,
loki_discarded_bytes_total, has been added to report the total discarded
bytes.

Fixes grafana#1255.

pkg/distributor/distributor.go

@@ -191,6 +191,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
 	keys := make([]uint32, 0, len(req.Streams))
 	var validationErr error
 	validatedSamplesSize := 0
+	validatedSamplesCount := 0
 
 	for _, stream := range req.Streams {
 		if err := d.validateLabels(userID, stream.Labels); err != nil {
@@ -208,6 +209,7 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
 			}
 			entries = append(entries, entry)
 			validatedSamplesSize += len(entry.Line)
+			validatedSamplesCount++
 		}
 
 		if len(entries) == 0 {
@@ -228,8 +230,9 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
 	if !limiter.AllowN(time.Now(), validatedSamplesSize) {
 		// Return a 4xx here to have the client discard the data and not retry. If a client
 		// is sending too much data consistently we will unlikely ever catch up otherwise.
-		validation.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedSamplesSize))
-		return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%d) exceeded while adding %d lines", int(limiter.Limit()), lineCount)
+		validation.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedSamplesCount))
+		validation.DiscardedBytes.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedSamplesSize))
+		return nil, httpgrpc.Errorf(http.StatusTooManyRequests, "ingestion rate limit (%d) exceeded while adding %d lines", int(limiter.Limit()), validatedSamplesCount)
 	}
 
 	const maxExpectedReplicationSet = 5 // typical replication factor 3 plus one for inactive plus one for luck

pkg/util/validation/validate.go

@@ -10,6 +10,15 @@ const (
 	RateLimited = "rate_limited"
 )
 
+// DiscardedBytes is a metric of the total discarded bytes, by reason.
+var DiscardedBytes = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "loki_discarded_bytes_total",
+		Help: "The total number of bytes that were discarded.",
+	},
+	[]string{discardReasonLabel, "user"},
+)
+
 // DiscardedSamples is a metric of the number of discarded samples, by reason.
 var DiscardedSamples = prometheus.NewCounterVec(
 	prometheus.CounterOpts{