Checkpoint

collectors/monitoring_metrics.go

@@ -142,6 +142,7 @@ func (t *timeSeriesMetrics) newConstHistogram(fqName string, reportTime time.Tim
 		prometheus.MustNewConstHistogram(
 			t.newMetricDesc(fqName, labelKeys),
 			count,
+			// TODO why do this calc here vs storing sum on the histogram model
 			mean*float64(count), // Stackdriver does not provide the sum, but we can fake it
 			buckets,
 			labelValues...,

delta/histogram.go

@@ -102,22 +102,19 @@ func toHistogramKey(hist *collectors.HistogramMetric) uint64 {
 }
 
 func mergeHistograms(existing *collectors.HistogramMetric, current *collectors.HistogramMetric) *collectors.HistogramMetric {
+	// Calculate a new mean based on the factor
+	feCount := float64(existing.Count)
+	ccCount := float64(current.Count)
+	existingMeanFactor := existing.Mean * feCount
+	currentMeanFactor := current.Mean * ccCount
+	mean := (existingMeanFactor + currentMeanFactor) / (feCount + ccCount)
+	current.Mean = mean
+
+	// We merged the buckets and need to re-compute the total count
 	for key, value := range existing.Buckets {
 		current.Buckets[key] += value
 	}
-
-	// Calculate a new mean and overall count
-	mean := existing.Mean
-	mean += current.Mean
-	mean /= 2
-
-	var count uint64
-	for _, v := range current.Buckets {
-		count += v
-	}
-
-	current.Mean = mean
-	current.Count = count
+	current.Count = existing.Count + current.Count
 
 	return current
 }
@@ -136,7 +133,7 @@ func (s *InMemoryHistogramStore) ListMetrics(metricDescriptorName string) []*col
 	entry.mutex.Lock()
 	defer entry.mutex.Unlock()
 	for key, collected := range entry.Collected {
-		//Scan and remove metrics which are outside the TTL
+		// Scan and remove metrics which are outside the TTL
 		if ttlWindowStart.After(collected.CollectionTime) {
 			level.Debug(s.logger).Log("msg", "Deleting histogram entry outside of TTL", "key", key, "fqName", collected.FqName)
 			delete(entry.Collected, key)