Merge pull request google#852 from anushree-n/genericCollector

Modify generic collector
erichhsun · Aug 13, 2015 · 2a022a4 · 2a022a4
2 parents 2a7218e + 04a7850
commit 2a022a4
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 51 deletions.
diff --git a/api/versions.go b/api/versions.go
@@ -372,19 +372,33 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 		if err != nil {
 			return err
 		}
-		specs, err := m.GetContainerSpec(containerName, opt)
-		if err != nil {
-			return err
-		}
-		contMetrics := make(map[string]map[string][]info.MetricVal, 0)
+		contMetrics := make(map[string]map[string]map[string][]info.MetricValBasic, 0)
 		for _, cont := range conts {
-			metrics := map[string][]info.MetricVal{}
+			metrics := make(map[string]map[string][]info.MetricValBasic, 0)
 			contStats := convertStats(cont)
-			spec := specs[cont.Name]
 			for _, contStat := range contStats {
-				for _, ms := range spec.CustomMetrics {
-					if contStat.HasCustomMetrics && !contStat.CustomMetrics[ms.Name].Timestamp.IsZero() {
-						metrics[ms.Name] = append(metrics[ms.Name], contStat.CustomMetrics[ms.Name])
+				if contStat.HasCustomMetrics {
+					for name, allLabels := range contStat.CustomMetrics {
+						metricLabels := make(map[string][]info.MetricValBasic, 0)
+						for _, metric := range allLabels {
+							if !metric.Timestamp.IsZero() {
+								metVal := info.MetricValBasic{
+									Timestamp:  metric.Timestamp,
+									IntValue:   metric.IntValue,
+									FloatValue: metric.FloatValue,
+								}
+								labels := metrics[name]
+								if labels != nil {
+									values := labels[metric.Label]
+									values = append(values, metVal)
+									labels[metric.Label] = values
+									metrics[name] = labels
+								} else {
+									metricLabels[metric.Label] = []info.MetricValBasic{metVal}
+									metrics[name] = metricLabels
+								}
+							}
+						}
 					}
 				}
 			}

diff --git a/collector/collector_manager.go b/collector/collector_manager.go
@@ -71,12 +71,12 @@ func (cm *GenericCollectorManager) GetSpec() ([]v1.MetricSpec, error) {
 	return metricSpec, nil
 }
 
-func (cm *GenericCollectorManager) Collect() (time.Time, map[string]v1.MetricVal, error) {
+func (cm *GenericCollectorManager) Collect() (time.Time, map[string][]v1.MetricVal, error) {
 	var errors []error
 
 	// Collect from all collectors that are ready.
 	var next time.Time
-	metrics := map[string]v1.MetricVal{}
+	metrics := map[string][]v1.MetricVal{}
 	for _, c := range cm.Collectors {
 		if c.nextCollectionTime.Before(time.Now()) {
 			var err error

diff --git a/collector/collector_manager_test.go b/collector/collector_manager_test.go
@@ -28,7 +28,7 @@ type fakeCollector struct {
 	collectedFrom      int
 }
 
-func (fc *fakeCollector) Collect(metric map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
+func (fc *fakeCollector) Collect(metric map[string][]v1.MetricVal) (time.Time, map[string][]v1.MetricVal, error) {
 	fc.collectedFrom++
 	return fc.nextCollectionTime, metric, fc.err
 }

diff --git a/collector/fakes.go b/collector/fakes.go
@@ -31,7 +31,7 @@ func (fkm *FakeCollectorManager) GetSpec() ([]v1.MetricSpec, error) {
 	return []v1.MetricSpec{}, nil
 }
 
-func (fkm *FakeCollectorManager) Collect(metric map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
+func (fkm *FakeCollectorManager) Collect(metric map[string][]v1.MetricVal) (time.Time, map[string][]v1.MetricVal, error) {
 	var zero time.Time
 	return zero, metric, nil
 }
diff --git a/collector/generic_collector.go b/collector/generic_collector.go
@@ -116,7 +116,7 @@ func (collector *GenericCollector) GetSpec() []v1.MetricSpec {
 }
 
 //Returns collected metrics and the next collection time of the collector
-func (collector *GenericCollector) Collect(metrics map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error) {
+func (collector *GenericCollector) Collect(metrics map[string][]v1.MetricVal) (time.Time, map[string][]v1.MetricVal, error) {
 	currentTime := time.Now()
 	nextCollectionTime := currentTime.Add(time.Duration(collector.info.minPollingFrequency))
 
@@ -142,16 +142,16 @@ func (collector *GenericCollector) Collect(metrics map[string]v1.MetricVal) (tim
 				if err != nil {
 					errorSlice = append(errorSlice, err)
 				}
-				metrics[metricConfig.Name] = v1.MetricVal{
-					FloatValue: regVal, Timestamp: currentTime,
+				metrics[metricConfig.Name] = []v1.MetricVal{
+					{FloatValue: regVal, Timestamp: currentTime},
 				}
 			} else if metricConfig.DataType == v1.IntType {
 				regVal, err := strconv.ParseInt(strings.TrimSpace(matchString[1]), 10, 64)
 				if err != nil {
 					errorSlice = append(errorSlice, err)
 				}
-				metrics[metricConfig.Name] = v1.MetricVal{
-					IntValue: regVal, Timestamp: currentTime,
+				metrics[metricConfig.Name] = []v1.MetricVal{
+					{IntValue: regVal, Timestamp: currentTime},
 				}
 
 			} else {

diff --git a/collector/generic_collector_test.go b/collector/generic_collector_test.go
@@ -148,20 +148,20 @@ func TestMetricCollection(t *testing.T) {
 	defer tempServer.Close()
 	fakeCollector.configFile.Endpoint = tempServer.URL
 
-	metrics := map[string]v1.MetricVal{}
+	metrics := map[string][]v1.MetricVal{}
 	_, metrics, errMetric := fakeCollector.Collect(metrics)
 	assert.NoError(errMetric)
 	metricNames := []string{"activeConnections", "reading", "writing", "waiting"}
 	// activeConnections = 3
-	assert.Equal(metrics[metricNames[0]].IntValue, 3)
-	assert.Equal(metrics[metricNames[0]].FloatValue, 0)
+	assert.Equal(metrics[metricNames[0]][0].IntValue, 3)
+	assert.Equal(metrics[metricNames[0]][0].FloatValue, 0)
 	// reading = 0
-	assert.Equal(metrics[metricNames[1]].IntValue, 0)
-	assert.Equal(metrics[metricNames[1]].FloatValue, 0)
+	assert.Equal(metrics[metricNames[1]][0].IntValue, 0)
+	assert.Equal(metrics[metricNames[1]][0].FloatValue, 0)
 	// writing = 1
-	assert.Equal(metrics[metricNames[2]].IntValue, 1)
-	assert.Equal(metrics[metricNames[2]].FloatValue, 0)
+	assert.Equal(metrics[metricNames[2]][0].IntValue, 1)
+	assert.Equal(metrics[metricNames[2]][0].FloatValue, 0)
 	// waiting = 2
-	assert.Equal(metrics[metricNames[3]].IntValue, 2)
-	assert.Equal(metrics[metricNames[3]].FloatValue, 0)
+	assert.Equal(metrics[metricNames[3]][0].IntValue, 2)
+	assert.Equal(metrics[metricNames[3]][0].FloatValue, 0)
 }
diff --git a/collector/types.go b/collector/types.go
@@ -27,7 +27,7 @@ type Collector interface {
 	// Returns the next time this collector should be collected from.
 	// Next collection time is always returned, even when an error occurs.
 	// A collection time of zero means no more collection.
-	Collect(map[string]v1.MetricVal) (time.Time, map[string]v1.MetricVal, error)
+	Collect(map[string][]v1.MetricVal) (time.Time, map[string][]v1.MetricVal, error)
 
 	// Return spec for all metrics associated with this collector
 	GetSpec() []v1.MetricSpec
@@ -45,7 +45,7 @@ type CollectorManager interface {
 	// at which a collector will be ready to collect from.
 	// Next collection time is always returned, even when an error occurs.
 	// A collection time of zero means no more collection.
-	Collect() (time.Time, map[string]v1.MetricVal, error)
+	Collect() (time.Time, map[string][]v1.MetricVal, error)
 
 	// Get metric spec from all registered collectors.
 	GetSpec() ([]v1.MetricSpec, error)

diff --git a/info/v1/container.go b/info/v1/container.go
@@ -427,7 +427,7 @@ type ContainerStats struct {
 	TaskStats LoadStats `json:"task_stats,omitempty"`
 
 	//Custom metrics from all collectors
-	CustomMetrics map[string]MetricVal `json:"custom_metrics,omitempty"`
+	CustomMetrics map[string][]MetricVal `json:"custom_metrics,omitempty"`
 }
 
 func timeEq(t1, t2 time.Time, tolerance time.Duration) bool {

diff --git a/info/v1/metric.go b/info/v1/metric.go
@@ -55,8 +55,21 @@ type MetricSpec struct {
 	Units string `json:"units"`
 }
 
+// An exported metric.
+type MetricValBasic struct {
+	// Time at which the metric was queried
+	Timestamp time.Time `json:"timestamp"`
+
+	// The value of the metric at this point.
+	IntValue   int64   `json:"int_value,omitempty"`
+	FloatValue float64 `json:"float_value,omitempty"`
+}
+
 // An exported metric.
 type MetricVal struct {
+	// Label associated with a metric
+	Label string `json:"label,omitempty"`
+
 	// Time at which the metric was queried
 	Timestamp time.Time `json:"timestamp"`
 

diff --git a/info/v2/container.go b/info/v2/container.go
@@ -104,8 +104,8 @@ type ContainerStats struct {
 	HasLoad bool         `json:"has_load"`
 	Load    v1.LoadStats `json:"load_stats,omitempty"`
 	// Custom Metrics
-	HasCustomMetrics bool                    `json:"has_custom_metrics"`
-	CustomMetrics    map[string]v1.MetricVal `json:"custom_metrics,omitempty"`
+	HasCustomMetrics bool                      `json:"has_custom_metrics"`
+	CustomMetrics    map[string][]v1.MetricVal `json:"custom_metrics,omitempty"`
 }
 
 type Percentiles struct {

diff --git a/manager/container.go b/manager/container.go
@@ -529,7 +529,7 @@ func (c *containerData) updateStats() error {
 	return customStatsErr
 }
 
-func (c *containerData) updateCustomStats() (map[string]info.MetricVal, error) {
+func (c *containerData) updateCustomStats() (map[string][]info.MetricVal, error) {
 	_, customStats, customStatsErr := c.collectorManager.Collect()
 	if customStatsErr != nil {
 		if !c.handler.Exists() {

diff --git a/pages/static/containers_js.go b/pages/static/containers_js.go
@@ -740,31 +740,31 @@ function drawCustomMetrics(elementId, containerInfo, metricsInfo) {
 	if(metricsInfo.length == 0) {
 		return;
 	}
-
         var metricSpec = containerInfo.spec.custom_metrics;
-
 	for (var containerName in metricsInfo) {
 		var container = metricsInfo[containerName];
 		for (i=0; i<metricSpec.length; i++) {
 			metricName = metricSpec[i].name;
 			metricUnits = metricSpec[i].units;
-
 			var titles = ["Time", metricName];
 			var data = [];
-			metricVal = container[metricName];
-			for (var index in metricVal) {
-                                metric = metricVal[index];
-                                var elements = [];
-                                for (var attribute in metric) {
-                                        value = metric[attribute];
-                                        elements.push(value);
-                                }
-                                if (elements.length<2) {
-                                        elements.push(0);
-                                }
-                                data.push(elements);
-                        }
-                       drawLineChart(titles, data, elementId+"-"+metricName, metricUnits);			
+			metricLabelVal = container[metricName];
+			for (var label in metricLabelVal) {
+				metricVal = metricLabelVal[label];
+				for (var index in metricVal) {
+        	                        metric = metricVal[index];
+                	                var elements = [];
+                        	        for (var attribute in metric) {
+                                	        value = metric[attribute];
+                                        	elements.push(value);
+                                	}
+                    	       		if (elements.length<2) {
+                        	                elements.push(0);
+                                	}
+                                	data.push(elements);
+                        	}
+  		                drawLineChart(titles, data, elementId+"-"+metricName, metricUnits);
+			}
 		}
 	}
 }