Expose metrics on job status

autoheal_actions_initiated_total is renamed to autoheal_actions_launched
and is now a gauge instead of a counter. this gauge is used from the
moment we launch a job and until it is finished(including).

Author: Mooli Tayer

cmd/autoheal/active_jobs_worker.go

@@ -19,23 +19,30 @@ package main
 import (
 	"github.com/golang/glog"
 	"k8s.io/apimachinery/pkg/util/runtime"
+
+	"github.com/openshift/autoheal/pkg/apis/autoheal"
 )
 
 func (h *Healer) runActiveJobsWorker() {
 	glog.Infof("Going over active jobs queue.")
 
 	finishedJobs := make([]int, 0)
 
-	h.activeJobs.Range(func(_, value interface{}) bool {
-		id := value.(int)
-
+	h.activeJobs.Range(func(key interface{}, value interface{}) bool {
+		id := key.(int)
+		rule := value.(*autoheal.HealingRule)
 		finished, err := h.checkAWXJobStatus(id)
 		if err != nil {
 			runtime.HandleError(err)
 		}
 
 		if finished {
 			finishedJobs = append(finishedJobs, id)
+			h.actionCompleted(
+				"AWXJob",
+				rule.AWXJob.Template,
+				rule.ObjectMeta.Name,
+			)
 		}
 		return true
 	})

cmd/autoheal/awx_job.go

@@ -111,10 +111,14 @@ func (h *Healer) launchAWXJob(
 		templateName,
 		response.Job,
 	)
-	h.incrementAwxActions(action, rule.ObjectMeta.Name)
+	h.actionStarted(
+		"AWXJob",
+		templateName,
+		rule.ObjectMeta.Name,
+	)
 
 	// Add the job to active jobs map for tracking
-	h.activeJobs.Store(response.Job, response.Job)
+	h.activeJobs.Store(response.Job, rule)
 
 	return nil
 }
@@ -158,7 +162,6 @@ func (h *Healer) checkAWXJobStatus(jobID int) (finished bool, err error) {
 	)
 
 	finished = job.IsFinished()
-	// TODO: save status as metric
 
 	return
 }

cmd/autoheal/metrics_exporter.go

@@ -5,8 +5,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/openshift/autoheal/pkg/apis/autoheal"
 )
 
 var (
@@ -17,12 +15,12 @@ var (
 		},
 		[]string{"type", "rule", "alert"},
 	)
-	actionsInitiated = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "autoheal_actions_initiated_total",
-			Help: "Number of initiated healing actions",
+	actionsLaunched = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "autoheal_actions_launched",
+			Help: "Number of launched healing actions(including completed)",
 		},
-		[]string{"type", "template", "rule"},
+		[]string{"type", "template", "rule", "status"},
 	)
 )
 
@@ -31,18 +29,43 @@ func (h *Healer) metricsHandler() http.Handler {
 }
 
 func (h *Healer) initExportedMetrics() {
-	prometheus.MustRegister(actionsRequested, actionsInitiated)
+	prometheus.MustRegister(actionsRequested, actionsLaunched)
 }
 
-func (h *Healer) incrementAwxActions(
-	action *autoheal.AWXJobAction,
+func (h *Healer) actionStarted(
+	actionType,
+	templateName,
 	ruleName string,
 ) {
-	actionsInitiated.With(
+	actionsLaunched.With(
+		map[string]string{
+			"type":     actionType,
+			"template": templateName,
+			"rule":     ruleName,
+			"status":   "running",
+		},
+	).Inc()
+}
+
+func (h *Healer) actionCompleted(
+	actionType,
+	templateName,
+	ruleName string,
+) {
+	actionsLaunched.With(
+		map[string]string{
+			"type":     actionType,
+			"template": templateName,
+			"rule":     ruleName,
+			"status":   "running",
+		},
+	).Dec()
+	actionsLaunched.With(
 		map[string]string{
-			"type":     "awxJob",
-			"template": action.Template,
+			"type":     actionType,
+			"template": templateName,
 			"rule":     ruleName,
+			"status":   "completed",
 		},
 	).Inc()
 }

documentation/metrics.md

@@ -20,16 +20,16 @@ All these metrics are prefixed with `autoheal_actions_`
 
 | Name             | Description                         | Type    |
 |------------------|-------------------------------------|---------|
-| initiated_total  | Number of initiated healing actions | Counter |
+| launched         | Number of started healing actions   | Gauge   |
 | requested_total  | Number of requested healing actions | Counter |
 
-`initiated_total` indicates how many healing actions were successfully kicked off by the server. An AWX type action is counted when a SUCCESSFUL `launch` request was done against an AWX server.
-
 `requested_total` indicates how many healing actions were triggered by the server. An action that
 was rate limited by the server is counted here as well as a heal that failed to run for some reason.
 For example if autoheal failed to contact AWX for an AWX job, a heal will not start
 but it will be counted as requested.
 
+`launched` indicates how many healing actions started, partitioned by status `running`|`completed`.
+
 ## Prometheus supplied metrics
 
 The Prometheus client library provides a number of metrics under the `go` and `process` namespaces that pertain to the entire process and the go runtime of the entire process. To find out more about these, see: