Add rule evaluation time

rules/alerting.go

@@ -99,6 +99,8 @@ type AlertingRule struct {
 	labels labels.Labels
 	// Non-identifying key/value pairs.
 	annotations labels.Labels
+	// Time in seconds taken to evaluate rule.
+	evaluationTimeSeconds float64
 
 	// Protects the below.
 	mtx sync.Mutex
@@ -149,6 +151,20 @@ func (r *AlertingRule) sample(alert *Alert, ts time.Time) promql.Sample {
 	return s
 }
 
+// setEvaluationTimeSeconds updates evaluationSeconds to the time in seconds it took to evaluate the rule on its last evaluation.
+func (r *AlertingRule) setEvaluationTimeSeconds(seconds float64) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	r.evaluationTimeSeconds = seconds
+}
+
+// GetEvaluationTimeSeconds returns the time in seconds it took to evaluate the alerting rule.
+func (r *AlertingRule) GetEvaluationTimeSeconds() float64 {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	return r.evaluationTimeSeconds
+}
+
 // resolvedRetention is the duration for which a resolved alert instance
 // is kept in memory state and consequentally repeatedly sent to the AlertManager.
 const resolvedRetention = 15 * time.Minute

rules/manager.go

@@ -115,19 +115,24 @@ type Rule interface {
 	Eval(context.Context, time.Time, *promql.Engine, *url.URL) (promql.Vector, error)
 	// String returns a human-readable string representation of the rule.
 	String() string
+
+	setEvaluationTimeSeconds(float64)
+	GetEvaluationTimeSeconds() float64
 	// HTMLSnippet returns a human-readable string representation of the rule,
 	// decorated with HTML elements for use the web frontend.
 	HTMLSnippet(pathPrefix string) html_template.HTML
 }
 
 // Group is a set of rules that have a logical relation.
 type Group struct {
-	name                 string
-	file                 string
-	interval             time.Duration
-	rules                []Rule
-	seriesInPreviousEval []map[string]labels.Labels // One per Rule.
-	opts                 *ManagerOptions
+	name                  string
+	file                  string
+	interval              time.Duration
+	rules                 []Rule
+	seriesInPreviousEval  []map[string]labels.Labels // One per Rule.
+	opts                  *ManagerOptions
+	evaluationTimeSeconds float64
+	mtx                   sync.Mutex
 
 	done       chan struct{}
 	terminated chan struct{}
@@ -176,6 +181,7 @@ func (g *Group) run() {
 		g.Eval(start)
 
 		iterationDuration.Observe(time.Since(start).Seconds())
+		g.setEvaluationTimeSeconds(time.Since(start).Seconds())
 	}
 	lastTriggered := time.Now()
 	iter()
@@ -218,6 +224,19 @@ func (g *Group) hash() uint64 {
 	return l.Hash()
 }
 
+// GetEvaluationTimeSeconds returns the time in seconds it took to evaluate the rule group.
+func (g *Group) GetEvaluationTimeSeconds() float64 {
+	g.mtx.Lock()
+	defer g.mtx.Unlock()
+	return g.evaluationTimeSeconds
+}
+
+func (g *Group) setEvaluationTimeSeconds(seconds float64) {
+	g.mtx.Lock()
+	defer g.mtx.Unlock()
+	g.evaluationTimeSeconds = seconds
+}
+
 // offset returns until the next consistently slotted evaluation interval.
 func (g *Group) offset() time.Duration {
 	now := time.Now().UnixNano()
@@ -294,6 +313,7 @@ func (g *Group) Eval(ts time.Time) {
 		func(i int, rule Rule) {
 			defer func(t time.Time) {
 				evalDuration.WithLabelValues(rtyp).Observe(time.Since(t).Seconds())
+				rule.setEvaluationTimeSeconds(time.Since(t).Seconds())
 			}(time.Now())
 
 			evalTotal.WithLabelValues(rtyp).Inc()

rules/recording.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"html/template"
 	"net/url"
+	"sync"
 	"time"
 
 	yaml "gopkg.in/yaml.v2"
@@ -30,9 +31,11 @@ import (
 
 // A RecordingRule records its vector expression into new timeseries.
 type RecordingRule struct {
-	name   string
-	vector promql.Expr
-	labels labels.Labels
+	name                  string
+	vector                promql.Expr
+	labels                labels.Labels
+	mtx                   sync.Mutex
+	evaluationTimeSeconds float64
 }
 
 // NewRecordingRule returns a new recording rule.
@@ -45,12 +48,12 @@ func NewRecordingRule(name string, vector promql.Expr, lset labels.Labels) *Reco
 }
 
 // Name returns the rule name.
-func (rule RecordingRule) Name() string {
+func (rule *RecordingRule) Name() string {
 	return rule.name
 }
 
 // Eval evaluates the rule and then overrides the metric names and labels accordingly.
-func (rule RecordingRule) Eval(ctx context.Context, ts time.Time, engine *promql.Engine, _ *url.URL) (promql.Vector, error) {
+func (rule *RecordingRule) Eval(ctx context.Context, ts time.Time, engine *promql.Engine, _ *url.URL) (promql.Vector, error) {
 	query, err := engine.NewInstantQuery(rule.vector.String(), ts)
 	if err != nil {
 		return nil, err
@@ -98,7 +101,7 @@ func (rule RecordingRule) Eval(ctx context.Context, ts time.Time, engine *promql
 	return vector, nil
 }
 
-func (rule RecordingRule) String() string {
+func (rule *RecordingRule) String() string {
 	r := rulefmt.Rule{
 		Record: rule.name,
 		Expr:   rule.vector.String(),
@@ -113,8 +116,22 @@ func (rule RecordingRule) String() string {
 	return string(byt)
 }
 
+// setEvaluationTimeSeconds updates evaluationTimeSeconds to the time in seconds it took to evaluate the rule on its last evaluation.
+func (rule *RecordingRule) setEvaluationTimeSeconds(seconds float64) {
+	rule.mtx.Lock()
+	defer rule.mtx.Unlock()
+	rule.evaluationTimeSeconds = seconds
+}
+
+// GetEvaluationTimeSeconds returns the time in seconds it took to evaluate the recording rule.
+func (rule *RecordingRule) GetEvaluationTimeSeconds() float64 {
+	rule.mtx.Lock()
+	defer rule.mtx.Unlock()
+	return rule.evaluationTimeSeconds
+}
+
 // HTMLSnippet returns an HTML snippet representing this rule.
-func (rule RecordingRule) HTMLSnippet(pathPrefix string) template.HTML {
+func (rule *RecordingRule) HTMLSnippet(pathPrefix string) template.HTML {
 	ruleExpr := rule.vector.String()
 	labels := make(map[string]string, len(rule.labels))
 	for _, l := range rule.labels {