Skip to content

Commit

Permalink
refactor: alert rule mute strategy
Browse files Browse the repository at this point in the history
  • Loading branch information
710leo committed Aug 19, 2023
1 parent 9c4775f commit 9cdbda0
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 55 deletions.
5 changes: 1 addition & 4 deletions alert/eval/alert_rule.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,7 @@ func (s *Scheduler) syncAlertRules() {
if rule == nil {
continue
}
// 同步rule到Scheduler时,未生效的rule应该直接drop,减少对时序数据库查询次数
if rule.TimeSpanMuteStrategy() {
continue
}

if rule.IsPrometheusRule() {
datasourceIds := s.promClients.Hit(rule.DatasourceIdsJson)
for _, dsId := range datasourceIds {
Expand Down
54 changes: 50 additions & 4 deletions alert/mute/mute.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ func IsMuted(rule *models.AlertRule, event *models.AlertCurEvent, targetCache *m
return true
}

// 移到Sync Rule之前就判断
// if TimeSpanMuteStrategy(rule, event) {
// return true
// }
if TimeSpanMuteStrategy(rule, event) {
return true
}

if IdentNotExistsMuteStrategy(rule, event, targetCache) {
return true
Expand All @@ -37,6 +36,53 @@ func IsMuted(rule *models.AlertRule, event *models.AlertCurEvent, targetCache *m
return false
}

// TimeSpanMuteStrategy 根据规则配置的告警生效时间段过滤,如果产生的告警不在规则配置的告警生效时间段内,则不告警,即被mute
// 时间范围,左闭右开,默认范围:00:00-24:00
func TimeSpanMuteStrategy(rule *models.AlertRule, event *models.AlertCurEvent) bool {
tm := time.Unix(event.TriggerTime, 0)
triggerTime := tm.Format("15:04")
triggerWeek := strconv.Itoa(int(tm.Weekday()))

enableStime := strings.Fields(rule.EnableStime)
enableEtime := strings.Fields(rule.EnableEtime)
enableDaysOfWeek := strings.Split(rule.EnableDaysOfWeek, ";")
length := len(enableDaysOfWeek)
// enableStime,enableEtime,enableDaysOfWeek三者长度肯定相同,这里循环一个即可
for i := 0; i < length; i++ {
enableDaysOfWeek[i] = strings.Replace(enableDaysOfWeek[i], "7", "0", 1)
if !strings.Contains(enableDaysOfWeek[i], triggerWeek) {
continue
}

if enableStime[i] < enableEtime[i] {
if enableEtime[i] == "23:59" {
// 02:00-23:59,这种情况做个特殊处理,相当于左闭右闭区间了
if triggerTime < enableStime[i] {
// mute, 即没生效
continue
}
} else {
// 02:00-04:00 或者 02:00-24:00
if triggerTime < enableStime[i] || triggerTime >= enableEtime[i] {
// mute, 即没生效
continue
}
}
} else if enableStime[i] > enableEtime[i] {
// 21:00-09:00
if triggerTime < enableStime[i] && triggerTime >= enableEtime[i] {
// mute, 即没生效
continue
}
}

// 到这里说明当前时刻在告警规则的某组生效时间范围内,即没有 mute,直接返回 false
return false
}

return true
}

// IdentNotExistsMuteStrategy 根据ident是否存在过滤,如果ident不存在,则target_up的告警直接过滤掉
func IdentNotExistsMuteStrategy(rule *models.AlertRule, event *models.AlertCurEvent, targetCache *memsto.TargetCacheType) bool {
ident, has := event.TagsMap["ident"]
Expand Down
47 changes: 0 additions & 47 deletions models/alert_rule.go
Original file line number Diff line number Diff line change
Expand Up @@ -908,50 +908,3 @@ func AlertRuleUpgradeToV6(ctx *ctx.Context, dsm map[string]Datasource) error {
}
return nil
}

// TimeSpanMuteStrategy 根据规则配置的告警生效时间段过滤,如果产生的告警不在规则配置的告警生效时间段内,则不告警,即被mute
// 时间范围,左闭右开,默认范围:00:00-24:00
func (ar *AlertRule) TimeSpanMuteStrategy() bool {
tm := time.Unix(time.Now().Unix(), 0)
triggerTime := tm.Format("15:04")
triggerWeek := strconv.Itoa(int(tm.Weekday()))

enableStime := strings.Fields(ar.EnableStime)
enableEtime := strings.Fields(ar.EnableEtime)
enableDaysOfWeek := strings.Split(ar.EnableDaysOfWeek, ";")
length := len(enableDaysOfWeek)
// enableStime,enableEtime,enableDaysOfWeek三者长度肯定相同,这里循环一个即可
for i := 0; i < length; i++ {
enableDaysOfWeek[i] = strings.Replace(enableDaysOfWeek[i], "7", "0", 1)
if !strings.Contains(enableDaysOfWeek[i], triggerWeek) {
continue
}

if enableStime[i] < enableEtime[i] {
if enableEtime[i] == "23:59" {
// 02:00-23:59,这种情况做个特殊处理,相当于左闭右闭区间了
if triggerTime < enableStime[i] {
// mute, 即没生效
continue
}
} else {
// 02:00-04:00 或者 02:00-24:00
if triggerTime < enableStime[i] || triggerTime >= enableEtime[i] {
// mute, 即没生效
continue
}
}
} else if enableStime[i] > enableEtime[i] {
// 21:00-09:00
if triggerTime < enableStime[i] && triggerTime >= enableEtime[i] {
// mute, 即没生效
continue
}
}

// 到这里说明当前时刻在告警规则的某组生效时间范围内,即没有 mute,直接返回 false
return false
}

return true
}

0 comments on commit 9cdbda0

Please sign in to comment.