Merge pull request valeriansaliou#103 from Eijebong/stop-bullying-me

Add a `enable_timer_backoff` config option to avoid spamming reminders

README.md

@@ -195,6 +195,7 @@ Use the sample [config.cfg](https://github.com/valeriansaliou/vigil/blob/master/
 
 * `startup_notification` (type: _boolean_, allowed: `true`, `false`, default: `true`) — Whether to send startup notification or not (stating that systems are `healthy`)
 * `reminder_interval` (type: _integer_, allowed: seconds, no default) — Interval at which downtime reminder notifications should be sent (if any)
+* `enable_timer_backoff` (type: _bool_, allowed: `true`, `false`, default: `false`) — If enabled, the downtime reminder interval will get larger as remainders are sent. The value will be `reminder_interval` * N with N being the number of reminders sent since the service went down.
 
 **[notify.email]**
 

config.cfg

@@ -67,6 +67,7 @@ queue_loaded_retry_delay = 500
 
 startup_notification = true
 reminder_interval = 300
+enable_timer_backoff = false
 
 [notify.email]
 

src/aggregator/manager.rs

@@ -228,6 +228,11 @@ fn scan_and_bump_states() -> Option<BumpedStates> {
     let mut should_notify = (store.states.status != Status::Dead && general_status == Status::Dead)
         || (store.states.status == Status::Dead && general_status != Status::Dead);
 
+    // Reset the backoff counter when we're back to being healthy
+    if store.states.status != Status::Healthy && general_status == Status::Healthy {
+        store.states.backoff_counter = 1;
+    }
+
     // Check if should re-notify? (in case status did not change; only if dead)
     // Notice: this is used to send periodic reminders of downtime (ie. 'still down' messages)
     if has_changed == false && should_notify == false && general_status == Status::Dead {
@@ -240,14 +245,21 @@ fn scan_and_bump_states() -> Option<BumpedStates> {
                         SystemTime::now().duration_since(last_notified)
                     {
                         // Duration since last notified exceeds reminder interval, should re-notify
-                        if duration_since_notified >= Duration::from_secs(reminder_interval) {
+                        // We use backoff_counter all the time because if it's disabled then the
+                        // value is 1 at all time thus not impacting the interval
+                        if duration_since_notified
+                            >= Duration::from_secs(reminder_interval * store.states.backoff_counter.pow(2))
+                        {
                             info!("should re-notify about unchanged status");
 
-                            should_notify = true
+                            should_notify = true;
+                            if notify.enable_timer_backoff {
+                                store.states.backoff_counter += 1;
+                            }
                         } else {
                             debug!(
-                                "should not re-notify about unchanged status (interval: {})",
-                                reminder_interval
+                                "should not re-notify about unchanged status (interval: {}, backoff_counter: {})",
+                                reminder_interval, store.states.backoff_counter
                             );
                         }
                     }

src/config/config.rs

@@ -102,6 +102,9 @@ pub struct ConfigNotify {
     pub startup_notification: bool,
 
     pub reminder_interval: Option<u64>,
+    #[serde(default = "defaults::enable_timer_backoff")]
+    pub enable_timer_backoff: bool,
+
     pub email: Option<ConfigNotifyEmail>,
     pub twilio: Option<ConfigNotifyTwilio>,
     pub slack: Option<ConfigNotifySlack>,

src/config/defaults.rs

@@ -94,3 +94,7 @@ pub fn notify_slack_mention_channel() -> bool {
 pub fn notify_generic_reminders_only() -> bool {
     false
 }
+
+pub fn enable_timer_backoff() -> bool {
+    false
+}

src/prober/manager.rs

@@ -42,6 +42,7 @@ lazy_static! {
             status: Status::Healthy,
             date: None,
             probes: IndexMap::new(),
+            backoff_counter: 1,
         },
         notified: None,
     }));

src/prober/states.rs

@@ -18,6 +18,7 @@ pub struct ServiceStates {
     pub status: Status,
     pub date: Option<String>,
     pub probes: IndexMap<String, ServiceStatesProbe>,
+    pub backoff_counter: u64,
 }
 
 #[derive(Serialize)]