Remove notifier module duplication (solana-labs#10051)

Cargo.toml

@@ -31,6 +31,7 @@ members = [
     "measure",
     "metrics",
     "net-shaper",
+    "notifier",
     "programs/bpf_loader",
     "programs/budget",
     "programs/btc_spv",

notifier/.gitignore

@@ -0,0 +1,2 @@
+/target/
+/farf/

notifier/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "solana-notifier"
+version = "1.2.0"
+description = "Solana Notifier"
+authors = ["Solana Maintainers <[email protected]>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde_json = "1.0"
+
+[lib]
+name = "solana_notifier"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

watchtower/src/notifier.rs → notifier/src/lib.rs

@@ -1,7 +1,26 @@
+/// To activate Slack, Discord and/or Telegram notifications, define these environment variables
+/// before using the `Notifier`
+/// ```bash
+/// export SLACK_WEBHOOK=...
+/// export DISCORD_WEBHOOK=...
+/// ```
+///
+/// Telegram requires the following two variables:
+/// ```bash
+/// export TELEGRAM_BOT_TOKEN=...
+/// export TELEGRAM_CHAT_ID=...
+/// ```
+///
+/// To receive a Twilio SMS notification on failure, having a Twilio account,
+/// and a sending number owned by that account,
+/// define environment variable before running `solana-watchtower`:
+/// ```bash
+/// export TWILIO_CONFIG='ACCOUNT=<account>,TOKEN=<securityToken>,TO=<receivingNumber>,FROM=<sendingNumber>'
+/// ```
 use log::*;
-use reqwest::blocking::Client;
+use reqwest::{blocking::Client, StatusCode};
 use serde_json::json;
-use std::env;
+use std::{env, thread::sleep, time::Duration};
 
 struct TelegramWebHook {
     bot_token: String,
@@ -65,11 +84,11 @@ pub struct Notifier {
 }
 
 impl Notifier {
-    pub fn new() -> Self {
-        Self::new_with_env_prefix("")
+    pub fn default() -> Self {
+        Self::new("")
     }
 
-    pub fn new_with_env_prefix(env_prefix: &str) -> Self {
+    pub fn new(env_prefix: &str) -> Self {
         info!("Initializing {}Notifier", env_prefix);
 
         let discord_webhook = env::var(format!("{}DISCORD_WEBHOOK", env_prefix))
@@ -107,9 +126,30 @@ impl Notifier {
 
     pub fn send(&self, msg: &str) {
         if let Some(webhook) = &self.discord_webhook {
-            let data = json!({ "content": msg });
-            if let Err(err) = self.client.post(webhook).json(&data).send() {
-                warn!("Failed to send Discord message: {:?}", err);
+            for line in msg.split('\n') {
+                // Discord rate limiting is aggressive, limit to 1 message a second
+                sleep(Duration::from_millis(1000));
+
+                info!("Sending {}", line);
+                let data = json!({ "content": line });
+
+                loop {
+                    let response = self.client.post(webhook).json(&data).send();
+
+                    if let Err(err) = response {
+                        warn!("Failed to send Discord message: \"{}\": {:?}", line, err);
+                        break;
+                    } else if let Ok(response) = response {
+                        info!("response status: {}", response.status());
+                        if response.status() == StatusCode::TOO_MANY_REQUESTS {
+                            warn!("rate limited!...");
+                            warn!("response text: {:?}", response.text());
+                            sleep(Duration::from_secs(2));
+                        } else {
+                            break;
+                        }
+                    }
+                }
             }
         }
 

ramp-tps/Cargo.toml

@@ -21,6 +21,7 @@ solana-client = { path = "../client", version = "1.2.0" }
 solana-logger = { path = "../logger", version = "1.2.0" }
 solana-metrics = { path = "../metrics", version = "1.2.0" }
 solana-net-utils = { path = "../net-utils", version = "1.2.0" }
+solana-notifier = { path = "../notifier", version = "1.2.0" }
 solana-sdk = { path = "../sdk", version = "1.2.0" }
 solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
 tar = "0.4.26"

ramp-tps/src/main.rs

@@ -1,6 +1,5 @@
 //! Ramp up TPS for Tour de SOL until all validators drop out
 
-mod notifier;
 mod results;
 mod stake;
 mod tps;
@@ -54,7 +53,7 @@ fn gift_for_round(tps_round: u32, initial_balance: u64) -> u64 {
 fn main() {
     solana_logger::setup_with_default("solana=debug");
     solana_metrics::set_panic_hook("ramp-tps");
-    let mut notifier = notifier::Notifier::new();
+    let mut notifier = solana_notifier::Notifier::default();
 
     let matches = App::new(crate_name!())
         .about(crate_description!())
@@ -199,7 +198,7 @@ fn main() {
     let _ = fs::remove_dir_all(&tmp_ledger_path);
     fs::create_dir_all(&tmp_ledger_path).expect("failed to create temp ledger path");
 
-    notifier.notify("Hi!");
+    notifier.send("Hi!");
     datapoint_info!("ramp-tps", ("event", "boot", String),);
 
     let entrypoint_str = matches.value_of("entrypoint").unwrap();
@@ -219,7 +218,7 @@ fn main() {
     debug!("First normal slot: {}", first_normal_slot);
     let sleep_slots = first_normal_slot.saturating_sub(current_slot);
     if sleep_slots > 0 {
-        notifier.notify(&format!(
+        notifier.send(&format!(
             "Waiting for warm-up epochs to complete (epoch {})",
             epoch_schedule.first_normal_epoch
         ));
@@ -291,7 +290,7 @@ fn main() {
     let mut tps_sampler = tps::Sampler::new(&entrypoint_addr);
 
     loop {
-        notifier.notify(&format!("Round {}!", tps_round));
+        notifier.send(&format!("Round {}!", tps_round));
         let tx_count = tx_count_for_round(tps_round, tx_count_baseline, tx_count_increment);
         datapoint_info!(
             "ramp-tps",
@@ -328,7 +327,7 @@ fn main() {
             ("validators", starting_validators.len(), i64)
         );
 
-        notifier.buffer(format!(
+        notifier.send(&format!(
             "There are {} validators present:",
             starting_validators.len()
         ));
@@ -338,11 +337,10 @@ fn main() {
             .map(|node_pubkey| format!("* {}", pubkey_to_keybase(&node_pubkey)))
             .collect();
         validators.sort();
-        notifier.buffer_vec(validators);
-        notifier.flush();
+        notifier.send(&validators.join("\n"));
 
         let client_tx_count = tx_count / NUM_BENCH_CLIENTS as u64;
-        notifier.notify(&format!(
+        notifier.send(&format!(
             "Starting transactions for {} minutes (batch size={})",
             round_minutes, tx_count,
         ));
@@ -393,7 +391,7 @@ fn main() {
             ("round", tps_round, i64),
         );
 
-        notifier.notify("Transactions stopped");
+        notifier.send("Transactions stopped");
         tps_sampler.report_results(&notifier);
 
         let remaining_validators = voters::fetch_active_validators(&rpc_client);

ramp-tps/src/stake.rs

@@ -1,4 +1,4 @@
-use crate::{notifier, utils};
+use crate::utils;
 use log::*;
 use solana_client::{rpc_client::RpcClient, rpc_response::RpcEpochInfo};
 use solana_sdk::{
@@ -64,11 +64,11 @@ pub fn wait_for_warm_up(
     rpc_client: &RpcClient,
     stake_config: &StakeConfig,
     genesis_config: &GenesisConfig,
-    notifier: &notifier::Notifier,
+    notifier: &solana_notifier::Notifier,
 ) {
     // Sleep until activation_epoch has finished
     if epoch_info.epoch <= activation_epoch {
-        notifier.notify(&format!(
+        notifier.send(&format!(
             "Waiting until epoch {} is finished...",
             activation_epoch
         ));
@@ -105,7 +105,7 @@ pub fn wait_for_warm_up(
             let warm_up_epochs = calculate_stake_warmup(stake_entry, stake_config);
             let stake_warmed_up_epoch = latest_epoch + warm_up_epochs;
             if stake_warmed_up_epoch > current_epoch {
-                notifier.notify(&format!(
+                notifier.send(&format!(
                     "Waiting until epoch {} for stake to warmup (current epoch is {})...",
                     stake_warmed_up_epoch, current_epoch
                 ));

ramp-tps/src/tps.rs

@@ -1,7 +1,7 @@
-use crate::notifier::Notifier;
 use log::*;
 use solana_client::perf_utils::{sample_txs, SampleStats};
 use solana_client::thin_client::ThinClient;
+use solana_notifier::Notifier;
 use solana_sdk::timing::duration_as_s;
 use std::{
     net::SocketAddr,
@@ -64,7 +64,7 @@ impl Sampler {
     pub fn report_results(&self, notifier: &Notifier) {
         let SampleStats { tps, elapsed, txs } = self.maxes.read().unwrap()[0].1;
         let avg_tps = txs as f32 / duration_as_s(&elapsed);
-        notifier.notify(&format!(
+        notifier.send(&format!(
             "Highest TPS: {:.0}, Average TPS: {:.0}",
             tps, avg_tps
         ));

ramp-tps/src/utils.rs

@@ -1,8 +1,8 @@
-use crate::notifier::Notifier;
 use bzip2::bufread::BzDecoder;
 use log::*;
 use solana_client::rpc_client::RpcClient;
 use solana_net_utils::parse_host;
+use solana_notifier::Notifier;
 use solana_sdk::{
     clock::{Epoch, Slot},
     genesis_config::GenesisConfig,
@@ -89,8 +89,8 @@ pub fn is_host(string: String) -> Result<(), String> {
     Ok(())
 }
 
-pub fn bail(notifier: &crate::notifier::Notifier, msg: &str) -> ! {
-    notifier.notify(msg);
+pub fn bail(notifier: &Notifier, msg: &str) -> ! {
+    notifier.send(msg);
     sleep(Duration::from_secs(30)); // Wait for notifications to send
     std::process::exit(1);
 }