tests: make cannons configurable + cleanup + better docs for e2e tooling

node/narwhal/tests/bft_e2e.rs

@@ -24,11 +24,13 @@ use tokio::time::sleep;
 #[ignore = "long-running e2e test"]
 async fn test_state_coherence() {
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
+
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: true,
         connect_all: true,
-        fire_cannons: true,
+        fire_cannons: Some(CANNON_INTERVAL_MS),
         // Set this to Some(0..=4) to see the logs.
         log_level: Some(0),
         log_connections: true,
@@ -43,11 +45,13 @@ async fn test_state_coherence() {
 async fn test_quorum_threshold() {
     // Start N nodes but don't connect them.
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
+
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: true,
         connect_all: false,
-        fire_cannons: false,
+        fire_cannons: None,
         // Set this to Some(0..=4) to see the logs.
         log_level: None,
         log_connections: true,
@@ -60,7 +64,7 @@ async fn test_quorum_threshold() {
     }
 
     // Start the cannons for node 0.
-    network.fire_cannons_at(0);
+    network.fire_cannons_at(0, CANNON_INTERVAL_MS);
 
     sleep(Duration::from_millis(MAX_BATCH_DELAY * 2)).await;
 
@@ -71,7 +75,7 @@ async fn test_quorum_threshold() {
 
     // Connect the first two nodes and start the cannons for node 1.
     network.connect_validators(0, 1).await;
-    network.fire_cannons_at(1);
+    network.fire_cannons_at(1, CANNON_INTERVAL_MS);
 
     sleep(Duration::from_millis(MAX_BATCH_DELAY * 2)).await;
 
@@ -83,7 +87,7 @@ async fn test_quorum_threshold() {
     // Connect the third node and start the cannons for it.
     network.connect_validators(0, 2).await;
     network.connect_validators(1, 2).await;
-    network.fire_cannons_at(2);
+    network.fire_cannons_at(2, CANNON_INTERVAL_MS);
 
     // Check the nodes reach quorum and advance through the rounds.
     const TARGET_ROUND: u64 = 4;
@@ -94,11 +98,12 @@ async fn test_quorum_threshold() {
 async fn test_quorum_break() {
     // Start N nodes, connect them and start the cannons for each.
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: true,
         connect_all: true,
-        fire_cannons: true,
+        fire_cannons: Some(CANNON_INTERVAL_MS),
         // Set this to Some(0..=4) to see the logs.
         log_level: None,
         log_connections: true,

node/narwhal/tests/common/primary.rs

@@ -36,35 +36,53 @@ use parking_lot::Mutex;
 use tokio::{task::JoinHandle, time::sleep};
 use tracing::*;
 
+/// The configuration for the test network.
 #[derive(Clone, Copy, Debug)]
 pub struct TestNetworkConfig {
+    /// The number of nodes to spin up.
     pub num_nodes: u16,
+    /// If this is set to `true`, the BFT protocol is started on top of Narwhal.
     pub bft: bool,
+    /// If this is set to `true`, all nodes are connected to each other (when they're first
+    /// started).
     pub connect_all: bool,
-    pub fire_cannons: bool,
+    /// If `Some(i)` is set, the cannons will fire every `i` milliseconds.
+    pub fire_cannons: Option<u64>,
+    /// The log level to use for the test.
     pub log_level: Option<u8>,
+    /// If this is set to `true`, the number of connections is logged every 5 seconds.
     pub log_connections: bool,
 }
 
+/// A test network.
 #[derive(Clone)]
 pub struct TestNetwork {
+    /// The configuration for the test network.
     pub config: TestNetworkConfig,
+    /// A map of node IDs to validators in the network.
     pub validators: HashMap<u16, TestValidator>,
 }
 
+/// A test validator.
 #[derive(Clone)]
 pub struct TestValidator {
+    /// The ID of the validator.
     pub id: u16,
+    /// The primary instance. When the BFT is enabled this is a clone of the BFT primary.
     pub primary: Primary<CurrentNetwork>,
+    /// The channel sender of the primary.
     pub primary_sender: Option<PrimarySender<CurrentNetwork>>,
+    /// The BFT instance. This is only set if the BFT is enabled.
     pub bft: Option<BFT<CurrentNetwork>>,
+    /// The tokio handles of all long-running tasks associated with the validator (incl. cannons).
     pub handles: Arc<Mutex<Vec<JoinHandle<()>>>>,
 }
 
 impl TestValidator {
-    pub fn fire_cannons(&mut self) {
-        let solution_handle = fire_unconfirmed_solutions(self.primary_sender.as_mut().unwrap(), self.id);
-        let transaction_handle = fire_unconfirmed_transactions(self.primary_sender.as_mut().unwrap(), self.id);
+    pub fn fire_cannons(&mut self, interval_ms: u64) {
+        let solution_handle = fire_unconfirmed_solutions(self.primary_sender.as_mut().unwrap(), self.id, interval_ms);
+        let transaction_handle =
+            fire_unconfirmed_transactions(self.primary_sender.as_mut().unwrap(), self.id, interval_ms);
 
         self.handles.lock().push(solution_handle);
         self.handles.lock().push(transaction_handle);
@@ -119,18 +137,17 @@ impl TestNetwork {
     pub async fn start(&mut self) {
         for validator in self.validators.values_mut() {
             let (primary_sender, primary_receiver) = init_primary_channels();
+            validator.primary_sender = Some(primary_sender.clone());
             if let Some(bft) = &mut validator.bft {
                 // Setup the channels and start the bft.
-                validator.primary_sender = Some(primary_sender.clone());
                 bft.run(primary_sender, primary_receiver, None).await.unwrap();
             } else {
                 // Setup the channels and start the primary.
-                validator.primary_sender = Some(primary_sender.clone());
                 validator.primary.run(primary_sender, primary_receiver, None).await.unwrap();
             }
 
-            if self.config.fire_cannons {
-                validator.fire_cannons();
+            if let Some(interval_ms) = self.config.fire_cannons {
+                validator.fire_cannons(interval_ms);
             }
 
             if self.config.log_connections {
@@ -144,8 +161,8 @@ impl TestNetwork {
     }
 
     // Starts the solution and trasnaction cannons for node.
-    pub fn fire_cannons_at(&mut self, id: u16) {
-        self.validators.get_mut(&id).unwrap().fire_cannons();
+    pub fn fire_cannons_at(&mut self, id: u16, interval_ms: u64) {
+        self.validators.get_mut(&id).unwrap().fire_cannons(interval_ms);
     }
 
     // Connects a node to another node.

node/narwhal/tests/common/utils.rs

@@ -71,7 +71,11 @@ pub fn initialize_logger(verbosity: u8) {
 }
 
 /// Fires *fake* unconfirmed solutions at the node.
-pub fn fire_unconfirmed_solutions(sender: &PrimarySender<CurrentNetwork>, node_id: u16) -> JoinHandle<()> {
+pub fn fire_unconfirmed_solutions(
+    sender: &PrimarySender<CurrentNetwork>,
+    node_id: u16,
+    interval_ms: u64,
+) -> JoinHandle<()> {
     let tx_unconfirmed_solution = sender.tx_unconfirmed_solution.clone();
     tokio::task::spawn(async move {
         // This RNG samples the *same* fake solutions for all nodes.
@@ -111,13 +115,17 @@ pub fn fire_unconfirmed_solutions(sender: &PrimarySender<CurrentNetwork>, node_i
             // Increment the counter.
             counter += 1;
             // Sleep briefly.
-            sleep(Duration::from_millis(10)).await;
+            sleep(Duration::from_millis(interval_ms)).await;
         }
     })
 }
 
 /// Fires *fake* unconfirmed transactions at the node.
-pub fn fire_unconfirmed_transactions(sender: &PrimarySender<CurrentNetwork>, node_id: u16) -> JoinHandle<()> {
+pub fn fire_unconfirmed_transactions(
+    sender: &PrimarySender<CurrentNetwork>,
+    node_id: u16,
+    interval_ms: u64,
+) -> JoinHandle<()> {
     let tx_unconfirmed_transaction = sender.tx_unconfirmed_transaction.clone();
     tokio::task::spawn(async move {
         // This RNG samples the *same* fake transactions for all nodes.
@@ -155,7 +163,7 @@ pub fn fire_unconfirmed_transactions(sender: &PrimarySender<CurrentNetwork>, nod
             // Increment the counter.
             counter += 1;
             // Sleep briefly.
-            sleep(Duration::from_millis(10)).await;
+            sleep(Duration::from_millis(interval_ms)).await;
         }
     })
 }

node/narwhal/tests/narwhal_e2e.rs

@@ -26,11 +26,13 @@ use tokio::time::sleep;
 #[ignore = "long-running e2e test"]
 async fn test_state_coherence() {
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
+
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: false,
         connect_all: true,
-        fire_cannons: true,
+        fire_cannons: Some(CANNON_INTERVAL_MS),
         // Set this to Some(0..=4) to see the logs.
         log_level: Some(0),
         log_connections: true,
@@ -48,11 +50,13 @@ async fn test_state_coherence() {
 async fn test_quorum_threshold() {
     // Start N nodes but don't connect them.
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
+
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: false,
         connect_all: false,
-        fire_cannons: false,
+        fire_cannons: None,
         // Set this to Some(0..=4) to see the logs.
         log_level: None,
         log_connections: true,
@@ -65,7 +69,7 @@ async fn test_quorum_threshold() {
     }
 
     // Start the cannons for node 0.
-    network.fire_cannons_at(0);
+    network.fire_cannons_at(0, CANNON_INTERVAL_MS);
 
     sleep(Duration::from_millis(MAX_BATCH_DELAY * 2)).await;
 
@@ -76,7 +80,7 @@ async fn test_quorum_threshold() {
 
     // Connect the first two nodes and start the cannons for node 1.
     network.connect_validators(0, 1).await;
-    network.fire_cannons_at(1);
+    network.fire_cannons_at(1, CANNON_INTERVAL_MS);
 
     sleep(Duration::from_millis(MAX_BATCH_DELAY * 2)).await;
 
@@ -88,7 +92,7 @@ async fn test_quorum_threshold() {
     // Connect the third node and start the cannons for it.
     network.connect_validators(0, 2).await;
     network.connect_validators(1, 2).await;
-    network.fire_cannons_at(2);
+    network.fire_cannons_at(2, CANNON_INTERVAL_MS);
 
     // Check the nodes reach quorum and advance through the rounds.
     const TARGET_ROUND: u64 = 4;
@@ -99,11 +103,12 @@ async fn test_quorum_threshold() {
 async fn test_quorum_break() {
     // Start N nodes, connect them and start the cannons for each.
     const N: u16 = 4;
+    const CANNON_INTERVAL_MS: u64 = 10;
     let mut network = TestNetwork::new(TestNetworkConfig {
         num_nodes: N,
         bft: false,
         connect_all: true,
-        fire_cannons: true,
+        fire_cannons: Some(CANNON_INTERVAL_MS),
         // Set this to Some(0..=4) to see the logs.
         log_level: None,
         log_connections: true,