[forge] Catchup as success criteria (aptos-labs#3010)

testsuite/forge-cli/src/main.rs

@@ -58,6 +58,8 @@ pub struct SuccessCriteriaArgs {
     avg_tps: usize,
     #[structopt(long, default_value = "10000")]
     max_latency_ms: usize,
+    #[structopt(long)]
+    wait_for_all_nodes_to_catchup_secs: Option<u64>,
 }
 
 #[derive(StructOpt, Debug)]
@@ -195,6 +197,9 @@ fn main() -> Result<()> {
     let success_criteria = SuccessCriteria::new(
         args.success_criteria.avg_tps,
         args.success_criteria.max_latency_ms,
+        args.success_criteria
+            .wait_for_all_nodes_to_catchup_secs
+            .map(Duration::from_secs),
     );
 
     let runtime = Runtime::new()?;

testsuite/forge/src/interface/network.rs

@@ -1,10 +1,12 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
+use std::time::Duration;
+
 use super::Test;
 use crate::success_criteria::SuccessCriteria;
 use crate::{CoreContext, Result, Swarm, TestReport};
-use transaction_emitter_lib::EmitJobRequest;
+use transaction_emitter_lib::{EmitJobRequest, TxnStats};
 
 /// The testing interface which defines a test written with full control over an existing network.
 /// Tests written against this interface will have access to both the Root account as well as the
@@ -46,7 +48,8 @@ impl<'t> NetworkContext<'t> {
     pub fn core(&mut self) -> &mut CoreContext {
         &mut self.core
     }
-    pub fn success_criteria(&self) -> &SuccessCriteria {
-        &self.success_criteria
+    pub fn check_for_success(&self, stats: &TxnStats, window: &Duration) -> Result<()> {
+        self.success_criteria
+            .check_for_success(stats, window, self.swarm)
     }
 }

testsuite/forge/src/success_criteria.rs

@@ -3,24 +3,37 @@
 
 use anyhow::bail;
 use serde::Serialize;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use transaction_emitter_lib::emitter::stats::TxnStats;
 
+use crate::{Swarm, SwarmExt};
+
 #[derive(Default, Clone, Debug, Serialize)]
 pub struct SuccessCriteria {
     avg_tps: usize,
     max_latency_ms: usize,
+    wait_for_all_nodes_to_catchup: Option<Duration>,
 }
 
 impl SuccessCriteria {
-    pub fn new(tps: usize, max_latency_ms: usize) -> Self {
+    pub fn new(
+        tps: usize,
+        max_latency_ms: usize,
+        wait_for_all_nodes_to_catchup: Option<Duration>,
+    ) -> Self {
         Self {
             avg_tps: tps,
             max_latency_ms,
+            wait_for_all_nodes_to_catchup,
         }
     }
 
-    pub fn check_for_success(&self, stats: &TxnStats, window: &Duration) -> anyhow::Result<()> {
+    pub fn check_for_success(
+        &self,
+        stats: &TxnStats,
+        window: &Duration,
+        swarm: &dyn Swarm,
+    ) -> anyhow::Result<()> {
         // TODO: Add more success criteria like expired transactions, CPU, memory usage etc
         let avg_tps = stats.committed / window.as_secs();
         let is_triggerd_by_github_actions =
@@ -36,6 +49,15 @@ impl SuccessCriteria {
             }
             bail!(error_message)
         }
+
+        if let Some(duration) = self.wait_for_all_nodes_to_catchup {
+            futures::executor::block_on(async {
+                swarm
+                    .wait_for_all_nodes_to_catchup(Instant::now() + duration)
+                    .await
+            })?;
+        }
+
         // TODO(skedia) Add latency success criteria after we have support for querying prometheus
         // latency
         Ok(())

testsuite/testcases/src/network_bandwidth_test.rs

@@ -50,8 +50,7 @@ impl NetworkTest for NetworkBandwidthTest {
         ctx.swarm().remove_chaos(bandwidth)?;
 
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         Ok(())
     }

testsuite/testcases/src/network_latency_test.rs

@@ -46,8 +46,7 @@ impl NetworkTest for NetworkLatencyTest {
         ctx.swarm().remove_chaos(delay)?;
 
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         Ok(())
     }

testsuite/testcases/src/network_loss_test.rs

@@ -46,8 +46,7 @@ impl NetworkTest for NetworkLossTest {
         ctx.swarm().remove_chaos(loss)?;
 
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         Ok(())
     }

testsuite/testcases/src/network_partition_test.rs

@@ -44,8 +44,7 @@ impl NetworkTest for NetworkPartitionTest {
         ctx.swarm().remove_chaos(partition)?;
 
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         Ok(())
     }

testsuite/testcases/src/performance_test.rs

@@ -35,8 +35,7 @@ impl NetworkTest for PerformanceBenchmark {
         ctx.report
             .report_txn_stats(self.name().to_string(), &txn_stat, duration);
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         let runtime = Runtime::new().unwrap();
 

testsuite/testcases/src/performance_with_fullnode_test.rs

@@ -27,8 +27,7 @@ impl NetworkTest for PerformanceBenchmarkWithFN {
         ctx.report
             .report_txn_stats(self.name().to_string(), &txn_stat, duration);
         // ensure we meet the success criteria
-        ctx.success_criteria()
-            .check_for_success(&txn_stat, &duration)?;
+        ctx.check_for_success(&txn_stat, &duration)?;
 
         Ok(())
     }