blockstm-only benchmark: measure sequential execution tps and speed up (

aptos-labs#7318)

aptos-move/aptos-transaction-benchmarks/src/main.rs

@@ -13,9 +13,8 @@ fn main() {
     let txns = [1000, 10000];
     let num_warmups = 2;
     let num_runs = 10;
-    let check_correctness = false;
 
-    let mut measurements = Vec::new();
+    let mut measurements: Vec<Vec<(usize, usize)>> = Vec::new();
     let concurrency_level = num_cpus::get();
 
     for block_size in txns {
@@ -26,16 +25,13 @@ fn main() {
                 num_warmups,
                 num_runs,
                 concurrency_level,
-                check_correctness,
             );
             times.sort();
             measurements.push(times);
         }
     }
-    if check_correctness {
-        println!("\nParallel execution output same as sequential!\n");
-    }
-    println!("concurrency_level = {}", concurrency_level);
+
+    println!("\nconcurrency_level = {}\n", concurrency_level);
 
     let mut i = 0;
     for block_size in txns {
@@ -44,12 +40,22 @@ fn main() {
                 "PARAMS: num_account = {}, block_size = {}",
                 num_accounts, block_size
             );
-            println!("TPS: {:?}", measurements[i]);
-            let mut sum = 0;
+            println!("Parallel/Sequential TPS: {:?}", measurements[i]);
+
+            let mut par_sum = 0;
             for m in &measurements[i] {
-                sum += m;
+                par_sum += m.0;
             }
-            println!("AVG TPS = {:?}", sum / measurements[i].len());
+            let mut seq_sum = 0;
+            for m in &measurements[i] {
+                seq_sum += m.1;
+            }
+            println!(
+                "Avg Parallel TPS = {:?}, Avg Sequential TPS = {:?}, speed up {}x",
+                par_sum / measurements[i].len(),
+                seq_sum / measurements[i].len(),
+                par_sum / seq_sum
+            );
             i += 1;
         }
         println!();

aptos-move/aptos-transaction-benchmarks/src/transactions.rs

@@ -101,8 +101,7 @@ where
         num_warmups: usize,
         num_runs: usize,
         concurrency_level: usize,
-        check_correctness: bool,
-    ) -> Vec<usize> {
+    ) -> Vec<(usize, usize)> {
         let mut ret = Vec::new();
 
         let total_runs = num_warmups + num_runs;
@@ -111,7 +110,7 @@ where
 
             if i < num_warmups {
                 println!("WARMUP - ignore results");
-                state.execute_blockstm_benchmark(concurrency_level, check_correctness);
+                state.execute_blockstm_benchmark(concurrency_level);
             } else {
                 println!(
                     "RUN BlockSTM-only benchmark for: num_threads = {}, \
@@ -121,7 +120,7 @@ where
                     num_accounts,
                     num_txn,
                 );
-                ret.push(state.execute_blockstm_benchmark(concurrency_level, check_correctness));
+                ret.push(state.execute_blockstm_benchmark(concurrency_level));
             }
         }
 
@@ -237,16 +236,11 @@ impl TransactionBenchState {
         .expect("VM should not fail to start");
     }
 
-    fn execute_blockstm_benchmark(
-        self,
-        concurrency_level: usize,
-        check_correctness: bool,
-    ) -> usize {
+    fn execute_blockstm_benchmark(self, concurrency_level: usize) -> (usize, usize) {
         BlockAptosVM::execute_block_benchmark(
             self.transactions,
             self.executor.get_state_view(),
             concurrency_level,
-            check_correctness,
         )
     }
 }

aptos-move/aptos-vm/src/block_executor/mod.rs

@@ -145,8 +145,7 @@ impl BlockAptosVM {
         transactions: Vec<Transaction>,
         state_view: &S,
         concurrency_level: usize,
-        check_correctness: bool,
-    ) -> usize {
+    ) -> (usize, usize) {
         // Verify the signatures of all the transactions in parallel.
         // This is time consuming so don't wait and do the checking
         // sequentially while executing the transactions.
@@ -175,27 +174,38 @@ impl BlockAptosVM {
         let executor = BlockExecutor::<PreprocessedTransaction, AptosExecutorTask<S>, S>::new(
             concurrency_level,
         );
-
+        println!("Parallel execution starts...");
         let timer = Instant::now();
         let ret = executor.execute_block(state_view, signature_verified_block, state_view);
         let exec_t = timer.elapsed();
+        println!(
+            "Parallel execution finishes, TPS = {}",
+            block_size * 1000 / exec_t.as_millis() as usize
+        );
 
         flush_speculative_logs();
 
-        if check_correctness {
-            // sequentially execute the block and check if the results match
-            let seq_executor =
-                BlockExecutor::<PreprocessedTransaction, AptosExecutorTask<S>, S>::new(1);
-            let seq_ret = seq_executor.execute_block(
-                state_view,
-                signature_verified_block_for_seq,
-                state_view,
-            );
-            assert_eq!(ret, seq_ret);
-            drop(seq_ret);
-        }
+        // sequentially execute the block and check if the results match
+        let seq_executor =
+            BlockExecutor::<PreprocessedTransaction, AptosExecutorTask<S>, S>::new(1);
+        println!("Sequential execution starts...");
+        let seq_timer = Instant::now();
+        let seq_ret =
+            seq_executor.execute_block(state_view, signature_verified_block_for_seq, state_view);
+        let seq_exec_t = seq_timer.elapsed();
+        println!(
+            "Sequential execution finishes, TPS = {}",
+            block_size * 1000 / seq_exec_t.as_millis() as usize
+        );
+
+        assert_eq!(ret, seq_ret);
+
         drop(ret);
+        drop(seq_ret);
 
-        block_size * 1000 / exec_t.as_millis() as usize
+        (
+            block_size * 1000 / exec_t.as_millis() as usize,
+            block_size * 1000 / seq_exec_t.as_millis() as usize,
+        )
     }
 }