[benchmarking] Refactor Benchmark (MystenLabs#1533)

* refactor * clean up * refactor rebase * rebase * rebase * add how-to * format * lint * don't pop key, copy it * Add a safe client function that augments the follower API with downloading the transactions information (MystenLabs#1446) Co-authored-by: George Danezis <[email protected]> * Removing dev-addresses from Move.toml (MystenLabs#1536) There is no undefined address in the [addresses] section (e.g. AddressToBeFilledIn = "_") so having a [dev-addresses] section is not necessary. * [move][adapter] Static init rules (MystenLabs#1532) - Made init rules static - The existence of the function is optional - The function must be named 'init' - The function must be private - The function can have a single parameter, &mut TxContext - Alternatively, the function can have zero parameters * adds a counter example (MystenLabs#1539) * sui: reorganize binaries * wallet: log git revision on start * Refactor Build index by common user workflow (MystenLabs#1530) * Refactor Build index by common user workflow * Update index.md Fix paths missing Markdown file extension * Update index.md (MystenLabs#1545) Removing tutorial series until ready * Update index.md Remove version from API link Capitalize REST * comments and add unwind logic for process * fix: remove one unneeded instance of key_pair.copy() * fix: remove three unneeded instances of key_pair.copy() * fix: remove one unneeded instance of key_pair.copy() * refactor rebase * rebase * refactor * rebase * rebase * add how-to * lint * don't pop key, copy it * rebase * Update index.md Remove version from API link Capitalize REST * rebase Co-authored-by: Lu Zhang <[email protected]> Co-authored-by: George Danezis <[email protected]> Co-authored-by: George Danezis <[email protected]> Co-authored-by: jaredcosulich <[email protected]> Co-authored-by: Todd Nowacki <[email protected]> Co-authored-by: Damir S <[email protected]> Co-authored-by: Brandon Williams <[email protected]> Co-authored-by: Clay-Mysten <[email protected]> Co-authored-by: François Garillot <[email protected]>
olegnodata · Apr 22, 2022 · 64357d7 · 64357d7
1 parent 825ad99
commit 64357d7
Show file tree

Hide file tree

Showing 12 changed files with 633 additions and 186 deletions.
diff --git a/doc/src/learn/index.md b/doc/src/learn/index.md
@@ -50,4 +50,4 @@ Take note of these related repositories of information to make best use of the k
   * [Book](https://github.com/diem/move/blob/main/language/documentation/book/src/introduction.md) - A summary with pages on [various topics](https://github.com/diem/move/tree/main/language/documentation/book/src).
   * [Examples](https://github.com/diem/move/tree/main/language/documentation/examples/experimental) - A set of samples, such as for [defining a coin](https://github.com/diem/move/tree/main/language/documentation/examples/experimental/basic-coin) and [swapping it](https://github.com/diem/move/tree/main/language/documentation/examples/experimental/coin-swap).
 * [Awesome Move](https://github.com/MystenLabs/awesome-move/blob/main/README.md) - A summary of resources related to Move, from blockchains through code samples.
-* [Sui API Reference](https://app.swaggerhub.com/apis/MystenLabs/sui-api/0.1 ) - The reference files for the Sui Rest API.
+* [Sui API Reference](https://app.swaggerhub.com/apis/MystenLabs/sui-api/) - The reference files for the Sui REST API.
diff --git a/network_utils/src/network.rs b/network_utils/src/network.rs
@@ -85,6 +85,14 @@ impl NetworkClient {
         }
     }
 
+    pub fn base_address(&self) -> &str {
+        &self.base_address
+    }
+
+    pub fn base_port(&self) -> u16 {
+        self.base_port
+    }
+
     async fn batch_send_one_chunk(
         &self,
         requests: Vec<Bytes>,

diff --git a/sui/Cargo.toml b/sui/Cargo.toml
@@ -73,3 +73,6 @@ jsonrpsee-proc-macros = "0.10.1"
 
 [dev-dependencies]
 tracing-test = "0.2.1"
+
+[features]
+benchmark = ["narwhal-node/benchmark"]
diff --git a/sui/src/benchmark.rs b/sui/src/benchmark.rs
@@ -5,6 +5,7 @@
 
 use futures::{join, StreamExt};
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use std::panic;
 use std::thread;
 use std::{thread::sleep, time::Duration};
 use sui_core::authority_client::{AuthorityAPI, NetworkAuthorityClient};
@@ -14,23 +15,24 @@ use sui_types::messages::{BatchInfoRequest, BatchInfoResponseItem};
 use sui_types::serialize::*;
 use tokio::runtime::{Builder, Runtime};
 use tracing::{error, info};
-
 pub mod bench_types;
 pub mod load_generator;
 pub mod transaction_creator;
+pub mod validator_preparer;
 use crate::benchmark::bench_types::{Benchmark, BenchmarkType};
 use crate::benchmark::load_generator::{
-    calculate_throughput, check_transaction_response, send_tx_chunks, spawn_authority_server,
-    FixedRateLoadGenerator,
+    calculate_throughput, check_transaction_response, send_tx_chunks, FixedRateLoadGenerator,
 };
 use crate::benchmark::transaction_creator::TransactionCreator;
+use crate::benchmark::validator_preparer::ValidatorPreparer;
 
 use self::bench_types::{BenchmarkResult, MicroBenchmarkResult, MicroBenchmarkType};
 
 const FOLLOWER_BATCH_SIZE: u64 = 10_000;
 
 pub fn run_benchmark(benchmark: Benchmark) -> BenchmarkResult {
-    // Only microbenchmark support is supported
+    // Only microbenchmark is supported
+    info!("benchmark : {:?}", benchmark);
     BenchmarkResult::MicroBenchmark(run_microbenchmark(benchmark))
 }
 
@@ -52,7 +54,14 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
     } else {
         num_cpus::get()
     };
-
+    let validator_preparer = ValidatorPreparer::new(
+        benchmark.running_mode,
+        benchmark.working_dir,
+        benchmark.committee_size,
+        network_client.base_address(),
+        network_client.base_port(),
+        benchmark.db_cpus,
+    );
     match type_ {
         MicroBenchmarkType::Throughput { num_transactions } => run_throughout_microbench(
             network_client,
@@ -61,8 +70,7 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
             benchmark.batch_size,
             benchmark.use_move,
             num_transactions,
-            benchmark.committee_size,
-            benchmark.db_cpus,
+            validator_preparer,
         ),
         MicroBenchmarkType::Latency {
             num_chunks,
@@ -73,11 +81,10 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
             network_server,
             connections,
             benchmark.use_move,
-            benchmark.committee_size,
-            benchmark.db_cpus,
             num_chunks,
             chunk_size,
             period_us,
+            validator_preparer,
         ),
     }
 }
@@ -89,8 +96,7 @@ fn run_throughout_microbench(
     batch_size: usize,
     use_move: bool,
     num_transactions: usize,
-    committee_size: usize,
-    db_cpus: usize,
+    mut validator_preparer: ValidatorPreparer,
 ) -> MicroBenchmarkResult {
     assert_eq!(
         num_transactions % batch_size,
@@ -100,66 +106,65 @@ fn run_throughout_microbench(
     // In order to simplify things, we send chunks on each connection and try to ensure all connections have equal load
     assert!(
         (num_transactions % connections) == 0,
-        "num_transactions must {} be multiple of number of TCP connections {}",
+        "num_transactions must be a multiple of number of TCP connections {}, got {}",
+        connections,
         num_transactions,
-        connections
     );
-    let mut tx_cr = TransactionCreator::new(committee_size, db_cpus);
+    let mut tx_cr = TransactionCreator::new();
 
     let chunk_size = batch_size * connections;
     let txes = tx_cr.generate_transactions(
         connections,
         use_move,
         batch_size * connections,
         num_transactions / chunk_size,
+        &mut validator_preparer,
     );
 
-    // Make multi-threaded runtime for the authority
-    thread::spawn(move || {
-        get_multithread_runtime().block_on(async move {
-            let server = spawn_authority_server(network_server, tx_cr.authority_state).await;
-            if let Err(e) = server.join().await {
-                error!("Server ended with an error: {e}");
-            }
+    validator_preparer.deploy_validator(network_server);
+
+    let result = panic::catch_unwind(|| {
+        // Follower to observe batches
+        let follower_network_client = network_client.clone();
+        thread::spawn(move || {
+            get_multithread_runtime()
+                .block_on(async move { run_follower(follower_network_client).await });
         });
-    });
 
-    // Wait for server start
-    sleep(Duration::from_secs(3));
+        sleep(Duration::from_secs(3));
 
-    // Follower to observe batches
-    let follower_network_client = network_client.clone();
-    thread::spawn(move || {
-        get_multithread_runtime()
-            .block_on(async move { run_follower(follower_network_client).await });
-    });
+        // Run load
+        let (elapsed, resp) = get_multithread_runtime()
+            .block_on(async move { send_tx_chunks(txes, network_client, connections).await });
 
-    sleep(Duration::from_secs(3));
+        let _: Vec<_> = resp
+            .par_iter()
+            .map(|q| check_transaction_response(deserialize_message(&(q.as_ref().unwrap())[..])))
+            .collect();
 
-    // Run load
-    let (elapsed, resp) = get_multithread_runtime()
-        .block_on(async move { send_tx_chunks(txes, network_client, connections).await });
+        elapsed
+    });
+    validator_preparer.clean_up();
 
-    let _: Vec<_> = resp
-        .par_iter()
-        .map(|q| check_transaction_response(deserialize_message(&(q.as_ref().unwrap())[..])))
-        .collect();
-    MicroBenchmarkResult::Throughput {
-        chunk_throughput: calculate_throughput(num_transactions, elapsed),
+    match result {
+        Ok(elapsed) => MicroBenchmarkResult::Throughput {
+            chunk_throughput: calculate_throughput(num_transactions, elapsed),
+        },
+        Err(err) => {
+            panic::resume_unwind(err);
+        }
     }
 }
 
 fn run_latency_microbench(
     network_client: NetworkClient,
-    network_server: NetworkServer,
+    _network_server: NetworkServer,
     connections: usize,
     use_move: bool,
-    committee_size: usize,
-    db_cpus: usize,
-
     num_chunks: usize,
     chunk_size: usize,
     period_us: u64,
+    mut validator_preparer: ValidatorPreparer,
 ) -> MicroBenchmarkResult {
     // In order to simplify things, we send chunks on each connection and try to ensure all connections have equal load
     assert!(
@@ -168,49 +173,57 @@ fn run_latency_microbench(
         num_chunks * chunk_size,
         connections
     );
-    let mut tx_cr = TransactionCreator::new(committee_size, db_cpus);
+
+    let mut tx_cr = TransactionCreator::new();
 
     // These TXes are to load the network
-    let load_gen_txes = tx_cr.generate_transactions(connections, use_move, chunk_size, num_chunks);
+    let load_gen_txes = tx_cr.generate_transactions(
+        connections,
+        use_move,
+        chunk_size,
+        num_chunks,
+        &mut validator_preparer,
+    );
 
     // These are tracer TXes used for measuring latency
-    let tracer_txes = tx_cr.generate_transactions(1, use_move, 1, num_chunks);
+    let tracer_txes =
+        tx_cr.generate_transactions(1, use_move, 1, num_chunks, &mut validator_preparer);
 
-    // Make multi-threaded runtime for the authority
-    thread::spawn(move || {
-        get_multithread_runtime().block_on(async move {
-            let server = spawn_authority_server(network_server, tx_cr.authority_state).await;
-            if let Err(e) = server.join().await {
-                error!("Server ended with an error: {e}");
-            }
+    validator_preparer.deploy_validator(_network_server);
+
+    let result = panic::catch_unwind(|| {
+        let runtime = get_multithread_runtime();
+        // Prep the generators
+        let (mut load_gen, mut tracer_gen) = runtime.block_on(async move {
+            join!(
+                FixedRateLoadGenerator::new(
+                    load_gen_txes,
+                    period_us,
+                    network_client.clone(),
+                    connections,
+                ),
+                FixedRateLoadGenerator::new(tracer_txes, period_us, network_client, 1),
+            )
         });
-    });
 
-    // Wait for server start
-    sleep(Duration::from_secs(3));
-    let runtime = get_multithread_runtime();
-    // Prep the generators
-    let (mut load_gen, mut tracer_gen) = runtime.block_on(async move {
-        join!(
-            FixedRateLoadGenerator::new(
-                load_gen_txes,
-                period_us,
-                network_client.clone(),
-                connections,
-            ),
-            FixedRateLoadGenerator::new(tracer_txes, period_us, network_client, 1),
-        )
-    });
+        // Run the load gen and tracers
+        let (load_latencies, tracer_latencies) =
+            runtime.block_on(async move { join!(load_gen.start(), tracer_gen.start()) });
 
-    // Run the load gen and tracers
-    let (load_latencies, tracer_latencies) =
-        runtime.block_on(async move { join!(load_gen.start(), tracer_gen.start()) });
+        (load_latencies, tracer_latencies)
+    });
+    validator_preparer.clean_up();
 
-    MicroBenchmarkResult::Latency {
-        load_chunk_size: chunk_size,
-        load_latencies,
-        tick_period_us: period_us as usize,
-        chunk_latencies: tracer_latencies,
+    match result {
+        Ok((load_latencies, tracer_latencies)) => MicroBenchmarkResult::Latency {
+            load_chunk_size: chunk_size,
+            load_latencies,
+            tick_period_us: period_us as usize,
+            chunk_latencies: tracer_latencies,
+        },
+        Err(err) => {
+            panic::resume_unwind(err);
+        }
     }
 }
 

diff --git a/sui/src/benchmark/bench_types.rs b/sui/src/benchmark/bench_types.rs
@@ -1,21 +1,21 @@
 // Copyright (c) 2022, Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
+use super::load_generator::calculate_throughput;
 use clap::*;
 use std::default::Default;
+use std::path::PathBuf;
 use strum_macros::EnumString;
 use sui_network::transport;
 
-use super::load_generator::calculate_throughput;
-
 #[derive(Debug, Clone, Parser)]
 #[clap(
     name = "Sui Benchmark",
     about = "Local test and benchmark of the Sui authorities"
 )]
 pub struct Benchmark {
-    /// Size of the Sui committee. Minimum size is 4 to tolerate one fault
-    #[clap(long, default_value = "10", global = true)]
+    /// Size of the Sui committee.
+    #[clap(long, default_value = "1", global = true)]
     pub committee_size: usize,
     /// Timeout for sending queries (us)
     #[clap(long, default_value = "40000000", global = true)]
@@ -38,6 +38,17 @@ pub struct Benchmark {
     #[clap(long, default_value = "2000", global = true)]
     pub batch_size: usize,
 
+    #[clap(
+        arg_enum,
+        default_value = "local-single-validator-thread",
+        global = true,
+        ignore_case = true
+    )]
+    pub running_mode: RunningMode,
+
+    #[clap(long, global = true)]
+    pub working_dir: Option<PathBuf>,
+
     /// Type of benchmark to run
     #[clap(subcommand)]
     pub bench_type: BenchmarkType,
@@ -60,6 +71,13 @@ pub enum BenchmarkType {
     // ... more benchmark types here
 }
 
+#[derive(Debug, Parser, Clone, Copy, ArgEnum, EnumString)]
+#[clap(rename_all = "kebab-case")]
+pub enum RunningMode {
+    LocalSingleValidatorThread,
+    LocalSingleValidatorProcess,
+}
+
 #[derive(Debug, Clone, Parser, Eq, PartialEq, EnumString)]
 #[clap(rename_all = "kebab-case")]
 pub enum MicroBenchmarkType {