Distributed bench (MystenLabs#1661)

* Remote benchmarking

faucet/src/test_utils.rs

@@ -85,7 +85,7 @@ pub async fn start_test_network(
         .collect();
     genesis_config.authorities = authorities;
 
-    let (network_config, accounts, mut keystore) = genesis(genesis_config).await?;
+    let (network_config, accounts, mut keystore) = genesis(genesis_config, None).await?;
     let key_pair_refs = key_pairs.iter().collect::<Vec<_>>();
     let network = SuiNetwork::start(&network_config, key_pair_refs).await?;
 

sui/src/benchmark.rs

@@ -57,7 +57,7 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
     } else {
         num_cpus::get()
     };
-    let validator_preparer = ValidatorPreparer::new(
+    let validator_preparer = ValidatorPreparer::new_for_local(
         benchmark.running_mode,
         benchmark.working_dir,
         benchmark.committee_size,
@@ -71,7 +71,7 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
             network_server,
             connections,
             benchmark.batch_size,
-            benchmark.use_move,
+            !benchmark.use_native,
             num_transactions,
             validator_preparer,
         ),
@@ -83,7 +83,7 @@ fn run_microbenchmark(benchmark: Benchmark) -> MicroBenchmarkResult {
             network_client,
             network_server,
             connections,
-            benchmark.use_move,
+            !benchmark.use_native,
             num_chunks,
             chunk_size,
             period_us,
@@ -121,6 +121,7 @@ fn run_throughout_microbench(
         use_move,
         batch_size * connections,
         num_transactions / chunk_size,
+        None,
         &mut validator_preparer,
     );
 
@@ -185,12 +186,13 @@ fn run_latency_microbench(
         use_move,
         chunk_size,
         num_chunks,
+        None,
         &mut validator_preparer,
     );
 
     // These are tracer TXes used for measuring latency
     let tracer_txes =
-        tx_cr.generate_transactions(1, use_move, 1, num_chunks, &mut validator_preparer);
+        tx_cr.generate_transactions(1, use_move, 1, num_chunks, None, &mut validator_preparer);
 
     validator_preparer.deploy_validator(_network_server);
 
@@ -218,7 +220,7 @@ fn run_latency_microbench(
     validator_preparer.clean_up();
 
     match result {
-        Ok((load_latencies, tracer_latencies)) => MicroBenchmarkResult::Latency {
+        Ok((load_latencies, tracer_latencies)) => MicroBenchmarkResult::CombinedLatency {
             load_chunk_size: chunk_size,
             load_latencies,
             tick_period_us: period_us as usize,

sui/src/benchmark/bench_types.rs

@@ -1,10 +1,18 @@
 // Copyright (c) 2022, Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
+use crate::config::{Config, NetworkConfig};
+
 use super::load_generator::calculate_throughput;
 use clap::*;
-use std::{default::Default, path::PathBuf};
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use std::collections::BTreeMap;
+use std::default::Default;
+use std::path::PathBuf;
 use strum_macros::EnumString;
+use sui_types::base_types::ObjectID;
+use sui_types::crypto::{KeyPair, PublicKeyBytes};
 
 #[derive(Debug, Clone, Parser)]
 #[clap(
@@ -32,13 +40,13 @@ pub struct Benchmark {
     pub db_cpus: usize,
     /// Use Move orders
     #[clap(long, global = true)]
-    pub use_move: bool,
+    pub use_native: bool,
     #[clap(long, default_value = "2000", global = true)]
     pub batch_size: usize,
 
     #[clap(
         arg_enum,
-        default_value = "local-single-validator-thread",
+        default_value = "single-validator-thread",
         global = true,
         ignore_case = true
     )]
@@ -72,8 +80,9 @@ pub enum BenchmarkType {
 #[derive(Debug, Parser, Clone, Copy, ArgEnum, EnumString)]
 #[clap(rename_all = "kebab-case")]
 pub enum RunningMode {
-    LocalSingleValidatorThread,
-    LocalSingleValidatorProcess,
+    SingleValidatorThread,
+    SingleValidatorProcess,
+    RemoteValidator,
 }
 
 #[derive(Debug, Clone, Parser, Eq, PartialEq, EnumString)]
@@ -134,12 +143,17 @@ pub enum MicroBenchmarkResult {
     Throughput {
         chunk_throughput: f64,
     },
-    Latency {
+    CombinedLatency {
         load_chunk_size: usize,
         tick_period_us: usize,
         load_latencies: Vec<u128>,
         chunk_latencies: Vec<u128>,
     },
+    Latency {
+        load_chunk_size: usize,
+        tick_period_us: usize,
+        latencies: Vec<u128>,
+    },
 }
 
 impl std::fmt::Display for MicroBenchmarkResult {
@@ -148,7 +162,7 @@ impl std::fmt::Display for MicroBenchmarkResult {
             MicroBenchmarkResult::Throughput { chunk_throughput } => {
                 write!(f, "Throughout: {} tps", chunk_throughput)
             }
-            MicroBenchmarkResult::Latency {
+            MicroBenchmarkResult::CombinedLatency {
                 chunk_latencies,
                 load_chunk_size,
                 tick_period_us: tick_period,
@@ -167,6 +181,37 @@ impl std::fmt::Display for MicroBenchmarkResult {
                     chunk_latencies.len()
                 )
             }
+            MicroBenchmarkResult::Latency {
+                load_chunk_size,
+                tick_period_us,
+                latencies,
+            } => {
+                let tracer_avg = latencies.iter().sum::<u128>() as f64 / latencies.len() as f64;
+
+                write!(
+                    f,
+                    "Average Latency {} us @ {} tps ({} samples)",
+                    tracer_avg,
+                    calculate_throughput(*load_chunk_size, *tick_period_us as u128),
+                    latencies.len()
+                )
+            }
         }
     }
 }
+
+#[serde_as]
+#[derive(Serialize, Deserialize)]
+pub struct RemoteLoadGenConfig {
+    /// Keypairs of all the validators
+    /// Ideally we wouldnt need this, but sometime we pre-sign certs
+    pub validator_keypairs: BTreeMap<PublicKeyBytes, KeyPair>,
+    /// Account keypair to use for transactions
+    pub account_keypair: KeyPair,
+    /// ObjectID offset for transaction objects
+    pub object_id_offset: ObjectID,
+    /// Network config for accessing validators
+    pub network_config: NetworkConfig,
+}
+
+impl Config for RemoteLoadGenConfig {}