Skip to content

Commit

Permalink
[State Sync] Add tests for bootstrapping failed validators.
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoshLind @aptos-bot
JoshLind authored and aptos-bot committed Apr 2, 2022
1 parent 682f9f2 commit 610aa7a
Showing 1 changed file with 154 additions and 51 deletions.
205 changes: 154 additions & 51 deletions testsuite/smoke-test/src/state_sync_v2.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use crate::{
};
use aptos_config::config::{BootstrappingMode, ContinuousSyncingMode, NodeConfig};
use aptos_rest_client::Client as RestClient;
use aptos_sdk::{transaction_builder::TransactionFactory, types::LocalAccount};
use aptos_types::PeerId;
use aptos_sdk::types::LocalAccount;
use aptos_types::{account_address::AccountAddress, PeerId};
use forge::{LocalSwarm, NodeExt, Swarm, SwarmExt};
use std::{
fs,
Expand Down Expand Up @@ -136,7 +136,6 @@ async fn test_full_node_continuous_sync_transactions() {

/// Creates a new full node using the given config and swarm
async fn create_full_node(full_node_config: NodeConfig, swarm: &mut LocalSwarm) -> PeerId {
// Create the fullnode
let validator_peer_id = swarm.validators().next().unwrap().peer_id();
let vfn_peer_id = swarm
.add_validator_fullnode(
Expand Down Expand Up @@ -164,14 +163,10 @@ async fn test_full_node_sync(vfn_peer_id: PeerId, mut swarm: LocalSwarm, epoch_c
// Execute a number of transactions on the validator
let validator_peer_id = swarm.validators().next().unwrap().peer_id();
let validator_client = swarm.validator(validator_peer_id).unwrap().rest_client();
let transaction_factory = swarm.chain_info().transaction_factory();

let mut account_0 = create_and_fund_account(&mut swarm, 1000).await;
let mut account_1 = create_and_fund_account(&mut swarm, 1000).await;
let (mut account_0, mut account_1) = create_test_accounts(&mut swarm).await;
execute_transactions(
&mut swarm,
&validator_client,
&transaction_factory,
&mut account_0,
&account_1,
epoch_changes,
Expand All @@ -185,25 +180,17 @@ async fn test_full_node_sync(vfn_peer_id: PeerId, mut swarm: LocalSwarm, epoch_c
.restart()
.await
.unwrap();
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();
wait_for_all_nodes(&mut swarm).await;

// Execute more transactions on the validator and verify the fullnode catches up
execute_transactions(
execute_transactions_and_wait(
&mut swarm,
&validator_client,
&transaction_factory,
&mut account_1,
&account_0,
epoch_changes,
)
.await;
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();
}

#[tokio::test]
Expand Down Expand Up @@ -256,36 +243,24 @@ async fn test_validator_sync(mut swarm: LocalSwarm) {
.validator(validator_peer_ids[0])
.unwrap()
.rest_client();
let transaction_factory = swarm.chain_info().transaction_factory();
let mut account_0 = create_and_fund_account(&mut swarm, 1000).await;
let mut account_1 = create_and_fund_account(&mut swarm, 1000).await;
execute_transactions(
let (mut account_0, mut account_1) = create_test_accounts(&mut swarm).await;
execute_transactions_and_wait(
&mut swarm,
&validator_client_0,
&transaction_factory,
&mut account_0,
&account_1,
true,
)
.await;
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();

// Stop validator 1 and delete the storage
let validator_1 = validator_peer_ids[1];
swarm.validator_mut(validator_1).unwrap().stop();
let node_config = swarm.validator_mut(validator_1).unwrap().config().clone();
let state_db_path = node_config.storage.dir().join("aptosdb");
assert!(state_db_path.as_path().exists());
fs::remove_dir_all(state_db_path).unwrap();
stop_validator_and_delete_storage(&mut swarm, validator_1);

// Execute more transactions
execute_transactions(
&mut swarm,
&validator_client_0,
&transaction_factory,
&mut account_1,
&account_0,
true,
Expand All @@ -294,25 +269,116 @@ async fn test_validator_sync(mut swarm: LocalSwarm) {

// Restart validator 1 and wait for all nodes to catchup
swarm.validator_mut(validator_1).unwrap().start().unwrap();
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();
wait_for_all_nodes(&mut swarm).await;

// Execute multiple transactions and verify validator 1 can sync
execute_transactions(
execute_transactions_and_wait(
&mut swarm,
&validator_client_0,
&transaction_factory,
&mut account_0,
&account_1,
true,
)
.await;
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();
}

#[tokio::test]
async fn test_validator_failure_bootstrap_outputs() {
// Create a swarm of 4 validators with state sync v2 enabled (account
// bootstrapping and transaction output application).
let mut swarm = new_local_swarm_with_aptos(4).await;
for validator in swarm.validators_mut() {
let mut config = validator.config().clone();
config.state_sync.state_sync_driver.enable_state_sync_v2 = true;
config.state_sync.state_sync_driver.bootstrapping_mode =
BootstrappingMode::DownloadLatestAccountStates;
config.state_sync.state_sync_driver.continuous_syncing_mode =
ContinuousSyncingMode::ApplyTransactionOutputs;
config.save(validator.config_path()).unwrap();
validator.restart().await.unwrap();
}

// Test the ability of the validators to sync
test_all_validator_failures(swarm).await;
}

#[tokio::test]
async fn test_validator_failure_bootstrap_execution() {
// Create a swarm of 4 validators with state sync v2 enabled (account
// bootstrapping and transaction execution).
let mut swarm = new_local_swarm_with_aptos(4).await;
for validator in swarm.validators_mut() {
let mut config = validator.config().clone();
config.state_sync.state_sync_driver.enable_state_sync_v2 = true;
config.state_sync.state_sync_driver.bootstrapping_mode =
BootstrappingMode::DownloadLatestAccountStates;
config.state_sync.state_sync_driver.continuous_syncing_mode =
ContinuousSyncingMode::ExecuteTransactions;
config.save(validator.config_path()).unwrap();
validator.restart().await.unwrap();
}

// Test the ability of the validators to sync
test_all_validator_failures(swarm).await;
}

/// A helper method that tests that all validators can sync after a failure and
/// continue to stay up-to-date.
async fn test_all_validator_failures(mut swarm: LocalSwarm) {
// Launch the swarm and wait for it to be ready
swarm.launch().await.unwrap();

// Execute multiple transactions through validator 0
let validator_peer_ids = swarm.validators().map(|v| v.peer_id()).collect::<Vec<_>>();
let validator_0 = validator_peer_ids[0];
let validator_client_0 = swarm.validator(validator_0).unwrap().rest_client();
let (mut account_0, mut account_1) = create_test_accounts(&mut swarm).await;
execute_transactions_and_wait(
&mut swarm,
&validator_client_0,
&mut account_0,
&account_1,
true,
)
.await;

// Go through each validator, stop the node, delete the storage and wait for it to come back
for validator in validator_peer_ids.clone() {
stop_validator_and_delete_storage(&mut swarm, validator);
swarm.validator_mut(validator).unwrap().start().unwrap();
wait_for_all_nodes(&mut swarm).await;
}

// Execute multiple transactions (no epoch changes) and verify validator 0 can sync
execute_transactions_and_wait(
&mut swarm,
&validator_client_0,
&mut account_1,
&account_0,
false,
)
.await;

// Go through each validator, stop the node, delete the storage and wait for it to come back
for validator in validator_peer_ids.clone() {
stop_validator_and_delete_storage(&mut swarm, validator);
swarm.validator_mut(validator).unwrap().start().unwrap();
wait_for_all_nodes(&mut swarm).await;
}

// Execute multiple transactions (with epoch changes) and verify validator 0 can sync
let validator_client_1 = swarm
.validator(validator_peer_ids[0])
.unwrap()
.rest_client();
execute_transactions_and_wait(
&mut swarm,
&validator_client_1,
&mut account_1,
&account_0,
true,
)
.await;
}

#[tokio::test]
Expand All @@ -332,13 +398,10 @@ async fn test_single_validator_failure() {
// Execute multiple transactions
let validator = swarm.validators_mut().next().unwrap();
let validator_client = validator.rest_client();
let transaction_factory = swarm.chain_info().transaction_factory();
let mut account_0 = create_and_fund_account(&mut swarm, 1000).await;
let mut account_1 = create_and_fund_account(&mut swarm, 1000).await;
let (mut account_0, mut account_1) = create_test_accounts(&mut swarm).await;
execute_transactions(
&mut swarm,
&validator_client,
&transaction_factory,
&mut account_0,
&account_1,
true,
Expand All @@ -354,7 +417,6 @@ async fn test_single_validator_failure() {
execute_transactions(
&mut swarm,
&validator_client,
&transaction_factory,
&mut account_1,
&account_0,
true,
Expand All @@ -368,16 +430,17 @@ async fn test_single_validator_failure() {
async fn execute_transactions(
swarm: &mut LocalSwarm,
client: &RestClient,
transaction_factory: &TransactionFactory,
sender: &mut LocalAccount,
receiver: &LocalAccount,
execute_epoch_changes: bool,
) {
let num_transfers = 10;

let transaction_factory = swarm.chain_info().transaction_factory();
if execute_epoch_changes {
transfer_and_reconfig(
client,
transaction_factory,
&transaction_factory,
swarm.chain_info().root_account,
sender,
receiver,
Expand All @@ -387,7 +450,47 @@ async fn execute_transactions(
} else {
for _ in 0..num_transfers {
// Execute simple transfer transactions
transfer_coins(client, transaction_factory, sender, receiver, 1).await;
transfer_coins(client, &transaction_factory, sender, receiver, 1).await;
}
}
}

/// Executes transactions and waits for all nodes to catch up
async fn execute_transactions_and_wait(
swarm: &mut LocalSwarm,
client: &RestClient,
sender: &mut LocalAccount,
receiver: &LocalAccount,
epoch_changes: bool,
) {
execute_transactions(swarm, client, sender, receiver, epoch_changes).await;
wait_for_all_nodes(swarm).await;
}

/// Waits for all nodes to catch up
async fn wait_for_all_nodes(swarm: &mut LocalSwarm) {
swarm
.wait_for_all_nodes_to_catchup(Instant::now() + Duration::from_secs(MAX_CATCH_UP_SECS))
.await
.unwrap();
}

/// Creates and funds two test accounts
async fn create_test_accounts(swarm: &mut LocalSwarm) -> (LocalAccount, LocalAccount) {
let token_amount = 1000;
let account_0 = create_and_fund_account(swarm, token_amount).await;
let account_1 = create_and_fund_account(swarm, token_amount).await;
(account_0, account_1)
}

/// Stops the specified validator and deletes storage
fn stop_validator_and_delete_storage(swarm: &mut LocalSwarm, validator: AccountAddress) {
// Stop the validator
swarm.validator_mut(validator).unwrap().stop();

// Delete the validator storage
let node_config = swarm.validator_mut(validator).unwrap().config().clone();
let state_db_path = node_config.storage.dir().join("aptosdb");
assert!(state_db_path.as_path().exists());
fs::remove_dir_all(state_db_path).unwrap();
}

0 comments on commit 610aa7a

Please sign in to comment.