Merge branch 'dev' into dvush/resotre-tree-restore-again

core/loadtest/src/configs/reallife.json

@@ -7,5 +7,19 @@
     "input_account": {
         "address": "0x36615Cf349d7F6344891B1e7CA7C72883F5dc049",
         "private_key": "0x7726827caac94a7f9e1b160f7ea819f172f7b6f9d2a97f992c38edeab82d4110"
-    }
+    },
+    "additional_accounts": [
+        {
+            "address": "0xa61464658AfeAf65CccaaFD3a512b69A83B77618",
+            "private_key": "0xac1e735be8536c6534bb4f17f06f6afc73b2b5ba84ac2cfb12f7461b20c0bbe3"
+        },
+        {
+            "address": "0x0D43eB5B8a47bA8900d84AA36656c92024e9772e",
+            "private_key": "0xd293c684d884d56f8d6abd64fc76757d3664904e309a0645baf8522ab6366d9e"
+        },
+        {
+            "address": "0xA13c10C0D5bd6f79041B9835c63f91de35A15883",
+            "private_key": "0x850683b40d4a740aa6e745f889a6fdc8327be76e122f5aba645a5b02d0248db8"
+        }
+    ]
 }

core/loadtest/src/scenarios/configs.rs

@@ -66,6 +66,8 @@ pub struct RealLifeConfig {
     pub use_all_block_sizes: bool,
     /// Main account to deposit ETH from.
     pub input_account: AccountInfo,
+    /// Additional accounts to deposit ETH from for the satellite scenario.
+    pub additional_accounts: Vec<AccountInfo>,
 }
 
 impl RealLifeConfig {

core/loadtest/src/scenarios/real_life.rs → core/loadtest/src/scenarios/real_life/mod.rs

@@ -71,6 +71,7 @@ use models::{
 };
 use testkit::zksync_account::ZksyncAccount;
 // Local deps
+use self::satellite::SatelliteScenario;
 use crate::{
     rpc_client::RpcClient,
     scenarios::{
@@ -82,6 +83,8 @@ use crate::{
     test_accounts::TestAccount,
 };
 
+mod satellite;
+
 #[derive(Debug)]
 struct ScenarioExecutor {
     rpc_client: RpcClient,
@@ -111,6 +114,9 @@ struct ScenarioExecutor {
     /// operate.
     estimated_fee_for_op: BigUint,
 
+    /// Satellite scenario to run alongside with the funds rotation cycles.
+    satellite_scenario: Option<SatelliteScenario>,
+
     /// Event loop handle so transport for Eth account won't be invalidated.
     _event_loop_handle: EventLoopHandle,
 }
@@ -133,6 +139,13 @@ impl ScenarioExecutor {
         // Create main account to deposit money from and to return money back later.
         let main_account = TestAccount::from_info(&config.input_account, &transport, &ctx.options);
 
+        // Load additional accounts for the satellite scenario.
+        let additional_accounts: Vec<_> = config
+            .additional_accounts
+            .iter()
+            .map(|acc| TestAccount::from_info(acc, &transport, &ctx.options))
+            .collect();
+
         let block_sizes = Self::get_block_sizes(config.use_all_block_sizes);
 
         if config.use_all_block_sizes {
@@ -143,6 +156,14 @@ impl ScenarioExecutor {
         }
 
         let transfer_size = closest_packable_token_amount(&BigUint::from(config.transfer_size));
+        let verify_timeout = Duration::from_secs(config.block_timeout);
+
+        let satellite_scenario = Some(SatelliteScenario::new(
+            rpc_client.clone(),
+            additional_accounts,
+            transfer_size.clone(),
+            verify_timeout,
+        ));
 
         Self {
             rpc_client,
@@ -156,10 +177,12 @@ impl ScenarioExecutor {
 
             block_sizes,
 
-            verify_timeout: Duration::from_secs(config.block_timeout),
+            verify_timeout,
 
             estimated_fee_for_op: 0u32.into(),
 
+            satellite_scenario,
+
             _event_loop_handle,
         }
     }
@@ -221,7 +244,15 @@ impl ScenarioExecutor {
         self.initialize().await?;
         self.deposit().await?;
         self.initial_transfer().await?;
-        self.funds_rotation().await?;
+
+        // Take the satellite scenario, as we have to borrow it mutably.
+        let mut satellite_scenario = self.satellite_scenario.take().unwrap();
+
+        // Run funds rotation phase and the satellite scenario in parallel.
+        let funds_rotation_future = self.funds_rotation();
+        let satellite_scenario_future = satellite_scenario.run();
+        futures::try_join!(funds_rotation_future, satellite_scenario_future)?;
+
         self.collect_funds().await?;
         self.withdraw().await?;
         self.finish().await?;
@@ -249,7 +280,11 @@ impl ScenarioExecutor {
         // And after that we have to make the fee packable.
         fee = closest_packable_fee_amount(&fee);
 
-        self.estimated_fee_for_op = fee;
+        self.estimated_fee_for_op = fee.clone();
+
+        if let Some(scenario) = self.satellite_scenario.as_mut() {
+            scenario.set_estimated_fee(fee);
+        };
 
         Ok(())
     }
@@ -547,8 +582,6 @@ impl ScenarioExecutor {
 
     /// Withdraws the money from the main account back to the Ethereum.
     async fn withdraw(&mut self) -> Result<(), failure::Error> {
-        let mut sent_txs = SentTransactions::new();
-
         let current_balance = self.main_account.eth_acc.eth_balance().await?;
 
         let fee = self.withdraw_fee(&self.main_account.zk_acc).await;
@@ -574,6 +607,7 @@ impl ScenarioExecutor {
             .rpc_client
             .send_tx(tx.clone(), eth_sign.clone())
             .await?;
+        let mut sent_txs = SentTransactions::new();
         sent_txs.add_tx_hash(tx_hash);
 
         wait_for_verify(sent_txs, self.verify_timeout, &self.rpc_client).await?;

core/loadtest/src/scenarios/real_life/satellite.rs

@@ -0,0 +1,210 @@
+//! Satellite scenario for real-life loadtest.
+//!
+//! Satellite scenario is ran concurrently to the main scenario
+//! and it performs several deposit / withdraw operations at the same
+//! time as the funds are rotated in the main scenario.
+//!
+//! The purpose of the satellite scenario is to ensure that deposits
+//! and withdraws are processed correctly when the node is under a
+//! load of many transfers.
+
+// Built-in deps
+use std::time::{Duration, Instant};
+// External deps
+use num::BigUint;
+use tokio::time;
+use web3::types::Address;
+// Workspace deps
+use models::node::{closest_packable_fee_amount, closest_packable_token_amount};
+// Local deps
+use crate::{
+    rpc_client::RpcClient,
+    scenarios::utils::{deposit_single, wait_for_verify},
+    sent_transactions::SentTransactions,
+    test_accounts::TestAccount,
+};
+
+#[derive(Debug)]
+pub struct SatelliteScenario {
+    rpc_client: RpcClient,
+    accounts: Vec<TestAccount>,
+    deposit_size: BigUint,
+    verify_timeout: Duration,
+    estimated_fee_for_op: BigUint,
+}
+
+impl SatelliteScenario {
+    pub fn new(
+        rpc_client: RpcClient,
+        accounts: Vec<TestAccount>,
+        deposit_size: BigUint,
+        verify_timeout: Duration,
+    ) -> Self {
+        Self {
+            rpc_client,
+            accounts,
+            deposit_size,
+            verify_timeout,
+            estimated_fee_for_op: 0u32.into(),
+        }
+    }
+
+    pub fn set_estimated_fee(&mut self, estimated_fee_for_op: BigUint) {
+        self.estimated_fee_for_op = estimated_fee_for_op
+    }
+
+    pub async fn run(&mut self) -> Result<(), failure::Error> {
+        self.initialize().await?;
+
+        // Deposit & withdraw phase.
+        for account_id in 0..self.accounts.len() {
+            self.deposit_withdraw(account_id).await?;
+        }
+
+        // Deposit & full exit phase.
+        for account_id in 0..self.accounts.len() {
+            self.deposit_full_exit(account_id).await?;
+        }
+
+        Ok(())
+    }
+
+    async fn initialize(&mut self) -> Result<(), failure::Error> {
+        for account in self.accounts.iter_mut() {
+            account.update_nonce_values(&self.rpc_client).await?;
+        }
+
+        Ok(())
+    }
+
+    async fn deposit_withdraw(&mut self, account_id: usize) -> Result<(), failure::Error> {
+        log::info!(
+            "Satellite deposit/withdraw iteration {} started",
+            account_id
+        );
+
+        self.deposit(account_id).await?;
+        log::info!("Satellite deposit iteration {} completed", account_id);
+
+        self.withdraw(account_id).await?;
+        log::info!("Satellite withdraw iteration {} completed", account_id);
+
+        Ok(())
+    }
+
+    async fn deposit_full_exit(&mut self, account_id: usize) -> Result<(), failure::Error> {
+        log::info!(
+            "Satellite deposit/full exit iteration {} started",
+            account_id
+        );
+
+        self.deposit(account_id).await?;
+        log::info!("Satellite deposit iteration {} completed", account_id);
+
+        self.full_exit(account_id).await?;
+        log::info!("Satellite full exit iteration {} completed", account_id);
+
+        Ok(())
+    }
+
+    async fn deposit(&mut self, account_id: usize) -> Result<(), failure::Error> {
+        let account = &mut self.accounts[account_id];
+
+        let amount_to_deposit = self.deposit_size.clone() + self.estimated_fee_for_op.clone();
+
+        // Ensure that account does have enough money.
+        let account_balance = account.eth_acc.eth_balance().await?;
+        if amount_to_deposit > account_balance {
+            panic!("Main ETH account does not have enough balance to run the test with the provided config");
+        }
+
+        // Deposit funds and wait for operation to be executed.
+        deposit_single(account, amount_to_deposit, &self.rpc_client).await?;
+
+        // Now when deposits are done it is time to update account id.
+        account.update_account_id(&self.rpc_client).await?;
+
+        // ...and change the main account pubkey.
+        // We have to change pubkey after the deposit so we'll be able to use corresponding
+        // `zkSync` account.
+        let (change_pubkey_tx, eth_sign) = (account.sign_change_pubkey(), None);
+        let mut sent_txs = SentTransactions::new();
+        let tx_hash = self.rpc_client.send_tx(change_pubkey_tx, eth_sign).await?;
+        sent_txs.add_tx_hash(tx_hash);
+        wait_for_verify(sent_txs, self.verify_timeout, &self.rpc_client).await?;
+
+        Ok(())
+    }
+
+    async fn withdraw(&mut self, account_id: usize) -> Result<(), failure::Error> {
+        let account = &mut self.accounts[account_id];
+
+        let current_balance = account.eth_acc.eth_balance().await?;
+
+        let fee = self
+            .rpc_client
+            .get_tx_fee("Withdraw", account.eth_acc.address, "ETH")
+            .await
+            .expect("Can't get tx fee");
+
+        let fee = closest_packable_fee_amount(&fee);
+
+        let comitted_account_state = self
+            .rpc_client
+            .account_state_info(account.zk_acc.address)
+            .await?
+            .committed;
+        let account_balance = comitted_account_state.balances["ETH"].0.clone();
+        let withdraw_amount = &account_balance - &fee;
+        let withdraw_amount = closest_packable_token_amount(&withdraw_amount);
+
+        let (tx, eth_sign) = account.sign_withdraw(withdraw_amount.clone(), fee);
+        let tx_hash = self
+            .rpc_client
+            .send_tx(tx.clone(), eth_sign.clone())
+            .await?;
+        let mut sent_txs = SentTransactions::new();
+        sent_txs.add_tx_hash(tx_hash);
+
+        wait_for_verify(sent_txs, self.verify_timeout, &self.rpc_client).await?;
+
+        let expected_balance = current_balance + withdraw_amount;
+
+        let timeout_minutes = 10;
+        let timeout = Duration::from_secs(timeout_minutes * 60);
+        let start = Instant::now();
+
+        let polling_interval = Duration::from_millis(250);
+        let mut timer = time::interval(polling_interval);
+
+        loop {
+            let current_balance = account.eth_acc.eth_balance().await?;
+            if current_balance == expected_balance {
+                break;
+            }
+            if start.elapsed() > timeout {
+                failure::bail!(
+                    "ETH funds were not received for {} minutes",
+                    timeout_minutes
+                );
+            }
+            timer.tick().await;
+        }
+
+        Ok(())
+    }
+
+    async fn full_exit(&mut self, account_id: usize) -> Result<(), failure::Error> {
+        let account = &mut self.accounts[account_id];
+
+        let eth_token_address = Address::zero();
+        let zksync_account_id = account.zk_acc.get_account_id().expect("No account ID set");
+
+        account
+            .eth_acc
+            .full_exit(zksync_account_id, eth_token_address)
+            .await?;
+
+        Ok(())
+    }
+}

core/prover/Cargo.toml

@@ -32,3 +32,4 @@ failure = "0.1"
 backoff = "0.1.6"
 clap = "2.33.0"
 ctrlc = { version = "3.1", features = ["termination"] }
+rand = "0.7"

core/prover/src/client.rs

@@ -190,7 +190,7 @@ impl crate::ApiClient for ApiClient {
                 .map_err(|e| format_err!("failed to read prover data response: {}", e))?;
             let res: Option<ProverData> = serde_json::from_str(&text)
                 .map_err(|e| format_err!("failed to parse prover data response: {}", e))?;
-            Ok(res.ok_or_else(|| format_err!("couldn't get ProverData for block {}", block))?)
+            Ok(res.ok_or_else(|| format_err!("ProverData for block {} is not ready yet", block))?)
         };
 
         let prover_data = self.with_retries(&op)?;