Make revert blocks works correct (#2145)

* Add tx hash to mempool Signed-off-by: deniallugo <[email protected]> * Select incomplete blocks Signed-off-by: deniallugo <[email protected]> * Return old version of state keeper Signed-off-by: deniallugo <[email protected]> * Save reverted block into database Signed-off-by: deniallugo <[email protected]> * Remove reverted block Signed-off-by: deniallugo <[email protected]> * Fix test Signed-off-by: deniallugo <[email protected]> * Restore reverted blocks directly from state keeper Signed-off-by: deniallugo <[email protected]> * Revert incomplete and pending block Signed-off-by: deniallugo <[email protected]> remove notify Signed-off-by: deniallugo <[email protected]> Setup envs Signed-off-by: deniallugo <[email protected]> Logs Signed-off-by: deniallugo <[email protected]> * Incude allowed failure percent Signed-off-by: deniallugo <[email protected]> Fix env Signed-off-by: deniallugo <[email protected]> * Restore tests Signed-off-by: deniallugo <[email protected]> Self review Signed-off-by: deniallugo <[email protected]> * Update core/lib/storage/src/chain/mempool/records.rs Co-authored-by: Igor Aleksanov <[email protected]> * Rewrite script to ts Signed-off-by: deniallugo <[email protected]> * Fix typescript Signed-off-by: deniallugo <[email protected]> * Update core/bin/zksync_core/src/state_keeper/mod.rs Co-authored-by: Igor Aleksanov <[email protected]> Co-authored-by: Igor Aleksanov <[email protected]>
advandrev16 · Feb 3, 2022 · caf5bd1 · caf5bd1
1 parent e0eb33b
commit caf5bd1
Show file tree

Hide file tree

Showing 31 changed files with 1,368 additions and 402 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,7 +46,7 @@ jobs:
         run: |
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
-        
+
       - name: start-services
         run: |
           docker-compose -f docker-compose-runner.yml down
@@ -87,7 +87,7 @@ jobs:
         run: |
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
-        
+
       - name: start-services
         run: |
           docker-compose -f docker-compose-runner.yml down
@@ -126,7 +126,7 @@ jobs:
 
       - name: integration-rust-sdk
         run: ci_run zk test i rust-sdk
-      
+
       - name: integration-withdrawal-helpers
         run: ci_run zk test i withdrawal-helpers
 
@@ -186,11 +186,75 @@ jobs:
       - name: integration-testkit
         run: ci_run zk test integration testkit --offline
 
+  revert-blocks:
+    runs-on: [self-hosted, FAST]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+
+      - name: start-services
+        run: |
+          docker-compose -f docker-compose-runner.yml down
+          docker-compose -f docker-compose-runner.yml pull
+          docker-compose -f docker-compose-runner.yml up --build -d geth postgres zk
+          ci_run sccache --start-server
+
+      - name: init
+        run: |
+          ci_run ln -s /usr/src/keys/setup keys/setup
+          ci_run zk
+          ci_run zk dummy-prover enable --no-redeploy
+          ci_run zk init
+
+      - name: restart dev-liquidity-token-watcher and dev-ticker
+        run: docker-compose -f docker-compose-runner.yml restart dev-liquidity-token-watcher dev-ticker
+
+      - name: run-services
+        run: |
+          ci_run zk server core &>server.log &
+          ci_run zk server api &>api.log &
+          ci_run sleep 10
+
+      - name: loadtest
+        run: |
+          ci_run cargo build --release --bin loadnext 
+          cd $ZKSYNC_HOME
+          docker-compose -f docker-compose-runner.yml exec -T -e ALLOWED_PERCENT=20 -e RUST_LOG=loadnext=debug -e ZKSYNC_RPC_ADDR=http://127.0.0.2:3030 -e WEB3_URL=http://geth:8545 -e ETH_NETWORK=localhost -e MASTER_WALLET_PK=74d8b3a188f7260f67698eb44da07397a298df5427df681ef68c45b34b61f998 -e ACCOUNTS_AMOUNT=5 -e OPERATIONS_PER_ACCOUNT=5 -e MAIN_TOKEN=DAI zk ./target/release/loadnext
+          docker-compose -f docker-compose-runner.yml exec -T -e ZKSYNC_REST_ADDR=http://127.0.0.2:3001  zk ts-node core/tests/check-block-root-hahes.ts
+
+
+      - name: stop-server
+        run: |
+          ci_run killall zksync_server
+
+      - name: revert-blocks
+        run: |
+          ci_run zk f cargo build --release --bin block_revert
+          ci_run zk f ./target/release/block_revert --last-correct-block=2 all
+      - name: check-server
+        run: |
+          ci_run zk server &>server.log &
+          ci_run sleep 30
+          docker-compose -f docker-compose-runner.yml exec -T -e ZKSYNC_REST_ADDR=http://127.0.0.2:3001  zk ts-node core/tests/check-block-root-hahes.ts
+
+
+      - name: Show logs
+        if: always()
+        run: |
+          ci_run cat server.log
+          ci_run cat api.log
+
+
   notify:
     if: always()
     name: Notify on failures
     runs-on: ubuntu-latest
-    needs: [lint, unit-tests, integration, circuit-tests, testkit]
+    needs: [lint, unit-tests, integration, circuit-tests, testkit, revert-blocks]
     steps:
       -
         if: failure()

diff --git a/.gitignore b/.gitignore
@@ -36,6 +36,7 @@ Cargo.lock
 /volumes
 /logs
 /loadtest-config
+/core/tests/blocks.json
 
 .ipynb_checkpoints
 

diff --git a/core/bin/block_revert/src/main.rs b/core/bin/block_revert/src/main.rs
@@ -76,12 +76,6 @@ async fn revert_blocks_in_storage(
         .remove_eth_unprocessed_aggregated_ops()
         .await?;
     println!("`eth_unprocessed_aggregated_ops` table is cleaned");
-    transaction
-        .chain()
-        .operations_schema()
-        .return_executed_priority_operations_to_mempool(last_block)
-        .await?;
-    println!("`executed_priority_operations` table is cleaned");
     transaction
         .chain()
         .operations_schema()
@@ -160,18 +154,18 @@ async fn revert_blocks_on_contract(
 ) -> anyhow::Result<()> {
     let tx_arg = Token::Array(blocks.iter().map(stored_block_info).collect());
     let data = client.encode_tx_data("revertBlocks", tx_arg);
-    let gas_limit = 200000 + 5000 * blocks.len();
+    let gas_limit = 200000 + 15000 * blocks.len();
     let signed_tx = client
         .sign_prepared_tx(data, Options::with(|f| f.gas = Some(U256::from(gas_limit))))
         .await
         .map_err(|e| format_err!("Revert blocks send err: {}", e))?;
     let receipt = send_raw_tx_and_wait_confirmation(client, signed_tx.raw_tx).await?;
     storage.ethereum_schema().get_next_nonce().await
         .expect("Ethereum tx has been sent but updating operator nonce in storage has failed. You need to update it manually");
-    ensure!(
-        receipt.status == Some(U64::from(1)),
-        "Tx to contract failed"
-    );
+    if receipt.status != Some(U64::from(1)) {
+        let reason = client.failure_reason(signed_tx.hash).await?;
+        anyhow::bail!("Tx to contract failed {:?}", reason);
+    }
 
     println!("Blocks were reverted on contract");
     Ok(())
@@ -218,7 +212,7 @@ struct Opt {
     #[structopt(subcommand)]
     command: Command,
     /// Private key of operator which will call the contract function.
-    #[structopt(long = "key", env = "REVERT_TOOL_OPERATOR_PRIVATE_KEY")]
+    #[structopt(long = "key", env = "ETH_SENDER_SENDER_OPERATOR_PRIVATE_KEY")]
     operator_private_key: String,
 }
 

diff --git a/core/bin/zksync_api/src/api_server/web3/rpc_impl.rs b/core/bin/zksync_api/src/api_server/web3/rpc_impl.rs
@@ -428,7 +428,7 @@ impl Web3RpcApp {
                             block_number: block_number.0,
                             block_index: Some(op.block_index),
                             from: op.priority_op.data.from_account(),
-                            to: op.priority_op.data.to_account(),
+                            to: Some(op.priority_op.data.to_account()),
                             nonce: op.priority_op.serial_id as u32,
                             tx_hash: H256::from_slice(op.priority_op.tx_hash().as_ref()),
                         },

diff --git a/core/bin/zksync_core/src/committer/mod.rs b/core/bin/zksync_core/src/committer/mod.rs
@@ -21,6 +21,7 @@ mod aggregated_committer;
 #[derive(Debug)]
 pub enum CommitRequest {
     PendingBlock((PendingBlock, AppliedUpdatesRequest)),
+    RemoveRevertedBlock(BlockNumber),
     SealIncompleteBlock((BlockCommitRequest, AppliedUpdatesRequest)),
     FinishBlock(BlockFinishRequest),
 }
@@ -56,6 +57,7 @@ async fn handle_new_commit_task(
     mut mempool_req_sender: Sender<MempoolBlocksRequest>,
     pool: ConnectionPool,
 ) {
+    vlog::info!("Run committer");
     while let Some(request) = rx_for_ops.next().await {
         match request {
             CommitRequest::SealIncompleteBlock((block_commit_request, applied_updates_req)) => {
@@ -73,10 +75,28 @@ async fn handle_new_commit_task(
             CommitRequest::FinishBlock(request) => {
                 finish_block(request, &pool).await;
             }
+            CommitRequest::RemoveRevertedBlock(block_number) => {
+                remove_reverted_block(block_number, &pool).await;
+            }
         }
     }
 }
 
+async fn remove_reverted_block(block_number: BlockNumber, pool: &ConnectionPool) {
+    let start = Instant::now();
+    let mut storage = pool
+        .access_storage()
+        .await
+        .expect("db connection fail for committer");
+    storage
+        .chain()
+        .mempool_schema()
+        .remove_reverted_block(block_number)
+        .await
+        .expect("Failed to remove reverted blocks");
+    metrics::histogram!("committer.remove_reverted_block", start.elapsed());
+}
+
 async fn save_pending_block(
     pending_block: PendingBlock,
     applied_updates_request: AppliedUpdatesRequest,

diff --git a/core/bin/zksync_core/src/lib.rs b/core/bin/zksync_core/src/lib.rs
@@ -123,9 +123,14 @@ pub async fn run_core(
     let mut storage_processor = connection_pool.access_storage().await?;
 
     // Start state keeper and root hash calculator.
-    let state_keeper_init = ZkSyncStateInitParams::restore_from_db(&mut storage_processor).await;
+    let state_keeper_init = ZkSyncStateInitParams::restore_from_db(
+        &mut storage_processor,
+        config.chain.state_keeper.fee_account_addr,
+        &config.chain.state_keeper.block_chunk_sizes,
+    )
+    .await;
 
-    let (state_keeper, root_hash_calculator) = ZkSyncStateKeeper::new(
+    let (mut state_keeper, root_hash_calculator) = ZkSyncStateKeeper::new(
         state_keeper_init,
         config.chain.state_keeper.fee_account_addr,
         state_keeper_req_receiver,
@@ -136,6 +141,9 @@ pub async fn run_core(
         processed_tx_events_sender,
     );
     let root_hash_queue = state_keeper.root_hash_queue();
+    // Execute reverted blocks before start
+    state_keeper.execute_reverted_blocks().await;
+
     let state_keeper_task = start_state_keeper(state_keeper);
     let root_hash_calculator_task = start_root_hash_calculator(root_hash_calculator);
 

diff --git a/core/bin/zksync_core/src/mempool/mod.rs b/core/bin/zksync_core/src/mempool/mod.rs
@@ -305,7 +305,12 @@ impl MempoolBlocksHandler {
             let priority_op =
                 iter::from_fn(|| mempool_state.transactions_queue.pop_front_priority_op())
                     .find(|op| op.serial_id == next_serial_id)
-                    .expect("Operation not found in the priority queue");
+                    .unwrap_or_else(|| {
+                        panic!(
+                            "Operation not found in the priority queue {}",
+                            next_serial_id,
+                        )
+                    });
 
             // If the operation doesn't fit, return the proposed block.
             if priority_op.data.chunks() <= chunks_left {

diff --git a/core/bin/zksync_core/src/state_keeper/init_params.rs b/core/bin/zksync_core/src/state_keeper/init_params.rs
@@ -1,9 +1,9 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 // External uses
 // Workspace uses
 use zksync_types::{
-    block::PendingBlock as SendablePendingBlock, Account, AccountId, AccountTree, Address,
-    BlockNumber, TokenId, NFT,
+    block::{IncompleteBlock, PendingBlock as SendablePendingBlock},
+    Account, AccountId, AccountTree, Address, BlockNumber, TokenId, NFT,
 };
 
 use super::{
@@ -21,6 +21,7 @@ pub struct ZkSyncStateInitParams {
 
     pub pending_block: Option<SendablePendingBlock>,
     pub root_hash_jobs: Vec<BlockRootHashJob>,
+    pub reverted_blocks: VecDeque<IncompleteBlock>,
 }
 
 impl Default for ZkSyncStateInitParams {
@@ -40,10 +41,15 @@ impl ZkSyncStateInitParams {
 
             pending_block: None,
             root_hash_jobs: Vec::new(),
+            reverted_blocks: Default::default(),
         }
     }
 
-    pub async fn restore_from_db(storage: &mut zksync_storage::StorageProcessor<'_>) -> Self {
+    pub async fn restore_from_db(
+        storage: &mut zksync_storage::StorageProcessor<'_>,
+        fee_account_addr: Address,
+        available_chunk_sizes: &[usize],
+    ) -> Self {
         let (last_block_number, tree, acc_id_by_addr) = Self::load_account_tree(storage).await;
 
         let unprocessed_priority_op =
@@ -52,6 +58,12 @@ impl ZkSyncStateInitParams {
 
         let pending_block = Self::load_pending_block(storage, last_block_number).await;
         let root_hash_jobs = Self::load_root_hash_jobs(storage).await;
+        let fee_account_id = acc_id_by_addr
+            .get(&fee_account_addr)
+            .cloned()
+            .expect("Fee account should be present in the account tree");
+        let reverted_blocks =
+            Self::load_reverted_blocks(storage, fee_account_id, available_chunk_sizes).await;
 
         let init_params = Self {
             tree,
@@ -61,12 +73,15 @@ impl ZkSyncStateInitParams {
             unprocessed_priority_op,
             pending_block,
             root_hash_jobs,
+            reverted_blocks,
         };
 
         vlog::info!(
-            "Loaded committed state: last block number: {}, unprocessed priority op: {}",
+            "Loaded committed state: last block number: {}, unprocessed priority op: {} reverted_blocks {}",
             *init_params.last_block_number,
-            init_params.unprocessed_priority_op
+            init_params.unprocessed_priority_op,
+            init_params.reverted_blocks.len()
+
         );
         init_params
     }
@@ -83,6 +98,18 @@ impl ZkSyncStateInitParams {
         )
     }
 
+    async fn load_reverted_blocks(
+        storage: &mut zksync_storage::StorageProcessor<'_>,
+        fee_account_id: AccountId,
+        available_block_chunk_sizes: &[usize],
+    ) -> VecDeque<IncompleteBlock> {
+        storage
+            .chain()
+            .mempool_schema()
+            .get_reverted_blocks(available_block_chunk_sizes, fee_account_id)
+            .await
+            .unwrap()
+    }
     async fn load_pending_block(
         storage: &mut zksync_storage::StorageProcessor<'_>,
         last_block_number: BlockNumber,