Refactore remove proof tool

Signed-off-by: deniallugo <[email protected]>
nguyenhoangduon473 · Dec 20, 2021 · 9639fc8 · 9639fc8
1 parent b5b7681
commit 9639fc8
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 85 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -7,7 +7,7 @@ members = [
     "core/bin/prover",
     "core/bin/parse_pub_data",
     "core/bin/block_revert",
-    "core/bin/aggregation_deleted",
+    "core/bin/remove_proofs",
 
     # Server micro-services
     "core/bin/zksync_api",

diff --git a/core/bin/aggregation_deleted/Cargo.toml → core/bin/remove_proofs/Cargo.toml b/core/bin/aggregation_deleted/Cargo.toml → core/bin/remove_proofs/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "aggregation_deleted"
+name = "remove_proofs"
 version = "1.0.0"
 edition = "2018"
 authors = ["The Matter Labs Team <[email protected]>"]

diff --git a/core/bin/aggregation_deleted/src/main.rs → core/bin/remove_proofs/src/main.rs b/core/bin/aggregation_deleted/src/main.rs → core/bin/remove_proofs/src/main.rs
@@ -3,7 +3,6 @@ use structopt::StructOpt;
 use zksync_storage::StorageProcessor;
 use zksync_types::BlockNumber;
 
-// TODO: don't use anyhow (ZKS-588)
 async fn remove_operations(
     storage: &mut StorageProcessor<'_>,
     last_block: BlockNumber,
@@ -13,7 +12,7 @@ async fn remove_operations(
         .prover_schema()
         .remove_witnesses(last_block)
         .await?;
-    println!("witness table is cleaned");
+    println!("`witness` table is cleaned");
 
     transaction
         .chain()
@@ -46,7 +45,7 @@ async fn remove_operations(
 
     transaction.commit().await?;
 
-    println!("Blocks were reverted in storage");
+    println!("Proofs were deleted from storage");
     Ok(())
 }
 
@@ -71,24 +70,25 @@ async fn main() -> anyhow::Result<()> {
         .block_schema()
         .get_last_committed_confirmed_block()
         .await?;
-    let last_verified_block = storage
+    let last_proven_block = storage
         .chain()
         .block_schema()
-        .get_last_verified_confirmed_block()
+        .get_last_proven_confirmed_block()
         .await?;
 
     println!(
-        "Last committed block {} verified {}",
-        &last_commited_block, &last_verified_block
+        "Last committed block {} proven {}",
+        &last_commited_block, &last_proven_block
     );
+
     ensure!(
-        *last_verified_block <= opt.last_correct_block,
-        "Some blocks to revert are already verified"
+        *last_proven_block <= opt.last_correct_block,
+        "Some proofs has already been published to ethereum"
     );
 
     let last_block = BlockNumber(opt.last_correct_block);
 
-    println!("Start remove block operations in database");
+    println!("Start remove proofs");
     remove_operations(&mut storage, last_block).await?;
 
     Ok(())

diff --git a/core/lib/prover_utils/src/fs_utils.rs b/core/lib/prover_utils/src/fs_utils.rs
@@ -1,7 +1,7 @@
 use super::{SETUP_MAX_POW2, SETUP_MIN_POW2};
 use anyhow::format_err;
 use std::fs::File;
-use std::io::{copy, BufReader, Read, Write};
+use std::io::{copy, BufReader, Read};
 use std::path::PathBuf;
 use zksync_crypto::bellman::kate_commitment::{Crs, CrsForLagrangeForm, CrsForMonomialForm};
 use zksync_crypto::params::{account_tree_depth, balance_tree_depth};

diff --git a/core/lib/prover_utils/src/lib.rs b/core/lib/prover_utils/src/lib.rs
@@ -1,5 +1,3 @@
-use crate::fs_utils::{get_block_verification_key_path, get_exodus_verification_key_path};
-use anyhow::Error;
 use lazy_static::lazy_static;
 use std::collections::HashMap;
 use std::fs::File;
@@ -25,6 +23,8 @@ use zksync_crypto::proof::SingleProof;
 use zksync_crypto::recursive_aggregation_circuit::circuit::create_vks_tree;
 use zksync_crypto::{Engine, Fr};
 
+use crate::fs_utils::{get_block_verification_key_path, get_exodus_verification_key_path};
+
 pub mod aggregated_proofs;
 pub mod api;
 pub mod exit_proof;
@@ -126,7 +126,6 @@ impl SetupForStepByStepProver {
         metrics::histogram!("prover", start.elapsed(), "stage" => "verify_proof", "type" => "single_proof");
         if !valid {
             let start = Instant::now();
-            // we do this way here so old precomp is dropped
             let mut cs = TestConstraintSystem::<Engine>::new();
             circuit.synthesize(&mut cs).unwrap();
 
@@ -195,7 +194,7 @@ pub fn get_universal_setup_monomial_form(
     } else {
         let start = Instant::now();
         // try to find cache on disk
-        let res = if Ok(res) = fs_utils::get_universal_setup_monomial_form(power_of_two) {
+        let res = if let Ok(res) = fs_utils::get_universal_setup_monomial_form(power_of_two) {
             res
         } else {
             network_utils::download_universal_setup_monomial_form(power_of_two)?;

diff --git a/core/lib/prover_utils/src/network_utils.rs b/core/lib/prover_utils/src/network_utils.rs
@@ -4,8 +4,6 @@ use anyhow::format_err;
 use backoff::Operation;
 use reqwest::blocking::Response;
 use std::time::Duration;
-use zksync_crypto::bellman::kate_commitment::{Crs, CrsForMonomialForm};
-use zksync_crypto::Engine;
 
 /// Downloads universal setup in the monomial form of the given power of two (range: SETUP_MIN_POW2..=SETUP_MAX_POW2)
 pub fn download_universal_setup_monomial_form(power_of_two: u32) -> Result<(), anyhow::Error> {

diff --git a/core/lib/storage/sqlx-data.json b/core/lib/storage/sqlx-data.json
@@ -2245,26 +2245,6 @@
       ]
     }
   },
-  "4868140cb6b644efc751687b9a8dcd04324072bffbcc69ec29306b8a9793645e": {
-    "query": "\n            SELECT SUM(usd_amount_scale6) as total FROM subsidies\n            WHERE subsidy_type = $1\n            ",
-    "describe": {
-      "columns": [
-        {
-          "ordinal": 0,
-          "name": "total",
-          "type_info": "Numeric"
-        }
-      ],
-      "parameters": {
-        "Left": [
-          "Text"
-        ]
-      },
-      "nullable": [
-        null
-      ]
-    }
-  },
   "4a0bc713a57201aa894b96acdb462c03d3ad63cf4fbc8a14b9ac5e2e02121207": {
     "query": "\n            SELECT * FROM ticker_market_volume\n            WHERE token_id = $1\n            LIMIT 1\n            ",
     "describe": {
@@ -4810,37 +4790,6 @@
       "nullable": []
     }
   },
-  "a665923ec57382f357f6bb65f6e35876fbfedbf1661b3ce34f2458b63eebc68e": {
-    "query": "\n            INSERT INTO subsidies ( tx_hash, usd_amount_scale6, full_cost_usd_scale6, token_id, token_amount, full_cost_token, subsidy_type )\n            VALUES ( $1, $2, $3, $4, $5, $6, $7 )\n            ",
-    "describe": {
-      "columns": [],
-      "parameters": {
-        "Left": [
-          "Bytea",
-          "Int8",
-          "Int8",
-          "Int4",
-          "Numeric",
-          "Numeric",
-          "Varchar"
-        ]
-      },
-      "nullable": []
-    }
-  },
-  "a77668a3dce7f7cd1f45816f932eea685d429c3d75b40ea8e1a1bb9fc29f11c6": {
-    "query": "UPDATE prover_job_queue SET (job_status, updated_at, updated_by) = ($1, now(), 'server_clean_idle')\n            WHERE job_status = $2 and (now() - updated_at) >= interval '120 seconds'",
-    "describe": {
-      "columns": [],
-      "parameters": {
-        "Left": [
-          "Int4",
-          "Int4"
-        ]
-      },
-      "nullable": []
-    }
-  },
   "a7c77ca1eaea92f29494328c6652246732e50e2c989ed87676e333c295e0c251": {
     "query": "UPDATE eth_parameters\n            SET last_committed_block = $1, last_verified_block = $2, last_executed_block = $3\n            WHERE id = true",
     "describe": {
@@ -7619,6 +7568,27 @@
       ]
     }
   },
+  "faefd7d582000800f48eddcc1e933a876f5b0cd556f0dc45c6241eb72da97894": {
+    "query": "UPDATE prover_job_queue SET (job_status, updated_at, updated_by) = ($1, now(), 'server_clean_idle')\n            WHERE job_status = $2 and (now() - updated_at) >= interval '120 seconds' RETURNING id",
+    "describe": {
+      "columns": [
+        {
+          "ordinal": 0,
+          "name": "id",
+          "type_info": "Int4"
+        }
+      ],
+      "parameters": {
+        "Left": [
+          "Int4",
+          "Int4"
+        ]
+      },
+      "nullable": [
+        false
+      ]
+    }
+  },
   "fd16aadbd04d4a48332d59c77290a588f1a33922418b55a08c656a44ff75b8e8": {
     "query": "SELECT * FROM account_balance_updates WHERE block_number = $1",
     "describe": {
@@ -7687,6 +7657,18 @@
       ]
     }
   },
+  "fd18b441397834df097a6b1a62a773fbbb482a42a6c8f698277be4d2f219ae79": {
+    "query": "DELETE FROM aggregate_operations WHERE from_block > $1 and confirmed=false",
+    "describe": {
+      "columns": [],
+      "parameters": {
+        "Left": [
+          "Int8"
+        ]
+      },
+      "nullable": []
+    }
+  },
   "fe0256b27116eafc9a83d0f9eff341751c6022a13d0bc3625c8c8f8b9001309e": {
     "query": "\n                        DELETE FROM mint_nft_updates\n                        WHERE token_id = $1 and block_number = $2\n                        ",
     "describe": {

diff --git a/core/lib/storage/src/chain/block/mod.rs b/core/lib/storage/src/chain/block/mod.rs
@@ -658,6 +658,22 @@ impl<'a, 'c> BlockSchema<'a, 'c> {
     }
 
     /// Returns the number of last block for which proof has been confirmed on Ethereum.
+    pub async fn get_last_proven_confirmed_block(&mut self) -> QueryResult<BlockNumber> {
+        let start = Instant::now();
+        let result = OperationsSchema(self.0)
+            .get_last_block_by_aggregated_action(
+                AggregatedActionType::PublishProofBlocksOnchain,
+                Some(true),
+            )
+            .await;
+        metrics::histogram!(
+            "sql.chain.block.get_last_verified_confirmed_block",
+            start.elapsed()
+        );
+        result
+    }
+
+    /// Returns the number of last block for which executed operations has been confirmed on Ethereum .
     /// Essentially, it's number of last block for which updates were applied to the chain state.
     pub async fn get_last_verified_confirmed_block(&mut self) -> QueryResult<BlockNumber> {
         let start = Instant::now();