Mirrors branch main from private at 22afe1c (matter-labs#102)

Cargo.toml

@@ -1,16 +1,18 @@
 [workspace]
 members = [
-    # Server
-    "core/bin/zksync_core",
+    # Binaries
+    "core/bin/block_reverter",
+    "core/bin/contract-verifier",
     "core/bin/external_node",
+    "core/bin/merkle_tree_consistency_checker",
+    "core/bin/rocksdb_util",
+    "core/bin/storage_logs_dedup_migration",
     # "core/bin/system-constants-generator",
-    "core/tests/cross_external_nodes_checker",
-    # Contract verifier
-    "core/bin/contract-verifier",
-    # Verification key generator and server
     "core/bin/verification_key_generator_and_server",
-    "core/bin/storage_logs_dedup_migration",
+    "core/bin/verified_sources_fetcher",
+    "core/bin/zksync_server",
     # Libraries
+    "core/lib/zksync_core",
     "core/lib/basic_types",
     "core/lib/config",
     "core/lib/contracts",
@@ -42,6 +44,7 @@ members = [
     "core/multivm_deps/vm_m6",
 
     # Test infrastructure
+    "core/tests/cross_external_nodes_checker",
     "core/tests/loadnext",
     "core/tests/vm-benchmark",
     "core/tests/vm-benchmark/harness",

core/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [15.0.1](https://github.com/matter-labs/zksync-2-dev/compare/core-v15.0.0...core-v15.0.1) (2023-09-27)
+
+
+### Bug Fixes
+
+* **vm:** Fix divergency of hashes for l2 block ([#2662](https://github.com/matter-labs/zksync-2-dev/issues/2662)) ([fb2e2ff](https://github.com/matter-labs/zksync-2-dev/commit/fb2e2ff5a79b7bbc8a1467f174fd6d26c3d11d36))
+
+## [15.0.0](https://github.com/matter-labs/zksync-2-dev/compare/core-v9.0.0...core-v15.0.0) (2023-09-26)
+
+
+### Features
+
+* Add replication lag checker to circuit breaker component ([#2620](https://github.com/matter-labs/zksync-2-dev/issues/2620)) ([a2b3395](https://github.com/matter-labs/zksync-2-dev/commit/a2b33950d884cbca1b1f7cc7b5a43ac3e38112dd))
+* Rewrite libraries to use `vise` metrics ([#2616](https://github.com/matter-labs/zksync-2-dev/issues/2616)) ([d8cdbe9](https://github.com/matter-labs/zksync-2-dev/commit/d8cdbe9ad8ce40f55bbd8c788f3ca055a33989e6))
+
+
+### Bug Fixes
+
+* **crypto:** update compressor to pass universal setup file ([#2610](https://github.com/matter-labs/zksync-2-dev/issues/2610)) ([39ea81c](https://github.com/matter-labs/zksync-2-dev/commit/39ea81c360026d58222826d8d97bf909ff2c6326))
+* **prover-fri:** move saving to GCS behind flag ([#2627](https://github.com/matter-labs/zksync-2-dev/issues/2627)) ([ed49420](https://github.com/matter-labs/zksync-2-dev/commit/ed49420fb782856541c632e64466ff4da8e05e81))
+* **tracer:** Fixed a bug in calltracer that resulted in empty traces ([#2636](https://github.com/matter-labs/zksync-2-dev/issues/2636)) ([7983edc](https://github.com/matter-labs/zksync-2-dev/commit/7983edc9f65b1accf960c71d1d99033f3ba0e111))
+* **vm:** next block assertion for the very first miniblock after upgrade ([#2655](https://github.com/matter-labs/zksync-2-dev/issues/2655)) ([7c00107](https://github.com/matter-labs/zksync-2-dev/commit/7c00107c0eb87092c1a5bab66b004c61d6917ac9))
+
 ## [9.0.0](https://github.com/matter-labs/zksync-2-dev/compare/core-v8.7.0...core-v9.0.0) (2023-09-21)
 
 

core/bin/block_reverter/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "block_reverter"
+version = "0.1.0"
+edition = "2021"
+authors = ["The Matter Labs Team <[email protected]>"]
+homepage = "https://zksync.io/"
+repository = "https://github.com/matter-labs/zksync-era"
+license = "MIT OR Apache-2.0"
+keywords = ["blockchain", "zksync"]
+categories = ["cryptography"]
+publish = false # We don't want to publish our binaries.
+
+[dependencies]
+zksync_config = { path = "../../lib/config" }
+zksync_dal = { path = "../../lib/dal" }
+zksync_types = { path = "../../lib/types" }
+zksync_core = { path = "../../lib/zksync_core" }
+vlog = { path = "../../lib/vlog" }
+
+anyhow = "1.0"
+clap = { version = "4.2.4", features = ["derive"] }
+tokio = { version = "1", features = ["full"] }
+serde_json = "1.0"

core/bin/zksync_core/src/bin/block_reverter.rs → core/bin/block_reverter/src/main.rs

@@ -80,17 +80,17 @@ async fn main() -> anyhow::Result<()> {
     if let Some(sentry_url) = sentry_url {
         builder = builder
             .with_sentry_url(&sentry_url)
-            .expect("Invalid Sentry URL")
+            .context("Invalid Sentry URL")?
             .with_sentry_environment(environment);
     }
     let _guard = builder.build();
 
-    let eth_sender = ETHSenderConfig::from_env();
-    let db_config = DBConfig::from_env();
-    let eth_client = ETHClientConfig::from_env();
+    let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?;
+    let db_config = DBConfig::from_env().context("DBConfig::from_env()")?;
+    let eth_client = ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?;
     let default_priority_fee_per_gas =
         U256::from(eth_sender.gas_adjuster.default_priority_fee_per_gas);
-    let contracts = ContractsConfig::from_env();
+    let contracts = ContractsConfig::from_env().context("ContractsConfig::from_env()")?;
     let config = BlockReverterEthConfig::new(eth_sender, contracts, eth_client.web3_url.clone());
 
     let connection_pool = ConnectionPool::builder(DbVariant::Master)

core/bin/contract-verifier/Cargo.toml

@@ -20,7 +20,8 @@ zksync_utils = { path = "../../lib/utils" }
 prometheus_exporter = { path = "../../lib/prometheus_exporter" }
 vlog = { path = "../../lib/vlog" }
 
-tokio = { version = "1", features = ["time"] }
+anyhow = "1.0"
+tokio = { version = "1", features = ["full"] }
 futures = { version = "0.3", features = ["compat"] }
 ctrlc = { version = "3.1", features = ["termination"] }
 thiserror = "1.0"

core/bin/contract-verifier/src/main.rs

@@ -1,5 +1,6 @@
 use std::cell::RefCell;
 
+use anyhow::Context as _;
 use prometheus_exporter::PrometheusExporterConfig;
 use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig};
 use zksync_dal::ConnectionPool;
@@ -17,8 +18,8 @@ pub mod zksolc_utils;
 pub mod zkvyper_utils;
 
 async fn update_compiler_versions(connection_pool: &ConnectionPool) {
-    let mut storage = connection_pool.access_storage().await;
-    let mut transaction = storage.start_transaction().await;
+    let mut storage = connection_pool.access_storage().await.unwrap();
+    let mut transaction = storage.start_transaction().await.unwrap();
 
     let zksync_home = std::env::var("ZKSYNC_HOME").unwrap_or_else(|_| ".".into());
 
@@ -107,7 +108,7 @@ async fn update_compiler_versions(connection_pool: &ConnectionPool) {
         .await
         .unwrap();
 
-    transaction.commit().await;
+    transaction.commit().await.unwrap();
 }
 
 use structopt::StructOpt;
@@ -122,13 +123,13 @@ struct Opt {
 }
 
 #[tokio::main]
-async fn main() {
+async fn main() -> anyhow::Result<()> {
     let opt = Opt::from_args();
 
-    let verifier_config = ContractVerifierConfig::from_env();
+    let verifier_config = ContractVerifierConfig::from_env().context("ContractVerifierConfig")?;
     let prometheus_config = PrometheusConfig {
         listener_port: verifier_config.prometheus_port,
-        ..ApiConfig::from_env().prometheus
+        ..ApiConfig::from_env().context("ApiConfig")?.prometheus
     };
     let pool = ConnectionPool::singleton(DbVariant::Master)
         .build()
@@ -195,4 +196,5 @@ async fn main() {
 
     // Sleep for some time to let verifier gracefully stop.
     tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    Ok(())
 }

core/bin/contract-verifier/src/verifier.rs

@@ -3,6 +3,7 @@ use std::env;
 use std::path::Path;
 use std::time::{Duration, Instant};
 
+use anyhow::Context as _;
 use chrono::Utc;
 use ethabi::{Contract, Token};
 use lazy_static::lazy_static;
@@ -454,8 +455,8 @@ impl JobProcessor for ContractVerifier {
     const SERVICE_NAME: &'static str = "contract_verifier";
     const BACKOFF_MULTIPLIER: u64 = 1;
 
-    async fn get_next_job(&self) -> Option<(Self::JobId, Self::Job)> {
-        let mut connection = self.connection_pool.access_storage().await;
+    async fn get_next_job(&self) -> anyhow::Result<Option<(Self::JobId, Self::Job)>> {
+        let mut connection = self.connection_pool.access_storage().await.unwrap();
 
         // Time overhead for all operations except for compilation.
         const TIME_OVERHEAD: Duration = Duration::from_secs(10);
@@ -467,13 +468,13 @@ impl JobProcessor for ContractVerifier {
             .contract_verification_dal()
             .get_next_queued_verification_request(self.config.compilation_timeout() + TIME_OVERHEAD)
             .await
-            .unwrap();
+            .context("get_next_queued_verification_request()")?;
 
-        job.map(|job| (job.id, job))
+        Ok(job.map(|job| (job.id, job)))
     }
 
     async fn save_failure(&self, job_id: usize, _started_at: Instant, error: String) {
-        let mut connection = self.connection_pool.access_storage().await;
+        let mut connection = self.connection_pool.access_storage().await.unwrap();
 
         connection
             .contract_verification_dal()
@@ -492,13 +493,14 @@ impl JobProcessor for ContractVerifier {
         &self,
         job: VerificationRequest,
         started_at: Instant,
-    ) -> tokio::task::JoinHandle<()> {
+    ) -> tokio::task::JoinHandle<anyhow::Result<()>> {
         let connection_pool = self.connection_pool.clone();
         tokio::task::spawn(async move {
             tracing::info!("Started to process request with id = {}", job.id);
 
-            let config: ContractVerifierConfig = ContractVerifierConfig::from_env();
-            let mut connection = connection_pool.access_storage().await;
+            let config: ContractVerifierConfig =
+                ContractVerifierConfig::from_env().context("ContractVerifierConfig")?;
+            let mut connection = connection_pool.access_storage().await.unwrap();
 
             let job_id = job.id;
             let verification_result = Self::verify(&mut connection, job, config).await;
@@ -508,10 +510,17 @@ impl JobProcessor for ContractVerifier {
                 "api.contract_verifier.request_processing_time",
                 started_at.elapsed()
             );
+            Ok(())
         })
     }
 
-    async fn save_result(&self, _: Self::JobId, _: Instant, _: Self::JobArtifacts) {
+    async fn save_result(
+        &self,
+        _: Self::JobId,
+        _: Instant,
+        _: Self::JobArtifacts,
+    ) -> anyhow::Result<()> {
         // Do nothing
+        Ok(())
     }
 }

core/bin/external_node/Cargo.toml

@@ -11,7 +11,7 @@ categories = ["cryptography"]
 publish = false # We don't want to publish our binaries.
 
 [dependencies]
-zksync_core = { path = "../zksync_core" }
+zksync_core = { path = "../../lib/zksync_core" }
 zksync_dal = { path = "../../lib/dal" }
 zksync_config = { path = "../../lib/config" }
 zksync_storage = { path = "../../lib/storage" }
@@ -27,7 +27,7 @@ zksync_types = { path = "../../lib/types" }
 vlog = { path = "../../lib/vlog" }
 
 anyhow = "1.0"
-tokio = { version = "1", features = ["time"] }
+tokio = { version = "1", features = ["full"] }
 futures = "0.3"
 serde = { version = "1.0", features = ["derive"] }
 envy = "0.4"

core/bin/external_node/src/main.rs

@@ -362,8 +362,12 @@ async fn main() -> anyhow::Result<()> {
             L1ExecutedBatchesRevert::Allowed,
         );
 
-        let mut connection = connection_pool.access_storage().await;
-        let sealed_l1_batch_number = connection.blocks_dal().get_sealed_l1_batch_number().await;
+        let mut connection = connection_pool.access_storage().await.unwrap();
+        let sealed_l1_batch_number = connection
+            .blocks_dal()
+            .get_sealed_l1_batch_number()
+            .await
+            .unwrap();
         drop(connection);
 
         tracing::info!("Rolling back to l1 batch number {sealed_l1_batch_number}");
@@ -385,7 +389,7 @@ async fn main() -> anyhow::Result<()> {
 
     // Make sure that genesis is performed.
     perform_genesis_if_needed(
-        &mut connection_pool.access_storage().await,
+        &mut connection_pool.access_storage().await.unwrap(),
         config.remote.l2_chain_id,
         main_node_url.clone(),
     )

core/bin/merkle_tree_consistency_checker/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "merkle_tree_consistency_checker"
+version = "0.1.0"
+edition = "2021"
+authors = ["The Matter Labs Team <[email protected]>"]
+homepage = "https://zksync.io/"
+repository = "https://github.com/matter-labs/zksync-era"
+license = "MIT OR Apache-2.0"
+keywords = ["blockchain", "zksync"]
+categories = ["cryptography"]
+publish = false # We don't want to publish our binaries.
+
+[dependencies]
+zksync_config = { path = "../../lib/config" }
+zksync_merkle_tree = { path = "../../lib/merkle_tree" }
+zksync_types = { path = "../../lib/types" }
+zksync_storage = { path = "../../lib/storage" }
+vlog = { path = "../../lib/vlog" }
+
+anyhow = "1.0"
+clap = { version = "4.2.4", features = ["derive"] }
+tracing = "0.1"

core/bin/zksync_core/src/bin/merkle_tree_consistency_checker.rs → core/bin/merkle_tree_consistency_checker/src/main.rs

@@ -1,6 +1,7 @@
+use anyhow::Context as _;
 use clap::Parser;
 
-use std::time::Instant;
+use std::{path::Path, time::Instant};
 
 use zksync_config::DBConfig;
 use zksync_merkle_tree::domain::ZkSyncTree;
@@ -26,7 +27,7 @@ impl Cli {
         let db_path = &config.merkle_tree.path;
         tracing::info!("Verifying consistency of Merkle tree at {db_path}");
         let start = Instant::now();
-        let db = RocksDB::new(db_path, true);
+        let db = RocksDB::new(Path::new(db_path), true);
         let tree = ZkSyncTree::new_lightweight(db);
 
         let l1_batch_number = if let Some(number) = self.l1_batch {
@@ -46,7 +47,7 @@ impl Cli {
     }
 }
 
-fn main() {
+fn main() -> anyhow::Result<()> {
     #[allow(deprecated)] // TODO (QIT-21): Use centralized configuration approach.
     let log_format = vlog::log_format_from_env();
     #[allow(deprecated)] // TODO (QIT-21): Use centralized configuration approach.
@@ -58,11 +59,12 @@ fn main() {
     if let Some(sentry_url) = sentry_url {
         builder = builder
             .with_sentry_url(&sentry_url)
-            .expect("Invalid Sentry URL")
+            .context("Invalid Sentry URL")?
             .with_sentry_environment(environment);
     }
     let _guard = builder.build();
 
-    let db_config = DBConfig::from_env();
+    let db_config = DBConfig::from_env().context("DBConfig::from_env()")?;
     Cli::parse().run(&db_config);
+    Ok(())
 }

core/bin/rocksdb_util/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "rocksdb_util"
+version = "0.1.0"
+edition = "2021"
+authors = ["The Matter Labs Team <[email protected]>"]
+homepage = "https://zksync.io/"
+repository = "https://github.com/matter-labs/zksync-era"
+license = "MIT OR Apache-2.0"
+keywords = ["blockchain", "zksync"]
+categories = ["cryptography"]
+publish = false # We don't want to publish our binaries.
+
+[dependencies]
+zksync_config = { path = "../../lib/config" }
+zksync_storage = { path = "../../lib/storage" }
+
+anyhow = "1.0"
+clap = { version = "4.2.4", features = ["derive"] }
+
+[dev-dependencies]
+tempfile = "3.0.2"