Merge branch 'dev' into deniallugo-refactor-eth-watcher

.github/workflows/npm.publish-packages.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   publish-zksync:
     name: Publish zksync.js
-    uses: matter-labs/zksync-dev/.github/workflows/npm.publish.yml@devops/publish-npm-zksync
+    uses: matter-labs/zksync-dev/.github/workflows/npm.publish.yml@dev
     with:
       working-directory: sdk/zksync.js
       build-command: |
@@ -19,3 +19,4 @@ jobs:
       ref: ${{ github.event.inputs.ref }}
     secrets:
       notify-webhook: ${{ secrets.MATTERMOST_WEBHOOK }}
+      token: ${{ secrets.NPM_TOKEN }}

.github/workflows/npm.publish.yml

@@ -20,7 +20,9 @@ on:
       notify-webhook:
         description: Chat notification webhook
         required: true
-
+      token:
+        description: NPM token
+        required: true
 jobs:
   local-call-publish:
     name: Build NPM package
@@ -53,7 +55,7 @@ jobs:
         run: |
           npm publish
         env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.token }}
       -
         if: failure()
         name: Notify to Mattermost (on incidents)

Cargo.toml

@@ -8,9 +8,7 @@ members = [
     "core/bin/parse_pub_data",
     "core/bin/block_revert",
     "core/bin/remove_proofs",
-    "core/bin/update_sequnce_number",
 
-    "core/bin/remove_outstanding_tx_filters",
     # Server micro-services
     "core/bin/zksync_api",
     "core/bin/zksync_core",
@@ -41,10 +39,15 @@ members = [
     "core/lib/balancer",
 
     # Test infrastructure
+    "core/tests/flamegraph_target",
     "core/tests/test_account",
     "core/tests/testkit",
     "core/tests/loadnext",
 
     # SDK section
     "sdk/zksync-rs"
 ]
+
+[profile.release.package.flamegraph_target]
+# We need both performance and debug info to analyze.
+debug = true

core/bin/zksync_core/src/state_keeper/mod.rs

@@ -432,6 +432,8 @@ impl ZkSyncStateKeeper {
         let non_executed_op = self
             .state
             .priority_op_to_zksync_op(priority_op.data.clone());
+
+        #[allow(clippy::question_mark)] // False positive, we aren't returning `Result`.
         if self
             .pending_block
             .gas_counter

core/bin/zksync_core/src/state_keeper/root_hash_calculator/mod.rs

@@ -102,6 +102,16 @@ impl RootHashCalculator {
             job.block.0 as f64,
             "stage" => "root_hash_calculator"
         );
+        self.report_memory_stats();
+    }
+
+    fn report_memory_stats(&self) {
+        let memory_stats = self.state.tree_memory_stats();
+        metrics::histogram!("tree_memory_usage", memory_stats.allocated_total as f64, "type" => "total");
+        metrics::histogram!("tree_memory_usage", memory_stats.items as f64, "type" => "items");
+        metrics::histogram!("tree_memory_usage", memory_stats.nodes as f64, "type" => "nodes");
+        metrics::histogram!("tree_memory_usage", memory_stats.prehashed as f64, "type" => "prehashed");
+        metrics::histogram!("tree_memory_usage", memory_stats.cache as f64, "type" => "cache");
     }
 }
 

core/lib/crypto/src/merkle_tree/mod.rs

@@ -8,3 +8,25 @@ mod tests;
 pub type SparseMerkleTree<T, H, HH> = parallel_smt::SparseMerkleTree<T, H, HH>;
 /// Default hasher used in the zkSync network for state hash calculations.
 pub type RescueHasher<T> = rescue_hasher::RescueHasher<T>;
+
+/// Represents the amount of RAM consumed by the tree.
+/// Only data allocated on the heap is counted.
+///
+/// Field represent the amount of memory actually requested by the system.
+/// For example, Rust `Vec`s allocate 2x previous amount on resize, so the `Vec` can
+/// request up to 2x the amount of memory than is needed to fit all the elements.
+///
+/// All the fields represent the memory amount in bytes.
+#[derive(Debug, Clone, Copy)]
+pub struct TreeMemoryUsage {
+    /// Memory used to store actual values in the tree.
+    pub items: usize,
+    /// Memory used to store hash nodes in the tree.
+    pub nodes: usize,
+    /// Memory used to store pre-calculated hashes for the "default" nodes.
+    pub prehashed: usize,
+    /// Memory used to store cache of calculated hashes for all the nodes in the tree.
+    pub cache: usize,
+    /// Total memory allocated by containers in the tree.
+    pub allocated_total: usize,
+}

core/lib/crypto/src/merkle_tree/parallel_smt.rs

@@ -1,5 +1,5 @@
 /// Sparse Merkle tree with batch updates
-use super::hasher::Hasher;
+use super::{hasher::Hasher, TreeMemoryUsage};
 use crate::{
     ff::{PrimeField, PrimeFieldRepr},
     primitives::GetBits,
@@ -196,6 +196,28 @@ where
             root: 0,
         }
     }
+
+    /// Roughly calculates the data on the RAM usage for this tree object.
+    /// See the [`TreeMemoryUsage`] doc-comments for details.
+    pub fn memory_stats(&self) -> TreeMemoryUsage {
+        use std::mem::size_of;
+
+        // For hashmaps, we use both size of keys and values.
+        let items = self.items.capacity() * (size_of::<ItemIndex>() + size_of::<T>());
+        let nodes = self.nodes.capacity() * size_of::<Node>();
+        let prehashed = self.prehashed.capacity() * size_of::<Hash>();
+        let cache =
+            self.cache.read().unwrap().capacity() * (size_of::<NodeIndex>() + size_of::<Hash>());
+        let allocated_total = items + nodes + prehashed + cache;
+
+        TreeMemoryUsage {
+            items,
+            nodes,
+            prehashed,
+            cache,
+            allocated_total,
+        }
+    }
 }
 
 impl<T, Hash, H> SparseMerkleTree<T, Hash, H>

core/lib/state/src/state.rs

@@ -1,7 +1,7 @@
 use num::BigUint;
 use std::collections::{HashMap, HashSet};
 
-use zksync_crypto::{params, params::NFT_STORAGE_ACCOUNT_ID, Fr};
+use zksync_crypto::{merkle_tree::TreeMemoryUsage, params, params::NFT_STORAGE_ACCOUNT_ID, Fr};
 use zksync_types::{
     helpers::reverse_updates,
     operations::{TransferOp, TransferToNewOp, ZkSyncOp},
@@ -111,6 +111,10 @@ impl ZkSyncState {
         }
     }
 
+    pub fn tree_memory_stats(&self) -> TreeMemoryUsage {
+        self.balance_tree.memory_stats()
+    }
+
     pub fn get_accounts(&self) -> Vec<(u32, Account)> {
         self.balance_tree
             .items