Merge branch 'dev' into breaking

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   lint:
-    runs-on: [self-hosted, CI-worker]
+    runs-on: [self-hosted, ci-runner]
 
     steps:
       - uses: actions/checkout@v2
@@ -37,7 +37,7 @@ jobs:
           ci_run zk lint --check
 
   unit-tests:
-    runs-on: [self-hosted, CI-worker]
+    runs-on: [self-hosted, ci-runner]
 
     steps:
       - uses: actions/checkout@v2
@@ -78,7 +78,7 @@ jobs:
         run: ci_run zk test server-rust
 
   integration:
-    runs-on: [self-hosted, FAST]
+    runs-on: [self-hosted, ci-runner]
 
     steps:
       - uses: actions/checkout@v2
@@ -138,7 +138,7 @@ jobs:
           ci_run cat dummy_prover.log
 
   circuit-tests:
-    runs-on: [self-hosted, CI-worker]
+    runs-on: [self-hosted, ci-runner]
 
     steps:
       - uses: actions/checkout@v2
@@ -147,6 +147,13 @@ jobs:
         run: |
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
+          
+      - name: start-services
+        run: |
+          docker-compose -f docker-compose-runner.yml down
+          docker-compose -f docker-compose-runner.yml pull
+          docker-compose -f docker-compose-runner.yml up --build -d geth postgres zk
+          ci_run sccache --start-server
 
       - name: init
         run: |
@@ -155,10 +162,10 @@ jobs:
           ci_run zk run verify-keys unpack
 
       - name: circuit-tests
-        run: ci_run zk test crypto-rust
+        run: ci_run zk test circuit 20
 
   testkit:
-    runs-on: [self-hosted, CI-worker]
+    runs-on: [self-hosted, ci-runner]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/npm.publish-packages.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   publish-zksync:
     name: Publish zksync.js
-    uses: matter-labs/zksync-dev/.github/workflows/npm.publish.yml@devops/publish-npm-zksync
+    uses: matter-labs/zksync-dev/.github/workflows/npm.publish.yml@dev
     with:
       working-directory: sdk/zksync.js
       build-command: |
@@ -19,3 +19,4 @@ jobs:
       ref: ${{ github.event.inputs.ref }}
     secrets:
       notify-webhook: ${{ secrets.MATTERMOST_WEBHOOK }}
+      token: ${{ secrets.NPM_TOKEN }}

.github/workflows/npm.publish.yml

@@ -20,7 +20,9 @@ on:
       notify-webhook:
         description: Chat notification webhook
         required: true
-
+      token:
+        description: NPM token
+        required: true
 jobs:
   local-call-publish:
     name: Build NPM package
@@ -53,7 +55,7 @@ jobs:
         run: |
           npm publish
         env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.token }}
       -
         if: failure()
         name: Notify to Mattermost (on incidents)

Cargo.toml

@@ -39,10 +39,15 @@ members = [
     "core/lib/balancer",
 
     # Test infrastructure
+    "core/tests/flamegraph_target",
     "core/tests/test_account",
     "core/tests/testkit",
     "core/tests/loadnext",
 
     # SDK section
     "sdk/zksync-rs"
 ]
+
+[profile.release.package.flamegraph_target]
+# We need both performance and debug info to analyze.
+debug = true

contracts/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    ...require('../etc/lint-config/ts.js')
+};

contracts/.prettierrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    ...require('../etc/prettier-config/ts.js')
+};

core/bin/zksync_api/src/api_server/helpers.rs

@@ -32,7 +32,7 @@ pub fn try_parse_hash(query: &str) -> Result<H256, hex::FromHexError> {
     Ok(H256::from_slice(&slice))
 }
 
-async fn depositing_from_pending_ops(
+pub(crate) async fn depositing_from_pending_ops(
     storage: &mut StorageProcessor<'_>,
     tokens: &TokenDBCache,
     pending_ops: OngoingDepositsResp,
@@ -72,7 +72,7 @@ async fn depositing_from_pending_ops(
     Ok(DepositingAccountBalances { balances })
 }
 
-async fn get_pending_ops(
+pub(crate) async fn get_pending_ops(
     core_api_client: &CoreApiClient,
     address: Address,
 ) -> Result<OngoingDepositsResp, Error> {

core/bin/zksync_api/src/api_server/rest/v02/account.rs

@@ -26,8 +26,8 @@ use super::{
     response::ApiResult,
 };
 use crate::{
-    api_server::helpers::get_depositing, api_try, core_api_client::CoreApiClient,
-    fee_ticker::PriceError, utils::token_db_cache::TokenDBCache,
+    api_try, core_api_client::CoreApiClient, fee_ticker::PriceError,
+    utils::token_db_cache::TokenDBCache,
 };
 
 /// Shared data between `api/v02/accounts` endpoints.
@@ -242,17 +242,23 @@ impl ApiAccountData {
         address: Address,
         account_id: Option<AccountId>,
     ) -> Result<AccountState, Error> {
+        // **Important**: We should get ongoing deposits *before* acquiring connection to the database.
+        // Otherwise we can starve the pool.
+        let pending_ops =
+            crate::api_server::helpers::get_pending_ops(&self.core_api_client, address).await?; // TODO: add timeout.
+
+        // Only acquire connection *after* we got info on deposits.
         let mut storage = self.pool.access_storage().await.map_err(Error::storage)?;
         let mut transaction = storage.start_transaction().await.map_err(Error::storage)?;
 
-        let depositing = get_depositing(
+        let depositing = crate::api_server::helpers::depositing_from_pending_ops(
             &mut transaction,
-            &self.core_api_client,
             &self.tokens,
-            address,
+            pending_ops,
             self.confirmations_for_eth_event,
         )
         .await?;
+
         let (committed, finalized) = if let Some(account_id) = account_id {
             let (finalized_state, committed_state) = transaction
                 .chain()

core/bin/zksync_api/src/api_server/rest/v02/transaction.rs

@@ -35,6 +35,7 @@ impl ApiTransactionData {
     }
 
     async fn tx_status(&self, tx_hash: TxHash) -> Result<Option<Receipt>, Error> {
+        // Try to find in the DB.
         let mut storage = self
             .tx_sender
             .pool
@@ -48,23 +49,31 @@ impl ApiTransactionData {
             .await
             .map_err(Error::storage)?
         {
-            Ok(Some(receipt))
-        } else if let Some(op) = self
+            return Ok(Some(receipt));
+        }
+
+        // DB lookup failed. Important to drop the connection, so it returns to the pool.
+        // Otherwise, if remote call takes too long, the connection will not be available for the whole time.
+        drop(storage);
+
+        // Try to get pending op.
+        if let Some(op) = self
             .tx_sender
             .core_api_client
             .get_unconfirmed_op(PriorityOpLookupQuery::ByAnyHash(tx_hash))
             .await
             .map_err(Error::core_api)?
         {
-            Ok(Some(Receipt::L1(L1Receipt {
+            return Ok(Some(Receipt::L1(L1Receipt {
                 status: TxInBlockStatus::Queued,
                 eth_block: EthBlockId(op.eth_block),
                 rollup_block: None,
                 id: op.serial_id,
-            })))
-        } else {
-            Ok(None)
+            })));
         }
+
+        // Nothing.
+        Ok(None)
     }
 
     async fn tx_data(&self, tx_hash: TxHash) -> Result<Option<TxData>, Error> {

core/bin/zksync_core/src/state_keeper/mod.rs

@@ -387,6 +387,8 @@ impl ZkSyncStateKeeper {
         let non_executed_op = self
             .state
             .priority_op_to_zksync_op(priority_op.data.clone());
+
+        #[allow(clippy::question_mark)] // False positive, we aren't returning `Result`.
         if self
             .pending_block
             .gas_counter

core/bin/zksync_core/src/state_keeper/root_hash_calculator/mod.rs

@@ -102,6 +102,16 @@ impl RootHashCalculator {
             job.block.0 as f64,
             "stage" => "root_hash_calculator"
         );
+        self.report_memory_stats();
+    }
+
+    fn report_memory_stats(&self) {
+        let memory_stats = self.state.tree_memory_stats();
+        metrics::histogram!("tree_memory_usage", memory_stats.allocated_total as f64, "type" => "total");
+        metrics::histogram!("tree_memory_usage", memory_stats.items as f64, "type" => "items");
+        metrics::histogram!("tree_memory_usage", memory_stats.nodes as f64, "type" => "nodes");
+        metrics::histogram!("tree_memory_usage", memory_stats.prehashed as f64, "type" => "prehashed");
+        metrics::histogram!("tree_memory_usage", memory_stats.cache as f64, "type" => "cache");
     }
 }