Merge branch 'dev' into lyova-983-refactor-shell-scripts-and-makefile

.dockerignore

@@ -1,6 +1,10 @@
 *
 !docker/prover/prover-entry.sh
+!docker/exit-tool/exit-tool-entry.sh
+!docker/exit-tool/configs
 !docker/keybase-secrets/entrypoint.sh
+!etc/env/dev.env.example
+!etc/tokens
 !keys/packed
 !docker/nginx/nginx.conf
 !bin/

Makefile

@@ -10,6 +10,7 @@ export DEV_TICKER_DOCKER_IMAGE ?= matterlabs/dev-ticker:latest
 export KEYBASE_DOCKER_IMAGE ?= matterlabs/keybase-secret:latest
 export CI_DOCKER_IMAGE ?= matterlabs/ci
 export FEE_SELLER_IMAGE ?=matterlabs/fee-seller:latest
+export EXIT_TOOL_IMAGE ?=matterlabs/exit-tool:latest
 
 # Getting started
 
@@ -294,3 +295,9 @@ push-image-dev-ticker: image-dev-ticker
 
 api-test:
 	@cd core/tests/ts-tests && yarn && yarn api-test
+
+image-exit-tool:
+	@docker build -t "${EXIT_TOOL_IMAGE}" -f ./docker/exit-tool/Dockerfile .
+
+push-image-exit-tool: image-exit-tool
+	@docker push "${EXIT_TOOL_IMAGE}"

bin/.setup_env

@@ -2,15 +2,18 @@
 
 pushd `dirname $0`/.. > /dev/null
 
+cd $ZKSYNC_HOME
+
 # Setup the git hooks folder.
-if ! git config --local core.hooksPath > /dev/null; then
-    git config --local core.hooksPath $ZKSYNC_HOME/.githooks/ > /dev/null
+if [ -d .git ]; then
+    if ! git config --local core.hooksPath > /dev/null; then
+        git config --local core.hooksPath $ZKSYNC_HOME/.githooks/ > /dev/null
+    fi
 fi
 
 # Setup env itself
 if [ -z "$ZKSYNC_ENV" ]
 then
-
     if [ -f etc/env/current ]; then
         export ZKSYNC_ENV=`cat etc/env/current`
     else 

bin/db-check-sqlx

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+cd core/lib/storage
+
+# Check generated sqlx data
+if ! cargo sqlx prepare --check
+then
+    # Prepare sqlx bindings
+    # We're going to do it even on CI, since it seems that this file can be invalidated after several subsequent compilations.
+    echo "Going to rerun 'sqlx prepare'"
+    cargo sqlx prepare
+fi

bin/db-insert-contract.sh_

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Force read env -- this is important, sp that we re-ready the new contract value after redeploy!!!
+ZKSYNC_ENV=
+. .setup_env
+
+psql "$DATABASE_URL" -c "INSERT INTO server_config (contract_addr, gov_contract_addr) \
+                         VALUES ('$CONTRACT_ADDR', '$GOVERNANCE_ADDR') \
+                         ON CONFLICT (id) DO UPDATE  \
+                         SET (contract_addr, gov_contract_addr) = ('$CONTRACT_ADDR', '$GOVERNANCE_ADDR')" || exit 1
+echo "successfully inserted contract address into the database"

bin/db-insert-eth-data.sh_

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Force read env -- this is important, sp that we re-ready the new contract value after redeploy!!!
+ZKSYNC_ENV=
+. .setup_env
+
+# Retrieve pending nonce from the node and obtain the value via `jq`.
+# NONCE variable will have the value like `"0x123"`.
+# Log output is redirected to the `/dev/null` to avoid garbage in the overall command output.
+NONCE=`curl \
+    -H "Accept: application/json" \
+    -H "Content-Type: application/json" \
+    -X POST \
+    --data '{"jsonrpc":"2.0","method":"eth_getTransactionCount","params":['"\"$OPERATOR_COMMIT_ETH_ADDRESS\""',"pending"],"id":1}' \
+    $WEB3_URL 2> /dev/null \
+    | jq '.result'`
+
+# Strip quotes around the nonce value. Result will be like `0x123`.
+eval NONCE=$NONCE
+
+# Convert the number from the hexadecimal form to the decimal. The result will be like `291`.
+NONCE=`printf "%d\n" $NONCE`
+
+# Insert data: nonce (obtained above), gas price limit (obtained from env), stats data (defaults to zero)
+psql "$DATABASE_URL" -c "INSERT INTO eth_parameters (nonce, gas_price_limit, commit_ops, verify_ops, withdraw_ops) \
+                         VALUES ('$NONCE', '$ETH_GAS_PRICE_DEFAULT_LIMIT', 0, 0, 0) \
+                         ON CONFLICT (id) DO UPDATE  \
+                         SET (commit_ops, verify_ops, withdraw_ops) = (0, 0, 0)" || exit 1
+
+echo "inserted Ethereum nonce ($NONCE)"
+echo "successfully initialized the Ethereum parameters table"

bin/db-insert-token.sh_

@@ -0,0 +1,10 @@
+#!/bin/bash
+# db-insert-token.sh id, address, symbol, precision
+
+# Force read env -- this is important, sp that we re-ready the new contract value after redeploy!!!
+ZKSYNC_ENV=
+. .setup_env
+
+psql "$DATABASE_URL" -c "INSERT INTO tokens \
+                         VALUES ($1, '$2', '$3', $4);" || exit 1
+echo "successfully inserted token into the database"

bin/db-setup_

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Force read env
+ZKSYNC_ENV=
+. .setup_env
+cd core/lib/storage
+
+echo DATABASE_URL=$DATABASE_URL
+diesel database setup
+diesel migration run
+
+# We don't need this file for sqlx
+rm src/schema.rs.generated
+
+# Check generated sqlx data
+if ! cargo sqlx prepare --check
+then
+    # Prepare sqlx bindings
+    # We're going to do it even on CI, since it seems that this file can be invalidated after several subsequent compilations.
+    echo "Going to rerun 'sqlx prepare'"
+    cargo sqlx prepare
+fi

bin/db-update-token-symbol.sh_

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Usage: db-update-token-symbol.sh token_address new_token_symbol
+
+. .setup_env
+
+set -e
+
+TOKEN_ADDRESS=$1
+SYMBOL=$2
+
+echo Setting token $2 symbol to $1
+psql "$DATABASE_URL" -c "UPDATE tokens \
+                         SET symbol = '$SYMBOL' \
+                         WHERE address = '$TOKEN_ADDRESS'"

bin/db-wait_

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Checks if db is up and accepting connections.
+
+. .setup_env
+
+echo $DATABASE_URL
+for i in $(seq 1 5);
+    do pg_isready -d "$DATABASE_URL" && s=0 && break || s=$? && sleep 5;
+done;
+exit $s

bin/genesis.sh_

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+. .setup_env
+
+set -e
+
+cargo run --bin zksync_server --release -- --genesis | tee genesis.log
+
+GENESIS_ROOT_NEW_VALUE=`grep GENESIS_ROOT genesis.log`
+
+export LABEL=$ZKSYNC_ENV-Genesis_gen-`date +%Y-%m-%d-%H%M%S`
+mkdir -p logs/$LABEL/
+cp ./$ENV_FILE logs/$LABEL/$ZKSYNC_ENV.bak
+cp genesis.log logs/$LABEL/
+echo $GENESIS_ROOT_NEW_VALUE
+python3 bin/replace-env-variable.py ./$ENV_FILE $GENESIS_ROOT_NEW_VALUE

bin/init

@@ -10,6 +10,7 @@ zksync yarn || true # It can fail.
 zksync plonk-setup check || zksync plonk-setup download
 zksync verify-keys unpack
 zksync db-setup
+zksync db-check-sqlx
 zksync build-dev-contracts
 zksync deploy-erc20 dev
 zksync build-contracts

core/bin/data_restore/Cargo.toml

@@ -17,6 +17,7 @@ ethabi = "12.0.0"
 web3 = "0.13.0"
 hex = "0.4"
 log = "0.4"
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.0"
 env_logger = "0.6"
 anyhow = "1.0"

core/bin/data_restore/src/data_restore_driver.rs

@@ -237,6 +237,11 @@ impl<T: Transport> DataRestoreDriver<T> {
             total_verified_blocks,
             self.tree_state.root_hash()
         );
+
+        if self.finite_mode && (total_verified_blocks == last_verified_block) {
+            // We've already completed finalizing the state, so exit immediately.
+            std::process::exit(0);
+        }
     }
 
     /// Activates states updates

core/bin/data_restore/src/events_state.rs

@@ -307,24 +307,21 @@ impl EventsState {
 
         for log in logs {
             let topic = log.topics[0];
-            assert!(log.topics.len() >= 2, "Cant get enouth topics from event");
 
             // Remove reverted committed blocks first
             if topic == reverted_topic {
-                assert_eq!(
-                    log.topics.len(),
-                    3,
-                    "Cant get enouth topics from reverted event"
-                );
-                let committed_total = U256::from(log.topics[2].as_bytes()).as_u32();
-                let mut i = 0;
-                while i != self.committed_events.len() {
-                    if self.committed_events[i].block_num > committed_total {
-                        self.committed_events.remove(i);
-                    } else {
-                        i += 1;
-                    }
-                }
+                const U256_SIZE: usize = 32;
+                // Fields in `BlocksRevert` are not `indexed`, thus they're located in `data`.
+                assert_eq!(log.data.0.len(), U256_SIZE * 2);
+                let total_verified = U256::from_big_endian(&log.data.0[..U256_SIZE]).as_u32();
+                let total_committed = U256::from_big_endian(&log.data.0[U256_SIZE..]).as_u32();
+
+                self.committed_events
+                    .retain(|bl| bl.block_num <= total_committed);
+                self.verified_events
+                    .retain(|bl| bl.block_num <= total_verified);
+
+                continue;
             }
 
             // Go into new blocks

core/bin/data_restore/src/main.rs

@@ -8,17 +8,19 @@ pub mod storage_interactor;
 pub mod tree_state;
 
 use crate::data_restore_driver::DataRestoreDriver;
+use serde::Deserialize;
 use structopt::StructOpt;
 use web3::transports::Http;
 use zksync_config::ConfigurationOptions;
 use zksync_crypto::convert::FeConvert;
 use zksync_storage::ConnectionPool;
 use zksync_types::{
     tokens::{get_genesis_token_list, Token},
-    TokenId,
+    Address, TokenId, H256,
 };
 
-const ETH_BLOCKS_STEP: u64 = 1;
+// How many blocks we will process at once.
+const ETH_BLOCKS_STEP: u64 = 10_000;
 const END_ETH_BLOCKS_OFFSET: u64 = 40;
 
 async fn add_tokens_to_db(pool: &ConnectionPool, eth_network: &str) {
@@ -61,7 +63,7 @@ struct Opt {
     genesis: bool,
 
     /// Continues data restoring
-    #[structopt(long, name = "continue")]
+    #[structopt(long = "continue", name = "continue")]
     continue_mode: bool,
 
     /// Restore data until the last verified block and exit
@@ -71,6 +73,43 @@ struct Opt {
     /// Expected tree root hash after restoring. This argument is ignored if mode is not `finite`
     #[structopt(long)]
     final_hash: Option<String>,
+
+    /// Sets the web3 API to be used to interact with the Ethereum blockchain
+    #[structopt(long = "web3", name = "web3")]
+    web3_url: Option<String>,
+
+    /// Provides a path to the configuration file for data restore
+    #[structopt(long = "config", name = "config")]
+    config_path: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct ContractsConfig {
+    eth_network: String,
+    governance_addr: Address,
+    genesis_tx_hash: H256,
+    contract_addr: Address,
+    available_block_chunk_sizes: Vec<usize>,
+}
+
+impl ContractsConfig {
+    pub fn from_file(path: &str) -> Self {
+        let content =
+            std::fs::read_to_string(path).expect("Unable to find the specified config file");
+        serde_json::from_str(&content).expect("Invalid configuration file provided")
+    }
+
+    pub fn from_env() -> Self {
+        let config_opts = ConfigurationOptions::from_env();
+
+        Self {
+            eth_network: config_opts.eth_network,
+            governance_addr: config_opts.governance_eth_addr,
+            genesis_tx_hash: config_opts.genesis_tx_hash,
+            contract_addr: config_opts.contract_eth_addr,
+            available_block_chunk_sizes: config_opts.available_block_chunk_sizes,
+        }
+    }
 }
 
 #[tokio::main]
@@ -82,11 +121,14 @@ async fn main() {
 
     let opt = Opt::from_args();
 
-    let transport = Http::new(&config_opts.web3_url).expect("failed to start web3 transport");
-    let governance_addr = config_opts.governance_eth_addr;
-    let genesis_tx_hash = config_opts.genesis_tx_hash;
-    let contract_addr = config_opts.contract_eth_addr;
-    let available_block_chunk_sizes = config_opts.available_block_chunk_sizes;
+    let web3_url = opt.web3_url.unwrap_or(config_opts.web3_url);
+
+    let transport = Http::new(&web3_url).expect("failed to start web3 transport");
+
+    let config = opt
+        .config_path
+        .map(|path| ContractsConfig::from_file(&path))
+        .unwrap_or_else(ContractsConfig::from_env);
 
     let finite_mode = opt.finite;
     let final_hash = if finite_mode {
@@ -99,11 +141,11 @@ async fn main() {
     let mut driver = DataRestoreDriver::new(
         connection_pool,
         transport,
-        governance_addr,
-        contract_addr,
+        config.governance_addr,
+        config.contract_addr,
         ETH_BLOCKS_STEP,
         END_ETH_BLOCKS_OFFSET,
-        available_block_chunk_sizes,
+        config.available_block_chunk_sizes,
         finite_mode,
         final_hash,
     );
@@ -112,9 +154,9 @@ async fn main() {
     if opt.genesis {
         // We have to load pre-defined tokens into the database before restoring state,
         // since these tokens do not have a corresponding Ethereum events.
-        add_tokens_to_db(&driver.connection_pool, &config_opts.eth_network).await;
+        add_tokens_to_db(&driver.connection_pool, &config.eth_network).await;
 
-        driver.set_genesis_state(genesis_tx_hash).await;
+        driver.set_genesis_state(config.genesis_tx_hash).await;
     }
 
     if opt.continue_mode {