Merge branch 'dev' into popzxc-920-refine-project-structure

infrastructure/analytics/.analytics-config.json

@@ -0,0 +1,13 @@
+{
+    "defaultNetwork": "mainnet",
+    "network": {
+        "mainnet": {
+            "OPERATOR_FEE_ETH_ADDRESS": "0x2a0a81e257a2f5d6ed4f07b81dbda09f107bd027",
+            "REST_API_ADDR": "https://api.zksync.io"
+        },
+        "localhost": {
+            "OPERATOR_FEE_ETH_ADDRESS": "0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7",
+            "REST_API_ADDR": "http://localhost:3001"
+        }
+    }
+}

infrastructure/analytics/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+build/

infrastructure/analytics/README.md

@@ -0,0 +1,121 @@
+# Analytics - CLI to provide finance consuming reports
+
+## Configuration
+
+The application tries to locate a configuration file in the current working directory - `./.analytics-config.json`.
+
+The configuration file contains the default network and a list of all networks with their parameters. Each network has the following arguments:
+
+| Parameter | Description |
+| :-- | :-- |
+| `OPERATOR_FEE_ETH_ADDRESS` | Ethereum Address to be used for zkSync account to collect fees |
+| `REST_API_ADDR` | Address where the zkSync REST API is located |
+
+Also load environment variable `ETHERSCAN_API_KEY` from .env file only once
+
+## Usage
+
+```console
+$ yarn start help
+
+  Usage: analytics [options] [command]
+
+  Options:
+    -V, --version            output the version number
+    -n, --network <network>  select network
+    -h, --help               display help for command
+    
+  Commands:
+    current-balances         output worth of tokens on operator balances in zkSync as ETH and USD
+    fees [options]           output information about collected fees in the selected period
+    liquidations [options]   output total amount of ETH accrued to the SENDER_ACCOUNT as a result of token liquidations during the specified period
+    help [command]           display help for command
+
+```
+
+## Commands
+
+All output printed to stdout is strict JSON
+
+### Options/flags
+
+- --network \<network\> (Default: from config file)  
+select a network from the list of the configuration file
+- --timeFrom \<time\>  
+start of time period in format 'YYYY-MM-DDTHH:MM:SS'
+- --timeTo \<time\> (Default - current time)  
+end of time period in format 'YYYY-MM-DDTHH:MM:SS' 
+
+### Current balances reports
+
+```console
+$ yarn start current-balances 
+```
+
+Output current balance of on operator balances in zkSync as ETH and USD.
+
+The report contains information about all tokens that are supported in zkSync.
+
+### Collected fees reports
+
+```console
+$ yarn start fees --timeFrom <time> [--timeTo <time>] 
+```
+Output such information:
+- amount of ETH spent for `commit`, `verify` and `completeWithdrawals` operations in L1 and it's equivalent in USD (at present moment)
+- information about fees collected during this period in each token and their equivalent in ETH and their equivalent in ETH and USD (at present moment)
+
+### Liquidations reports
+
+```console
+$ yarn start liquidations --timeFrom <time> [--timeTo <time>]
+```
+
+Output the total amount of ETH accrued as a result of token liquidations during the specified period.
+
+## Testing
+
+```console
+$ yarn test 
+```
+
+## Usage examples
+
+```console 
+$ yarn start current-balances 
+{
+    "total": {
+        "eth": 101.23231,
+        "usd": 37018.63
+    },
+    "BAT": {
+        "amount": 10000.32,
+        "eth": 320.01024,
+        "usd": 0.88726
+    },
+    ...
+}
+$ yarn start fees --timeFrom 2020-09-15T00:00:00
+{
+    "spent by SENDER ACCOUNT":{
+        "eth": 3.567,
+        "usd": 1303.85
+    },
+    "collected fees":{
+        "total":{
+            "eth": 11.1331,
+            "usd": 4069.48
+        },
+        "BAT":{
+            "amount":1000.32,
+            "eth":32.201024,
+            "usd":0.088726
+        },
+        ...
+    }
+}
+$ yarn start liquidations --timeFrom 2020-09-14 --timeTo 2020-09-15
+{
+    "Total amount of ETH": 37.157017451243775
+}
+```

infrastructure/analytics/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "analytics",
+  "version": "0.1.0",
+  "description": "Script to generate the finance reports",
+  "main": "build/index.js",
+  "author": "Vladyslav-Bochok",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^6.1.0",
+    "ethers": "^5.0.12",
+    "fs": "^0.0.1-security",
+    "isomorphic-fetch": "^2.2.1",
+    "zksync": "link:../../js/zksync.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "start": "node build/index.js",
+    "test": "f mocha -t 1000000 -r ts-node/register tests/**/*.test.ts",
+    "fmt": "prettier --tab-width 4 --print-width 120 --parser typescript --write \"{src,tests}/*.ts\""
+  },
+  "devDependencies": {
+    "@types/chai": "^4.2.12",
+    "@types/chai-as-promised": "^7.1.3",
+    "@types/mocha": "^8.0.3",
+    "@types/mock-fs": "^4.10.0",
+    "chai": "^4.2.0",
+    "chai-as-promised": "^7.1.1",
+    "mocha": "^8.1.3",
+    "mock-fs": "^4.13.0",
+    "prettier": "^2.1.1",
+    "ts-node": "^9.0.0",
+    "typescript": "^4.0.2"
+  }
+}

infrastructure/analytics/src/commands.ts

@@ -0,0 +1,194 @@
+import "isomorphic-fetch";
+import { Network, TokensInfo } from "./types";
+import * as zksync from "zksync";
+import * as ethers from "ethers";
+import * as utils from "./utils";
+
+export async function currentBalances(network: Network, operator_address: string) {
+    const zksProvider = await zksync.getDefaultProvider(network, "HTTP");
+    const ethProvider =
+        network == "localhost" ? new ethers.providers.JsonRpcProvider() : ethers.getDefaultProvider(network);
+
+    const balances: TokensInfo = { total: { eth: 0, usd: 0 } };
+
+    const eth_price = await zksProvider.getTokenPrice("ETH");
+    const tokens = await zksProvider.getTokens();
+
+    for (const token in tokens) {
+        if (zksProvider.tokenSet.resolveTokenSymbol(token) === "MLTT" || zksync.utils.isTokenETH(token)) continue;
+
+        const tokenAddress = tokens[token].address;
+
+        const erc20contract = new ethers.Contract(
+            tokenAddress,
+            zksync.utils.IERC20_INTERFACE as ethers.ethers.ContractInterface,
+            ethProvider
+        );
+
+        const tokenPrice = await zksProvider.getTokenPrice(token);
+        const contractBalance = await erc20contract.balanceOf(operator_address);
+        const tokenAmount = Number(zksProvider.tokenSet.formatToken(token, contractBalance));
+
+        const usd_cost = tokenPrice * tokenAmount;
+        const eth_cost = usd_cost / eth_price;
+
+        balances.total.eth += eth_cost;
+        balances.total.usd += usd_cost;
+
+        balances[token] = {
+            amount: tokenAmount,
+            eth: eth_cost,
+            usd: usd_cost,
+        };
+    }
+    return balances;
+}
+
+export async function collectedFees(network: Network, providerAddress: string, timePeriod: utils.TimePeriod) {
+    const MAX_LIMIT = 100; // maximum number of blocks that the server returns in one request
+    let currentBlock = 999_999_999; // the maximum block number that we request from the server
+    let currentBlockTime = new Date();
+
+    if (!timePeriod.isValid()) throw new Error(`Error time period ${timePeriod.timeFrom} - ${timePeriod.timeTo}`);
+
+    const zksProvider = await zksync.getDefaultProvider(network);
+    const ethProvider =
+        network == "localhost" ? new ethers.providers.JsonRpcProvider() : ethers.getDefaultProvider(network);
+
+    const eth_price = await zksProvider.getTokenPrice("ETH");
+    const tokens = await zksProvider.getTokens();
+
+    const senderAccountStat = { eth: 0, usd: 0 };
+    const tokensStat: TokensInfo = { total: { eth: 0, usd: 0 } };
+
+    // structure that stores data about each token from zSync
+    // so as not to request the server many times for the same data
+    const tokensCashed = new utils.TokensCashed();
+
+    for (const token in tokens) {
+        const tokenSymbol = zksProvider.tokenSet.resolveTokenSymbol(token);
+        const todenId = zksProvider.tokenSet.resolveTokenId(token);
+        const tokenPrice = await zksProvider.getTokenPrice(token);
+
+        tokensCashed.addToken(tokenSymbol, todenId, tokenPrice);
+        tokensStat[token] = { amount: 0, eth: 0, usd: 0 };
+    }
+
+    // traverse all blocks starting from the last one
+    while (!timePeriod.less(currentBlockTime)) {
+        const blockUrl = `${providerAddress}/api/v0.1/blocks?limit=${MAX_LIMIT}&max_block=${currentBlock}`;
+        const response = await fetch(blockUrl);
+        const blocks = await response.json();
+
+        if (blocks == null) break;
+
+        for (const block of blocks) {
+            console.log(
+                `Block number: ${block.block_number}, commit Txhash: ${block.commit_tx_hash}, verify Txhash: ${block.verify_tx_hash}`
+            );
+            // skip uncommited blocks
+            if (block.committed_at == null) continue;
+
+            currentBlock = block.block_number;
+            currentBlockTime = new Date(block.committed_at);
+
+            if (timePeriod.less(currentBlockTime)) break;
+
+            const commitTransactionFee = await utils.chainTransactionFee(ethProvider, block.commit_tx_hash);
+
+            // update statistics for `commit` operation in L1
+            senderAccountStat.eth += commitTransactionFee;
+            senderAccountStat.usd += commitTransactionFee * eth_price;
+
+            // skip unverified blocks
+            if (block.verified_at == null) continue;
+
+            currentBlockTime = new Date(block.verified_at);
+            if (timePeriod.less(currentBlockTime)) break;
+
+            const verifyTransactionFee = await utils.chainTransactionFee(ethProvider, block.verify_tx_hash);
+
+            // update statistics for `verify` operation in L1
+            senderAccountStat.eth += verifyTransactionFee;
+            senderAccountStat.usd += verifyTransactionFee * eth_price;
+
+            // Each block includes many transactions
+            // Some transactions include a fee that operator collect
+            const transactionUrl = `${providerAddress}/api/v0.1/blocks/${currentBlock}/transactions`;
+            const response = await fetch(transactionUrl);
+            const transactions = await response.json();
+
+            if (transactions == null) continue;
+
+            for (const transaction of transactions) {
+                const transactionTime = new Date(transaction.created_at);
+
+                // TODO: handle fee for `CompleteWithdrawals` operation in L1
+                // wait for update API
+
+                // some transactions that are included in the block do not contain fee
+                if (utils.correctTransactionWithFee(transaction) && timePeriod.contains(transactionTime)) {
+                    const transactionFee = utils.getTransactionFee(transaction);
+
+                    const tokenID = utils.getTransactionTokenID(transaction);
+                    const tokenSymbol = tokensCashed.getTokenSymbol(tokenID);
+                    const tokenPrice = tokensCashed.getTokenPrice(tokenSymbol);
+                    const tokenAmount = Number(zksProvider.tokenSet.formatToken(tokenSymbol, transactionFee));
+
+                    //update statistics on collected tokens
+                    tokensStat[tokenSymbol].amount += tokenAmount;
+                    tokensStat[tokenSymbol].usd += tokenAmount * tokenPrice;
+                    tokensStat[tokenSymbol].eth += (tokenAmount * tokenPrice) / eth_price;
+
+                    tokensStat.total.usd += tokenAmount * tokenPrice;
+                    tokensStat.total.eth += (tokenAmount * tokenPrice) / eth_price;
+                }
+            }
+        }
+    }
+
+    return Object.assign({ "spent by SENDER ACCOUNT": senderAccountStat }, { "collected fees": tokensStat });
+}
+
+export async function collectedTokenLiquidations(
+    network: Network,
+    operatorAddress: string,
+    timePeriod: utils.TimePeriod,
+    etherscan_api_key: string
+) {
+    if (!timePeriod.isValid()) throw new Error(`Error time period ${timePeriod.timeFrom} - ${timePeriod.timeTo}`);
+
+    // To view all transactions outgoing from the account use the Etherscan provider
+    const ethProvider = new ethers.providers.EtherscanProvider(network, etherscan_api_key);
+
+    let liquidationAmount = 0;
+    let history: ethers.ethers.providers.TransactionResponse[];
+
+    // Etherscan API has limits on the number of transactions in one request
+    // so request transactions until getting an empty list
+    do {
+        const { startBlock, endBlock } = await utils.getBlockInterval(
+            ethProvider.baseUrl,
+            etherscan_api_key,
+            timePeriod
+        );
+
+        history = await ethProvider.getHistory(operatorAddress, startBlock, endBlock);
+
+        for (const transaction of history) {
+            console.log(`Tx hash: ${transaction.hash}`);
+
+            // save the current time as the last viewed transaction + 1 second
+            timePeriod.timeFrom = new Date(transaction.timestamp * 1000 + 1000);
+
+            if (transaction.from == null || transaction.from.toLocaleLowerCase() != operatorAddress) continue;
+
+            const transactionValueWei = transaction.value;
+            const transactionValue = Number(ethers.utils.formatEther(transactionValueWei));
+
+            liquidationAmount += transactionValue;
+        }
+    } while (history.length > 0 && timePeriod.isValid());
+
+    return { "Total amount of ETH": liquidationAmount };
+}