Merge branch 'dev' into popzxc/handy-dummy-prover

Makefile

@@ -172,9 +172,6 @@ prepare-contracts:
 ci-check:
 	@ci-check.sh
 
-loadtest: confirm_action
-	@bin/loadtest.sh
-
 integration-testkit: build-contracts
 	cargo run --bin testkit --release
 	cargo run --bin exodus_test --release
@@ -206,20 +203,6 @@ circuit-tests:
 prover-tests:
 	f cargo test -p prover --release -- --ignored
 
-# Loadtest
-
-run-loadtest: confirm_action
-	@cd js/franklin_lib && yarn loadtest
-
-prepare-loadtest: confirm_action
-	@node js/loadtest/loadtest.js prepare
-
-rescue: confirm_action
-	@node js/loadtest/rescue.js
-
-deposit: confirm_action
-	@node contracts/scripts/deposit.js
-
 # Devops: main
 
 # Promote build

README.md

@@ -177,6 +177,15 @@ zksync dockerhub-push
   f cargo bench
   ```
 
+- Running  the loadtest:
+
+  ```sh
+  zksync server # Has to be run in the 1st terminal
+  zksync prover # Has to be run in the 2nd terminal
+  zksync loadtest # Has to be run in the 3rd terminal
+  ```
+
+
 ## Using Dummy Prover
 
 Using the real prover for the development can be not really handy, since it's pretty slow and resource consuming.
@@ -187,7 +196,7 @@ but acts like it does.
 To enable the dummy prover, run:
 
 ```sh
-zksync setup-dummy-prover
+zksync setup
 ```
 
 And after that you will be able to use the dummy prover instead of actual prover:

bin/loadtest

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+USAGE='Usage: zksync loadtest [-h|--help] [SCENARIO] [PATH_TO_CONFIG]
+where:
+    -h | --help     show this message
+    SCENARIO        {outgoing,execution} (outgoing by default)
+    PATH_TO_CONFIG  path to loadtest config (default is core/loadtest/src/loadtest.json)'
+
+if [ -z $ZKSYNC_ENV ];
+then 
+    echo "$USAGE"
+    exit 1
+fi
+
+SCENARIO=${1:-outgoing}
+CONFIG=${2:-"$ZKSYNC_HOME/core/loadtest/src/loadtest.json"}
+
+echo H: $SCENARIO $CONFIG
+
+case $SCENARIO in
+  outgoing|execution)
+      echo Executing $SCENARIO loadtest scenario
+    ;;
+  -h | --help)
+      echo "$USAGE"
+      exit 0
+    ;;
+  *)
+      echo Unknown loadtest scenario
+      echo "$USAGE"
+      exit 1
+    ;;
+esac
+
+
+cd $ZKSYNC_HOME
+cargo run --release --bin loadtest -- --scenario $SCENARIO $CONFIG

contracts/contracts/Franklin.sol

@@ -345,6 +345,8 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
         bytes calldata _ethWitness,
         uint32[] calldata _ethWitnessSizes
     ) external {
+        bytes memory publicData = _publicData;
+
         requireActive();
         require(_blockNumber == totalBlocksCommitted + 1, "fck11"); // only commit next block
         governance.requireActiveValidator(msg.sender);
@@ -356,12 +358,11 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
             uint64 firstOnchainOpId = totalOnchainOps;
             uint64 prevTotalCommittedPriorityRequests = totalCommittedPriorityRequests;
 
-            collectOnchainOps(_publicData, _ethWitness, _ethWitnessSizes);
+            uint64 nOnchainOpsProcessed = collectOnchainOps(publicData, _ethWitness, _ethWitnessSizes);
 
             uint64 nPriorityRequestProcessed = totalCommittedPriorityRequests - prevTotalCommittedPriorityRequests;
-            uint64 nOnchainOpsProcessed = totalOnchainOps - firstOnchainOpId;
 
-            createCommittedBlock(_blockNumber, _feeAccount, _newRoot, _publicData, firstOnchainOpId, nOnchainOpsProcessed, nPriorityRequestProcessed);
+            createCommittedBlock(_blockNumber, _feeAccount, _newRoot, publicData, firstOnchainOpId, nOnchainOpsProcessed, nPriorityRequestProcessed);
             totalBlocksCommitted++;
 
             emit BlockCommitted(_blockNumber);
@@ -410,29 +411,107 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
     /// @param _ethWitness Eth witness that was posted with commit
     /// @param _ethWitnessSizes Amount of eth witness bytes for the corresponding operation.
     function collectOnchainOps(bytes memory _publicData, bytes memory _ethWitness, uint32[] memory _ethWitnessSizes)
-        internal {
+        internal returns (uint64 processedOnchainOperations) {
         require(_publicData.length % 8 == 0, "fcs11"); // pubdata length must be a multiple of 8 because each chunk is 8 bytes
 
-        uint256 pubdataOffset = 0;
+        uint64 currentOnchainOps = 0;
+
+        uint256 pubDataPtr = 0;
+        uint256 pubDataStartPtr = 0;
+        uint256 pubDataEndPtr = 0;
+        assembly {
+            pubDataStartPtr := add(_publicData, 0x20)
+            pubDataPtr := pubDataStartPtr
+            pubDataEndPtr := add(pubDataStartPtr, mload(_publicData))
+        }
 
         uint64 ethWitnessOffset = 0;
-        uint32 processedZKSyncOperation = 0;
+        uint16 processedOperationsRequiringEthWitness = 0;
 
-        while (pubdataOffset < _publicData.length) {
-            require(processedZKSyncOperation < _ethWitnessSizes.length, "fcs13"); // eth witness data malformed
-            bytes memory zksyncOperationETHWitness = Bytes.slice(_ethWitness, ethWitnessOffset, _ethWitnessSizes[processedZKSyncOperation]);
+        while (pubDataPtr<pubDataEndPtr) {
+            uint8 opType;
+            // read operation type from public data (the first byte per each operation)
+            assembly {
+                opType := shr(0xf8, mload(pubDataPtr))
+            }
 
-            pubdataOffset += processNextOperation(
-                pubdataOffset,
-                _publicData,
-                zksyncOperationETHWitness
-            );
+            // cheap transfer operation processing
+            if (opType == uint8(Operations.OpType.Transfer)) {
+                pubDataPtr += TRANSFER_BYTES;
+            } else {
+                // other operations processing
+
+                // calculation of public data offset
+                uint256 pubdataOffset;
+                assembly {
+                    // Number of pubdata bytes processed equal to current pubData memory pointer minus pubData memory start pointer
+                    pubdataOffset := sub(pubDataPtr, pubDataStartPtr)
+                }
 
-            ethWitnessOffset += _ethWitnessSizes[processedZKSyncOperation];
-            processedZKSyncOperation++;
+                if (opType == uint8(Operations.OpType.Noop)) {
+                    pubDataPtr += NOOP_BYTES;
+                } else if (opType == uint8(Operations.OpType.TransferToNew)) {
+                    pubDataPtr += TRANSFER_TO_NEW_BYTES;
+                } else if (opType == uint8(Operations.OpType.CloseAccount)) {
+                    pubDataPtr += CLOSE_ACCOUNT_BYTES;
+                } else if (opType == uint8(Operations.OpType.Deposit)) {
+                    bytes memory pubData = Bytes.slice(_publicData, pubdataOffset + 1, DEPOSIT_BYTES - 1);
+                    onchainOps[totalOnchainOps + currentOnchainOps] = OnchainOperation(
+                        Operations.OpType.Deposit,
+                        pubData
+                    );
+                    commitNextPriorityOperation(onchainOps[totalOnchainOps + currentOnchainOps]);
+                    currentOnchainOps++;
+
+                    pubDataPtr += DEPOSIT_BYTES;
+                } else if (opType == uint8(Operations.OpType.PartialExit)) {
+                    bytes memory pubData = Bytes.slice(_publicData, pubdataOffset + 1, PARTIAL_EXIT_BYTES - 1);
+                    onchainOps[totalOnchainOps + currentOnchainOps] = OnchainOperation(
+                        Operations.OpType.PartialExit,
+                        pubData
+                    );
+                    currentOnchainOps++;
+
+                    pubDataPtr += PARTIAL_EXIT_BYTES;
+                } else if (opType == uint8(Operations.OpType.FullExit)) {
+                    bytes memory pubData = Bytes.slice(_publicData, pubdataOffset + 1, FULL_EXIT_BYTES - 1);
+                    onchainOps[totalOnchainOps + currentOnchainOps] = OnchainOperation(
+                        Operations.OpType.FullExit,
+                        pubData
+                    );
+                    commitNextPriorityOperation(onchainOps[totalOnchainOps + currentOnchainOps]);
+                    currentOnchainOps++;
+
+                    pubDataPtr += FULL_EXIT_BYTES;
+                } else if (opType == uint8(Operations.OpType.ChangePubKey)) {
+                    require(processedOperationsRequiringEthWitness < _ethWitnessSizes.length, "fcs13"); // eth witness data malformed
+                    Operations.ChangePubKey memory op = Operations.readChangePubKeyPubdata(_publicData, pubdataOffset + 1);
+
+                    if (_ethWitnessSizes[processedOperationsRequiringEthWitness] != 0) {
+                        bytes memory currentEthWitness = Bytes.slice(_ethWitness, ethWitnessOffset, _ethWitnessSizes[processedOperationsRequiringEthWitness]);
+
+                        bool valid = verifyChangePubkeySignature(currentEthWitness, op.pubKeyHash, op.nonce, op.owner);
+                        require(valid, "fpp15"); // failed to verify change pubkey hash signature
+                    } else {
+                        bool valid = keccak256(authFacts[op.owner][op.nonce]) == keccak256(op.pubKeyHash);
+                        require(valid, "fpp16"); // new pub key hash is not authenticated properly
+                    }
+
+                    ethWitnessOffset += _ethWitnessSizes[processedOperationsRequiringEthWitness];
+                    processedOperationsRequiringEthWitness++;
+
+                    pubDataPtr += CHANGE_PUBKEY_BYTES;
+                } else {
+                    revert("fpp14"); // unsupported op
+                }
+            }
         }
-        require(pubdataOffset == _publicData.length, "fcs12"); // last chunk exceeds pubdata
+        require(pubDataPtr == pubDataEndPtr, "fcs12"); // last chunk exceeds pubdata
         require(ethWitnessOffset == _ethWitness.length, "fcs14"); // _ethWitness was not used completely
+        require(processedOperationsRequiringEthWitness == _ethWitnessSizes.length, "fcs15"); // _ethWitnessSizes was not used completely
+
+        totalOnchainOps += currentOnchainOps;
+        return currentOnchainOps;
     }
 
     /// @notice Verifies ethereum signature for given message and recovers address of the signer
@@ -467,75 +546,6 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
         return recoveredAddress == _ethAddress;
     }
 
-    /// @notice On the first byte determines the type of operation, if it is an onchain operation - saves it in storage
-    /// @param _pubdataOffset Current offset in pubdata
-    /// @param _publicData Operation pubdata
-    /// @param _currentEthWitness current eth witness for operation
-    /// @return pubdata bytes processed
-    function processNextOperation(
-        uint256 _pubdataOffset,
-        bytes memory _publicData,
-        bytes memory _currentEthWitness
-    ) internal returns (uint256 _bytesProcessed) {
-        Operations.OpType opType = Operations.OpType(uint8(_publicData[_pubdataOffset]));
-
-        if (opType == Operations.OpType.Noop) return NOOP_BYTES;
-        if (opType == Operations.OpType.TransferToNew) return TRANSFER_TO_NEW_BYTES;
-        if (opType == Operations.OpType.Transfer) return TRANSFER_BYTES;
-        if (opType == Operations.OpType.CloseAccount) return CLOSE_ACCOUNT_BYTES;
-
-        if (opType == Operations.OpType.Deposit) {
-            bytes memory pubData = Bytes.slice(_publicData, _pubdataOffset + 1, DEPOSIT_BYTES - 1);
-            onchainOps[totalOnchainOps] = OnchainOperation(
-                Operations.OpType.Deposit,
-                pubData
-            );
-            commitNextPriorityOperation(onchainOps[totalOnchainOps]);
-
-            totalOnchainOps++;
-
-            return DEPOSIT_BYTES;
-        }
-
-        if (opType == Operations.OpType.PartialExit) {
-            bytes memory pubData = Bytes.slice(_publicData, _pubdataOffset + 1, PARTIAL_EXIT_BYTES - 1);
-            onchainOps[totalOnchainOps] = OnchainOperation(
-                Operations.OpType.PartialExit,
-                pubData
-            );
-            totalOnchainOps++;
-
-            return PARTIAL_EXIT_BYTES;
-        }
-
-        if (opType == Operations.OpType.FullExit) {
-            bytes memory pubData = Bytes.slice(_publicData, _pubdataOffset + 1, FULL_EXIT_BYTES - 1);
-            onchainOps[totalOnchainOps] = OnchainOperation(
-                Operations.OpType.FullExit,
-                pubData
-            );
-
-            commitNextPriorityOperation(onchainOps[totalOnchainOps]);
-
-            totalOnchainOps++;
-            return FULL_EXIT_BYTES;
-        }
-
-        if (opType == Operations.OpType.ChangePubKey) {
-            Operations.ChangePubKey memory op = Operations.readChangePubKeyPubdata(_publicData, _pubdataOffset + 1);
-            if (_currentEthWitness.length > 0) {
-                bool valid = verifyChangePubkeySignature(_currentEthWitness, op.pubKeyHash, op.nonce, op.owner);
-                require(valid, "fpp15"); // failed to verify change pubkey hash signature
-            } else {
-                bool valid = keccak256(authFacts[op.owner][op.nonce]) == keccak256(op.pubKeyHash);
-                require(valid, "fpp16"); // new pub key hash is not authenticated properly
-            }
-            return CHANGE_PUBKEY_BYTES;
-        }
-
-        revert("fpp14"); // unsupported op
-    }
-
     /// @notice Creates block commitment from its data
     /// @param _blockNumber Block number
     /// @param _feeAccount Account to collect fees
@@ -549,20 +559,41 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
         bytes32 _oldRoot,
         bytes32 _newRoot,
         bytes memory _publicData
-    ) internal pure returns (bytes32) {
+    ) internal view returns (bytes32 commitment) {
         bytes32 hash = sha256(
             abi.encodePacked(uint256(_blockNumber), uint256(_feeAccount))
         );
         hash = sha256(abi.encodePacked(hash, uint256(_oldRoot)));
         hash = sha256(abi.encodePacked(hash, uint256(_newRoot)));
-        // public data is committed with padding (TODO: check assembly and optimize to avoid copying data)
-        hash = sha256(
-            abi.encodePacked(
-                hash,
-                _publicData
+
+        /// The code below is equivalent to `commitment = sha256(abi.encodePacked(hash, _publicData))`
+
+        /// We use inline assembly instead of this concise and readable code in order to avoid copying of `_publicData` (which saves ~90 gas per transfer operation).
+
+        /// Specifically, we perform the following trick:
+        /// First, replace the first 32 bytes of `_publicData` (where normally its length is stored) with the value of `hash`.
+        /// Then, we call `sha256` precompile passing the `_publicData` pointer and the length of the concatenated byte buffer.
+        /// Finally, we put the `_publicData.length` back to its original location (to the first word of `_publicData`).
+        assembly {
+            let hashResult := mload(0x40)
+            let pubDataLen := mload(_publicData)
+            mstore(_publicData, hash)
+            // staticcall to the sha256 precompile at address 0x2
+            let success := staticcall(
+                gas,
+                0x2,
+                _publicData,
+                add(pubDataLen, 0x20),
+                hashResult,
+                0x20
             )
-        );
-        return hash;
+            mstore(_publicData, pubDataLen)
+
+            // Use "invalid" to make gas estimation work
+            switch success case 0 { invalid() }
+
+            commitment := mload(hashResult)
+        }
     }
 
     function commitNextPriorityOperation(OnchainOperation memory _onchainOp) internal {
@@ -680,7 +711,7 @@ contract Franklin is UpgradeableMaster, Storage, Config, Events {
 
         emit BlocksReverted(totalBlocksVerified, totalBlocksCommitted);
     }
-    
+
     /// @notice Checks that upgrade preparation is active and it is in lock period (period when contract will not add any new priority requests)
     function upgradePreparationLockStatus() public returns (bool) {
         return upgradePreparationActive && now < upgradePreparationActivationTime + UPGRADE_PREPARATION_LOCK_PERIOD;

contracts/contracts/test/ZKSyncUnitTest.sol

@@ -19,13 +19,12 @@ contract ZKSyncUnitTest is FranklinTest {
         storeWithdrawalAsPending(_to, _tokenId, _amount);
     }
 
-    function testProcessNextOperation(
-        uint256 _pubdataOffset,
+    function testProcessOperation(
         bytes calldata _publicData,
-        bytes calldata _currentEthWitness,
-        uint256 _expectedBytesProcessed
+        bytes calldata _ethWitness,
+        uint32[] calldata _ethWitnessSizes
     ) external {
-        require(processNextOperation(_pubdataOffset, _publicData, _currentEthWitness) == _expectedBytesProcessed, "bytes processed incorrect");
+        collectOnchainOps(_publicData, _ethWitness, _ethWitnessSizes);
     }
 
     function testVerifyEthereumSignature(bytes calldata _signature, bytes calldata _message) external pure returns (address) {