Merge branch 'dev' into lyova-983-refactor-shell-scripts-and-makefile

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ jobs:
       run: |
         rustup update stable
         cargo sqlx --version || cargo install --version=0.1.0-beta.1 sqlx-cli
+        docker pull matterlabs/ci-integration-test:latest
 
     - name: zk
       run: pushd infrastructure/zk && yarn && yarn build
@@ -40,48 +41,24 @@ jobs:
         cargo fmt -- --check
         cargo clippy --all --tests --benches -- -D warnings
 
-    - name: init
-      run: zk init
-
-    - name: js-tests
-      run: zk test js
-
-    - name: zcli-tests
-      run: zk test i zcli --with-server
-
     - name: integration-tests
-      run: zk test i server --with-server
-
-    - name: api-tests
-      run: zk test i api --with-server
-
-    - name: data-restore
-      run: zk run data-restore check-existing
+      run: zk test integration all --in-docker
 
     - name: integration-testkit
       run: zk test i testkit
 
-    - name: test-contracts
+    - name: contracts-unit-tests
       run: zk test contracts
 
-    - name: rust-tests
-      run: zk test rust
+    - name: js-unit-tests
+      run: zk test js
 
-    - name: rust-sdk-tests
-      run: zk test i rust-sdk --with-server
+    - name: rust-unit-tests
+      run: zk test rust
 
     - name: zksync-crypto-tests
       run: pushd sdk/zksync-crypto && f cargo test --release
 
-    - name: db-tests
-      run: zk test db --reset
-
-    - name: prover-tests
-      run: zk test prover
-
-    - name: circuit-tests
-      run: zk test circuit `nproc`
-
     services:
       geth:
         image: matterlabs/geth:latest

Makefile

@@ -11,6 +11,7 @@ export KEYBASE_DOCKER_IMAGE ?= matterlabs/keybase-secret:latest
 export CI_DOCKER_IMAGE ?= matterlabs/ci
 export FEE_SELLER_IMAGE ?=matterlabs/fee-seller:latest
 export EXIT_TOOL_IMAGE ?=matterlabs/exit-tool:latest
+export CI_INTEGRATION_TEST_IMAGE ?=matterlabs/ci-integration-test:latest
 
 # Getting started
 
@@ -301,3 +302,9 @@ image-exit-tool:
 
 push-image-exit-tool: image-exit-tool
 	@docker push "${EXIT_TOOL_IMAGE}"
+
+image-ci-integration:
+	@docker build -t "${CI_INTEGRATION_TEST_IMAGE}" ./docker/integration-test/
+
+push-image-ci-integration: image-ci-integration
+	@docker push "${CI_INTEGRATION_TEST_IMAGE}"

bin/all_integration_tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+cd $ZKSYNC_HOME
+. .setup_env
+
+zksync integration-test || exit 1
+zksync api-test || exit 1
+
+# zcli test
+yarn --cwd infrastructure/zcli test || exit 1
+
+# rust-sdk test
+cargo test -p zksync --release -- --ignored --test-threads=1 || exit 1
+
+# We have to kill the server before running data-restore
+killall zksync_server
+zksync data-restore check-existing || exit 1

bin/cat_logs.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Server logs:"
+echo "============"
+cat $ZKSYNC_HOME/server.log
+echo ""
+
+echo "Prover logs:"
+echo "============"
+cat $ZKSYNC_HOME/dummy_prover.log
+echo ""
+
+# If we're calling this script, previous command failed and we want to exit with an error code 
+exit 1

bin/ci_docker_container.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+
+# Example usage:
+# `ci_docker_container.sh COMMAND_TO_RUN`, e.g. `ci_docker_container.sh "zksync integration-simple"`
+
+# In this command we launch the container `matterlabs/ci-integration-test:latest`, which includes database and geth.
+# `entrypoint.sh` prepares database and network for interaction, and also launches `dev-ticker-server`, `server` and `dummy-prover`.
+# Note that contracts must be compiled and dummy-prover should be enabled prior to the command launch, as we mount $ZKSYNC_HOME from
+# the host system inside of the container, and expect environment to be prepared for the launch.
+docker run  -v $ZKSYNC_HOME:/usr/src/zksync matterlabs/ci-integration-test:latest bash -c "/usr/local/bin/entrypoint.sh && $1"

bin/dummy-prover

@@ -44,6 +44,14 @@ function dummy_prover_status {
     exit 0
 }
 
+function enable_dummy_prover_no_redeploy {
+    # Set the `DUMMY_VERIFIER` constant value in the contract to `true`.
+    ssed -E "s/(.*constant DUMMY_VERIFIER)(.*)\;/\1 = true\;/" -i $ZKSYNC_HOME/contracts/contracts/Verifier.sol
+    echo "Enabled the Dummy Prover in the contract..."
+
+    exit 0
+}
+
 function enable_dummy_prover {
     # Set the `DUMMY_VERIFIER` constant value in the contract to `true`.
     ssed -E "s/(.*constant DUMMY_VERIFIER)(.*)\;/\1 = true\;/" -i $ZKSYNC_HOME/contracts/contracts/Verifier.sol
@@ -98,6 +106,9 @@ case $COMMAND in
   enable)
       enable_dummy_prover
     ;;
+  enable-no-redeploy)
+      enable_dummy_prover_no_redeploy
+    ;;
   disable)
       disable_dummy_prover
     ;;

docker/integration-test/Dockerfile

@@ -0,0 +1,46 @@
+FROM matterlabs/geth:latest as geth
+FROM debian:buster-slim
+
+WORKDIR /usr/src/zksync
+
+# Install required dependencies
+RUN apt-get update; apt-get install -y make bash git postgresql openssl libssl-dev gcc g++ curl libpq-dev pkg-config software-properties-common jq
+
+# Install node and yarn
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get install -y nodejs
+RUN npm install -g yarn
+
+# Install Rust and required cargo packages
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+RUN cargo install diesel_cli --no-default-features --features postgres
+RUN cargo install --version=0.1.0-beta.1 sqlx-cli
+
+# Postgresql: Enable trust authentication to allow any connections.
+RUN mkdir -p /etc/postgresql/11/main
+RUN echo "host all  all    0.0.0.0/0  trust" > /etc/postgresql/11/main/pg_hba.conf
+
+# Copy geth files
+RUN apt-get install -y ca-certificates
+RUN mkdir -p /seed/keystore
+RUN mkdir -p /var/lib/geth/data
+COPY --from=geth /seed/* /seed/
+COPY --from=geth /seed/keystore/* /seed/keystore/
+COPY --from=geth /bin/geth-entry.sh /usr/local/bin/
+RUN curl https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.9.22-c71a7e26.tar.gz -o geth-linux-amd64-1.9.22-c71a7e26.tar.gz
+RUN tar -xzf geth-linux-amd64-1.9.22-c71a7e26.tar.gz && cp ./geth-linux-amd64-1.9.22-c71a7e26/geth /usr/local/bin
+
+# Install `solc`
+RUN curl -LO https://github.com/ethereum/solidity/releases/download/v0.5.16/solc-static-linux
+RUN chmod +x solc-static-linux
+RUN mv solc-static-linux /usr/local/bin/solc
+
+# Copy the setup script
+COPY entrypoint.sh /usr/local/bin/
+
+# Setup the environment
+ENV ZKSYNC_HOME=/usr/src/zksync
+ENV PATH="${ZKSYNC_HOME}/bin:${PATH}"

docker/integration-test/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -ue
+
+# Run geth
+nohup /usr/local/bin/geth-entry.sh &>/dev/null &
+
+# Initialize database
+service postgresql restart
+
+# Prepare dummy-prover in the contract (so the redeployed version will be OK)
+zksync dummy-prover enable-no-redeploy
+
+# Initialize the stack (mostly, it's an init command with some steps skipped for docker environment)
+zksync verify-keys unpack
+zksync yarn || true # It can fail.
+zksync db-setup
+zksync build-dev-contracts
+zksync deploy-erc20 dev
+zksync build-contracts
+zksync genesis
+zksync redeploy
+
+# Compile required dependencies
+f cargo build --bin zksync_server --release
+f cargo build --bin dummy_prover --release
+f cargo build --bin dev-ticker-server --release
+
+# Launch binaries
+echo "Launching dev-ticker-server..."
+nohup f $ZKSYNC_HOME/target/release/dev-ticker-server &>/dev/null &
+sleep 1
+
+echo "Launching server..."
+nohup f $ZKSYNC_HOME/target/release/zksync_server &>$ZKSYNC_HOME/server.log &
+sleep 1
+
+echo "Launching dummy-prover..."
+nohup f $ZKSYNC_HOME/target/release/dummy_prover "dummy-prover-instance" &>$ZKSYNC_HOME/dummy_prover.log &
+
+# Wait for server to start
+sleep 10
+
+echo "Done!"