Skip to content

Commit

Permalink
Merge branch 'dev' into vb-remove-codeowner
Browse files Browse the repository at this point in the history
  • Loading branch information
vladbochok authored Mar 28, 2022
2 parents 13074db + 4508a4c commit ea80101
Show file tree
Hide file tree
Showing 769 changed files with 38,201 additions and 40,035 deletions.
2 changes: 2 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
*
!docker/prover/graceful-run.sh
!docker/prover/prover-entry.sh
!docker/exit-tool/exit-tool-entry.sh
!docker/exit-tool/configs
Expand All @@ -10,6 +11,7 @@
!etc/env/docker.env
!etc/env/base
!etc/tokens
!etc/web3-abi
!keys/packed
!docker/nginx/nginx.conf
!bin/
Expand Down
12 changes: 12 additions & 0 deletions .github/scripts/write-kubeconf.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

confVar="$1"
mkdir -p ~/.kube &> /dev/null

if [ -n "$confVar" ]; then
read KUBECONF <<< $(eval echo \$$confVar)
else
: ${KUBECONF?KUBECONF or envvar name must be provided}
fi

echo $KUBECONF | base64 -d > ~/.kube/config
109 changes: 109 additions & 0 deletions .github/scripts/zksync-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/bin/bash
## This script updates server-env-custom (merges the etc/env/xxx file with configmap)
# loadtest (without parameters) - combined env
# --repo [loadtest] - env from file
# --kube [loadtest] - env from configmap
# --diff [loadtest] - show the diff to be applied to the configmap
# --merge [loadtest] - writes merged configmap with merged env values
# --update-from fromfile [loadtest] - updates loadtestet env with values fromfile
# --diff-from fromfile [loadtest] - show diff for loadtestet env with values fromfile

set -e

serverEnv="server-env-custom"
repoRoot=`cd $( dirname "${BASH_SOURCE[0]}" )/../.. >/dev/null 2>&1 && pwd`

cmd="$1"
opts=""

if ( echo "--update-from --diff-from" | grep -qw "\\$cmd" ); then
fromfile="$2"
namespace="${ZKSYNC_ENV:-$3}"
else
namespace="${ZKSYNC_ENV:-$2}"
fi

[ -z $namespace ] || opts="$opts -n $namespace"

kube_env() {
if ( kubectl $opts get cm $serverEnv &> /dev/null ); then
kubectl $opts get cm $serverEnv -o go-template --template='{{ range $k,$v := .data}}{{ printf "%s=%s" $k $v }}{{"\n"}}{{end}}'
fi
}

##
repo_env() {
. $repoRoot/etc/env/$namespace.env
export $(cut -d= -f1 etc/env/$namespace.env | sed -r '/^\s*#/d')
env | sed "/^\($1\)/d"
}

cmDiff() {
kubectl $opts create cm $serverEnv --from-env-file="$tmpfile" --dry-run -o yaml | kubectl diff -f - || :
}

cleanup() {
if [ -n "$tmpfile" ]; then
rm -f "$tmpfile"
fi
}

trap cleanup EXIT

## We call ourself - carefull!
case $cmd in
--kube)
kube_env
;;
--repo)
# current env to sanitize
sanitize=$(env | cut -d= -f1 | sed ':a;N;$!ba;s/\n/\\\|/g')
repo_env "$sanitize"
;;
--diff)
tmpfile=$(mktemp -u)
bash $0 $2 > "$tmpfile"
cmDiff
;;
--merge)
tmpfile=$(mktemp -u)
bash $0 $2 > "$tmpfile"

# overwrites configmap!
outp=$(cmDiff)
if ( echo "$outp" | grep -Fq '+++' ); then
kubectl $opts create cm $serverEnv --from-env-file="$tmpfile" --dry-run -o yaml | \
kubectl apply -f -
elif [ -n "$outp" ]; then
# write a error (since no diff)
echo $outp
exit 2
fi
;;
--update-from)
tmpfile=$(mktemp -u)
{ cat $fromfile; bash $0 --kube $namespace; } | sort -u -t '=' -k 1,1 > "$tmpfile"

# overwrites configmap!
outp=$(cmDiff)
if ( echo "$outp" | grep -Fq '+++' ); then
kubectl $opts create cm $serverEnv --from-env-file="$tmpfile" --dry-run -o yaml | \
kubectl apply -f -
elif [ -n "$outp" ]; then
# write a error (since no diff)
echo $outp
exit 2
fi
;;
--diff-from)
tmpfile=$(mktemp -u)
{ cat $fromfile; bash $0 --kube $namespace; } | sort -u -t '=' -k 1,1 > "$tmpfile"
cmDiff
;;
*)
# Combine two outputs.
# We favour output from kube!
#
{ bash $0 --kube $1; bash $0 --repo $1; } | sort -u -t '=' -k 1,1
;;
esac
120 changes: 103 additions & 17 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ on:
pull_request:
push:
branches:
- staging
- trying
- staging
- trying

jobs:
lint:
runs-on: [self-hosted, CI-worker]
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -37,7 +37,7 @@ jobs:
ci_run zk lint --check
unit-tests:
runs-on: [self-hosted, CI-worker]
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2
Expand All @@ -46,7 +46,7 @@ jobs:
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
- name: start-services
run: |
docker-compose -f docker-compose-runner.yml down
Expand Down Expand Up @@ -78,7 +78,7 @@ jobs:
run: ci_run zk test server-rust

integration:
runs-on: [self-hosted, FAST]
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2
Expand All @@ -87,7 +87,7 @@ jobs:
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
- name: start-services
run: |
docker-compose -f docker-compose-runner.yml down
Expand All @@ -107,10 +107,11 @@ jobs:

- name: run-services
run: |
ci_run zk server &>server.log &
ci_run sleep 1
ci_run zk dummy-prover run &>dummy_prover.log &
ci_run zk server core &>server.log &
ci_run zk server api &>api.log &
ci_run sleep 10
ci_run zk dummy-prover run &>dummy_prover.log &
ci_run sleep 100
- name: integration-server
run: ci_run zk test i server
Expand All @@ -120,23 +121,21 @@ jobs:
ci_run zk test i api
ci_run zk test i api-docs
- name: integration-zcli
run: ci_run zk test i zcli

- name: integration-rust-sdk
run: ci_run zk test i rust-sdk

- name: integration-withdrawal-helpers
run: ci_run zk test i withdrawal-helpers

- name: Show logs
if: always()
run: |
ci_run cat server.log
ci_run cat api.log
ci_run cat dummy_prover.log
circuit-tests:
runs-on: [self-hosted, CI-worker]
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2
Expand All @@ -146,17 +145,24 @@ jobs:
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
- name: start-services
run: |
docker-compose -f docker-compose-runner.yml down
docker-compose -f docker-compose-runner.yml pull
docker-compose -f docker-compose-runner.yml up --build -d geth postgres zk
ci_run sccache --start-server
- name: init
run: |
ci_run ln -s /usr/src/keys/setup keys/setup
ci_run zk
ci_run zk run verify-keys unpack
- name: circuit-tests
run: ci_run zk test crypto-rust
run: ci_run zk test circuit 20

testkit:
runs-on: [self-hosted, CI-worker]
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2
Expand All @@ -183,3 +189,83 @@ jobs:
- name: integration-testkit
run: ci_run zk test integration testkit --offline

revert-blocks:
runs-on: [self-hosted, ci-runner]

steps:
- uses: actions/checkout@v2

- name: setup-env
run: |
echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
echo $(pwd)/bin >> $GITHUB_PATH
- name: start-services
run: |
docker-compose -f docker-compose-runner.yml down
docker-compose -f docker-compose-runner.yml pull
docker-compose -f docker-compose-runner.yml up --build -d geth postgres zk
ci_run sccache --start-server
- name: init
run: |
ci_run ln -s /usr/src/keys/setup keys/setup
ci_run zk
ci_run zk dummy-prover enable --no-redeploy
ci_run zk init
- name: restart dev-liquidity-token-watcher and dev-ticker
run: docker-compose -f docker-compose-runner.yml restart dev-liquidity-token-watcher dev-ticker

- name: run-services
run: |
ci_run zk server core &>server.log &
ci_run zk server api &>api.log &
ci_run sleep 10
- name: loadtest
run: |
ci_run cargo build --release --bin loadnext
cd $ZKSYNC_HOME
docker-compose -f docker-compose-runner.yml exec -T -e ALLOWED_PERCENT=20 -e RUST_LOG=loadnext=debug -e ZKSYNC_RPC_ADDR=http://127.0.0.2:3030 -e WEB3_URL=http://geth:8545 -e ETH_NETWORK=localhost -e MASTER_WALLET_PK=74d8b3a188f7260f67698eb44da07397a298df5427df681ef68c45b34b61f998 -e ACCOUNTS_AMOUNT=5 -e OPERATIONS_PER_ACCOUNT=5 -e MAIN_TOKEN=DAI zk ./target/release/loadnext
docker-compose -f docker-compose-runner.yml exec -T -e ZKSYNC_REST_ADDR=http://127.0.0.2:3001 zk ts-node core/tests/check-block-root-hahes.ts
- name: stop-server
run: |
ci_run killall zksync_server
- name: revert-blocks
run: |
ci_run zk f cargo build --release --bin block_revert
ci_run zk f ./target/release/block_revert --last-correct-block=2 all
- name: check-server
run: |
ci_run zk server &>server.log &
ci_run sleep 30
docker-compose -f docker-compose-runner.yml exec -T -e ZKSYNC_REST_ADDR=http://127.0.0.2:3001 zk ts-node core/tests/check-block-root-hahes.ts
- name: Show logs
if: always()
run: |
ci_run cat server.log
ci_run cat api.log
notify:
if: always() && !cancelled()
name: Notify on failures
runs-on: ubuntu-latest
needs:
[lint, unit-tests, integration, circuit-tests, testkit, revert-blocks]
steps:
- uses: technote-space/workflow-conclusion-action@v2
-
name: Notify to Mattermost (on incidents)
uses: tferreira/matterfy@releases/v1
if: ${{ env.WORKFLOW_CONCLUSION == 'failure' }}
with:
type: failure
job_name: '*CI suites for PR #${{ github.event.pull_request.number }} failed*'
icon_emoji: octocat
channel: 'ci-notifications'
url: ${{ secrets.MATTERMOST_WEBHOOK }}
Loading

0 comments on commit ea80101

Please sign in to comment.