Merge branch 'dev' into deniallugo-zks-928-wrong-pagination-for-execu…

…ted-txs-in-v02
nguyenhoangduon473 · Jan 27, 2022 · 6b4ca47 · 6b4ca47
2 parents 652960e + 435cc64
commit 6b4ca47
Show file tree

Hide file tree

Showing 19 changed files with 516 additions and 75 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -185,3 +185,19 @@ jobs:
 
       - name: integration-testkit
         run: ci_run zk test integration testkit --offline
+
+  notify:
+    if: failure()
+    name: Notify on failures
+    runs-on: ubuntu-latest
+    needs: [lint, unit-tests, integration, circuit-tests, testkit]
+    steps:
+      -
+        name: Notify to Mattermost (on incidents)
+        uses: tferreira/matterfy@releases/v1
+        with:
+          type: ${{ job.status }}
+          job_name: '*CI suites at `${{ github.ref }}` did not succeed*'
+          icon_emoji: octocat
+          channel: 'ci-notifications'
+          url: ${{ secrets.MATTERMOST_WEBHOOK }}
diff --git a/.github/workflows/deploy-zksync.yml b/.github/workflows/deploy-zksync.yml
@@ -43,7 +43,6 @@ jobs:
       config_ref: ${{ github.event.inputs.config_ref }}
       deployment_id: ${{ github.event.inputs.deployment_id || steps.deployment.outputs.deployment_id }}
       environment: ${{ github.event.inputs.environment }}
-      kubeConf: ${{ steps.envMap.outputs.kubeConf }}
       hfEnv: ${{ steps.envMap.outputs.hfEnv }}
       namespace: ${{ steps.envMap.outputs.namespace }}
       runner: ${{ steps.envMap.outputs.runner }}
@@ -68,13 +67,12 @@ jobs:
             {
               ".*": {
                 "hfEnv": "${{ github.event.inputs.environment }}",
-                "kubeConf": "KUBECONF_STAGE",
                 "runner": "stage",
                 "namespace": "${{ github.event.inputs.environment }}"
               },
-              "^rinkeby$": { "kubeConf": "KUBECONF_TESTNET", "runner": "testnet" },
-              "^ropsten$": { "kubeConf": "KUBECONF_TESTNET", "runner": "testnet" },
-              "^mainnet$": { "kubeConf": "KUBECONF_MAINNET", "runner": "mainnet", "hfEnv": "prod", "namespace": "zksync" }
+              "^rinkeby$": { "runner": "testnet" },
+              "^ropsten$": { "runner": "testnet" },
+              "^mainnet$": { "runner": "mainnet", "hfEnv": "prod", "namespace": "zksync" }
             }
       -
         ## For manually invoked workflow_dispatch we create a deployment event
@@ -111,19 +109,29 @@ jobs:
     runs-on: [k8s, deployer, "${{ needs.setup.outputs.runner }}"]
     container:
       image: dysnix/kubectl:v1.20-gcloud
-
+      volumes:
+        - /var/run/secrets/tokens:/var/run/secrets/tokens
     env:
       DEPLOY_APPS: -l name=${{ github.event.inputs.server_release }} -l name=${{ github.event.inputs.prover_release }}
-      KUBECONF: ${{ secrets[needs.setup.outputs.kubeConf] }}
       # Helmfile environment name
       HFENV: ${{ needs.setup.outputs.hfEnv }}
       IMAGE_TAG: ${{ needs.setup.outputs.image_tag }}
       OUTCOMES: ${{ needs.setup.outputs.jobOutcomes }}
 
     steps:
       -
-        name: Create ~/.kube/config
-        run: mkdir -p ~/.kube && echo "$KUBECONF" | base64 -d > ~/.kube/config
+        id: kubeConf
+        run: |
+          kube_token=$(cat /var/run/secrets/tokens/gha-deployer-${{ needs.setup.outputs.runner }})
+          echo "::add-mask::$kube_token"
+          echo "::set-output name=token::$kube_token"
+      -
+        name: Generate ~/.kube/config
+        run: |
+          kubectl config set-credentials user --token=${{ steps.kubeConf.outputs.token }}
+          kubectl config set-cluster default --server=https://kubernetes.default --insecure-skip-tls-verify=true
+          kubectl config set-context matterlabs-zksync-${{ needs.setup.outputs.runner }} --user=user --cluster=default
+          kubectl config use-context matterlabs-zksync-${{ needs.setup.outputs.runner }}
       -
         name: Clone helm-infra
         uses: actions/checkout@v2
@@ -151,12 +159,12 @@ jobs:
           state: ${{ fromJSON(env.OUTCOMES)[job.status] }}
           deployment_id: ${{ needs.setup.outputs.deployment_id }}
       -
-        if: failure() || cancelled()
-        name: Notify to Mattermost (on unsuccessful deploys)
+        if: failure()
+        name: Notify to Mattermost (on incidents)
         uses: tferreira/matterfy@releases/v1
         with:
           type: ${{ job.status }}
-          job_name: '*Deployment to "${{ needs.setup.outputs.environment }}" failed*'
+          job_name: '*Deployment to `${{ needs.setup.outputs.environment }}` did not succeed*'
           icon_emoji: octocat
-          channel: 'matterlabs-alerts'
+          channel: 'ci-notifications'
           url: ${{ secrets.MATTERMOST_WEBHOOK }}
diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml
@@ -13,6 +13,8 @@ concurrency: deploy-${{ github.event.inputs.environment }}
 env:
   ZKSYNC_ENV: loadtest
   HF_ARGS: -e loadtest
+  DEPLOY_SA_NAME: gha-deployer-stage
+  DEPLOY_K8S_CONTEXT: matterlabs-zksync-stage
 
 jobs:
   update:
@@ -39,11 +41,10 @@ jobs:
     needs: [update]
     container:
       image: dysnix/kubectl:v1.20-gcloud
-
+      volumes:
+        - /var/run/secrets/tokens:/var/run/secrets/tokens
     env:
-      KUBECONF: ${{ secrets.STAGE_KUBECONF }}
       DEPLOY_APPS: -l name=sqlproxy
-
     outputs:
       image_tag: ${{ steps.set.outputs.shortRev }}
 
@@ -64,8 +65,18 @@ jobs:
           ref: master
           token: ${{ secrets.GH_TOKEN }}
       -
-        name: Create ~/.kube/config
-        run: ./.github/scripts/write-kubeconf.sh
+        id: kubeConf
+        run: |
+          kube_token=$(cat /var/run/secrets/tokens/${DEPLOY_SA_NAME})
+          echo "::add-mask::$kube_token"
+          echo "::set-output name=token::$kube_token"
+      -
+        name: Generate ~/.kube/config
+        run: |
+          kubectl config set-credentials user --token=${{ steps.kubeConf.outputs.token }}
+          kubectl config set-cluster default --server=https://kubernetes.default --insecure-skip-tls-verify=true
+          kubectl config set-context $DEPLOY_K8S_CONTEXT --user=user --cluster=default
+          kubectl config use-context $DEPLOY_K8S_CONTEXT
       -
         name: Provision SQLProxy
         working-directory: helm-infra
@@ -146,16 +157,26 @@ jobs:
     needs: [dbsetup]
     container:
       image: dysnix/kubectl:v1.20-gcloud
-    env:
-      KUBECONF: ${{ secrets.STAGE_KUBECONF }}
+      volumes:
+        - /var/run/secrets/tokens:/var/run/secrets/tokens
     steps:
       - uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
           name: deployed_contracts
       -
-        name: Create ~/.kube/config
-        run: .github/scripts/write-kubeconf.sh
+        id: kubeConf
+        run: |
+          kube_token=$(cat /var/run/secrets/tokens/${DEPLOY_SA_NAME})
+          echo "::add-mask::$kube_token"
+          echo "::set-output name=token::$kube_token"
+      -
+        name: Generate ~/.kube/config
+        run: |
+          kubectl config set-credentials user --token=${{ steps.kubeConf.outputs.token }}
+          kubectl config set-cluster default --server=https://kubernetes.default --insecure-skip-tls-verify=true
+          kubectl config set-context $DEPLOY_K8S_CONTEXT --user=user --cluster=default
+          kubectl config use-context $DEPLOY_K8S_CONTEXT
       -
         name: Update contracts in the ConfigMap
         run: .github/scripts/zksync-env.sh --update-from deployed_contracts.log
@@ -198,26 +219,45 @@ jobs:
       - name: Run loadtest
         run: cargo run --bin loadnext
 
+
   cleanup:
     name: Cleanup loadtest environment
     ## stage selects the specific cluster
     runs-on: [k8s, deployer, stage]
     needs: [dbsetup, loadtest]
     container:
       image: dysnix/kubectl:v1.20-gcloud
-    env:
-      KUBECONF: ${{ secrets.STAGE_KUBECONF }}
-
+      volumes:
+        - /var/run/secrets/tokens:/var/run/secrets/tokens
     if: always()
     steps:
       - uses: actions/checkout@v2
       - run: rm -f etc/env/${ZKSYNC_ENV}.env
       -
-        name: Create ~/.kube/config
-        run: .github/scripts/write-kubeconf.sh
+        id: kubeConf
+        run: |
+          kube_token=$(cat /var/run/secrets/tokens/${DEPLOY_SA_NAME})
+          echo "::add-mask::$kube_token"
+          echo "::set-output name=token::$kube_token"
+      -
+        name: Generate ~/.kube/config
+        run: |
+          kubectl config set-credentials user --token=${{ steps.kubeConf.outputs.token }}
+          kubectl config set-cluster default --server=https://kubernetes.default --insecure-skip-tls-verify=true
+          kubectl config set-context $DEPLOY_K8S_CONTEXT --user=user --cluster=default
+          kubectl config use-context $DEPLOY_K8S_CONTEXT
       -
         name: Scale loadtest to 0 replicas
         run: |
           kubectl -n $ZKSYNC_ENV scale deploy --replicas=0 server
           kubectl -n $ZKSYNC_ENV scale deploy --replicas=0 prover
           kubectl -n $ZKSYNC_ENV scale deploy --replicas=0 sqlproxy-gcloud-sqlproxy
+      -
+        name: Notify to Mattermost (on incidents)
+        uses: tferreira/matterfy@releases/v1
+        with:
+          type: ${{ job.status }}
+          job_name: '*Loadtest did not succeed*'
+          icon_emoji: octocat
+          channel: 'ci-notifications'
+          url: ${{ secrets.MATTERMOST_WEBHOOK }}
diff --git a/.github/workflows/npm.publish-packages.yml b/.github/workflows/npm.publish-packages.yml
@@ -0,0 +1,21 @@
+name: Publish NPM packages
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'ref or tag to publish NPM packages from'
+        default: ''
+        required: false
+
+jobs:
+  publish-zksync:
+    name: Publish zksync.js
+    uses: matter-labs/zksync-dev/.github/workflows/npm.publish.yml@devops/publish-npm-zksync
+    with:
+      working-directory: sdk/zksync.js
+      build-command: |
+        npm install
+        npm run build
+      ref: ${{ github.event.inputs.ref }}
+    secrets:
+      notify-webhook: ${{ secrets.MATTERMOST_WEBHOOK }}
diff --git a/.github/workflows/npm.publish.yml b/.github/workflows/npm.publish.yml
@@ -0,0 +1,66 @@
+name: Publish an NPM Package
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        description: Directory with the NPM package contnts
+        type: string
+        required: true
+      build-command:
+        description: Command to perform NPM pack
+        type: string
+        required: true
+      ref:
+        description: 'ref or tag to publish NPM package from'
+        type: string
+        required: false
+
+    secrets:
+      notify-webhook:
+        description: Chat notification webhook
+        required: true
+
+jobs:
+  local-call-publish:
+    name: Build NPM package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ inputs.ref }}
+      -
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.x'
+          registry-url: 'https://registry.npmjs.org'
+      -
+        name: Package info
+        id: info
+        uses: gregoranders/[email protected]
+        with:
+          path: ${{ inputs.working-directory }}/package.json
+      -
+        name: Build package
+        id: build
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          ${{ inputs.build-command }}
+      -
+        if: steps.build.outcome == 'success'
+        name: Publish to npmjs.com
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      -
+        if: failure()
+        name: Notify to Mattermost (on incidents)
+        uses: tferreira/matterfy@releases/v1
+        with:
+          type: ${{ job.status }}
+          job_name: '*Publishing `${{ steps.info.outputs.name }}` to npmjs.com did not succeed*'
+          icon_emoji: octocat
+          channel: 'ci-notifications'
+          url: ${{ secrets.notify-webhook }}
diff --git a/.github/workflows/update-config.yml b/.github/workflows/update-config.yml
@@ -17,7 +17,7 @@ defaults:
 concurrency: update-config-${{ github.event.inputs.environment }}
 
 jobs:
-  pre:
+  setup:
     runs-on: [k8s, stage]
     steps:
       - uses: actions/checkout@v2
@@ -33,51 +33,60 @@ jobs:
             {
               ".*": {
                 "hfEnv": "${{ github.event.inputs.environment }}",
-                "kubeConf": "KUBECONF_STAGE",
                 "runner": "stage",
                 "namespace": "${{ github.event.inputs.environment }}"
               },
-              "^rinkeby$": { "kubeConf": "KUBECONF_TESTNET", "runner": "testnet" },
-              "^ropsten$": { "kubeConf": "KUBECONF_TESTNET", "runner": "testnet" },
-              "^mainnet$": { "kubeConf": "KUBECONF_MAINNET", "runner": "mainnet", "hfEnv": "prod", "namespace": "zksync" }
+              "^rinkeby$": { "runner": "testnet" },
+              "^ropsten$": { "runner": "testnet" },
+              "^mainnet$": { "runner": "mainnet", "hfEnv": "prod", "namespace": "zksync" }
             }
 
     outputs:
       environment: ${{ github.event.inputs.environment }}
-      kubeConf: ${{ steps.envMap.outputs.kubeConf }}
       hfEnv: ${{ steps.envMap.outputs.hfEnv }}
       namespace: ${{ steps.envMap.outputs.namespace }}
       runner: ${{ steps.envMap.outputs.runner }}
       configRef: ${{ github.event.inputs.ref }}
 
   updateConfig:
     name: Update Config
-    needs: [pre]
-    runs-on: [k8s, deployer, "${{ needs.pre.outputs.runner }}"]
+    needs: [setup]
+    runs-on: [k8s, deployer, "${{ needs.setup.outputs.runner }}"]
     container:
       image: dysnix/kubectl:v1.20-gcloud
+      volumes:
+        - /var/run/secrets/tokens:/var/run/secrets/tokens
     env:
-      KUBECONF: ${{ secrets[needs.pre.outputs.kubeConf] }}
-      ENVFILE: ./compiled_envs/${{ needs.pre.outputs.environment }}.env
+      ENVFILE: ./compiled_envs/${{ needs.setup.outputs.environment }}.env
     steps:
       -
-        name: Create ~/.kube/config
-        run: mkdir -p ~/.kube && echo "$KUBECONF" | base64 -d > ~/.kube/config
+        id: kubeConf
+        run: |
+          kube_token=$(cat /var/run/secrets/tokens/gha-deployer-${{ needs.setup.outputs.runner }})
+          echo "::add-mask::$kube_token"
+          echo "::set-output name=token::$kube_token"
+      -
+        name: Generate ~/.kube/config
+        run: |
+          kubectl config set-credentials user --token=${{ steps.kubeConf.outputs.token }}
+          kubectl config set-cluster default --server=https://kubernetes.default --insecure-skip-tls-verify=true
+          kubectl config set-context matterlabs-zksync-${{ needs.setup.outputs.runner }} --user=user --cluster=default
+          kubectl config use-context matterlabs-zksync-${{ needs.setup.outputs.runner }}
       -
         name: Clone configs
         uses: actions/checkout@v2
         with:
           repository: matter-labs/configs
           path: configs
-          ref: ${{ needs.pre.outputs.configRef }}
+          ref: ${{ needs.setup.outputs.configRef }}
           token: ${{ secrets.GH_TOKEN }}
       -
         name: Update Server Config
         working-directory: configs
         run: |
           sudo apk --no-cache add yarn
           ./bin/config
-          ./bin/config compile ${{ needs.pre.outputs.environment }}
-          kubectl delete configmap -n ${{ needs.pre.outputs.namespace }} server-env-custom || /bin/true
-          kubectl create configmap -n ${{ needs.pre.outputs.namespace }} server-env-custom --from-env-file=${{ env.ENVFILE }}
-          # kubectl delete pod -n ${{ needs.pre.outputs.namespace }} -l app.kubernetes.io/instance=server
+          ./bin/config compile ${{ needs.setup.outputs.environment }}
+          kubectl delete configmap -n ${{ needs.setup.outputs.namespace }} server-env-custom || /bin/true
+          kubectl create configmap -n ${{ needs.setup.outputs.namespace }} server-env-custom --from-env-file=${{ env.ENVFILE }}
+          # kubectl delete pod -n ${{ needs.setup.outputs.namespace }} -l app.kubernetes.io/instance=server