Benchmarks (risc0#893)

This PR will enable to run application benchmarks on every push to `main` and upload the results on our `ghpages` on its `dev` branch. Example: https://risc0.github.io/ghpages/dev/benchmarks/index.html Resolves risc0#433
Nifitsas4 · Oct 17, 2023 · 1cc087f · 1cc087f
1 parent 33f9536
commit 1cc087f
Show file tree

Hide file tree

Showing 73 changed files with 11,933 additions and 60 deletions.
diff --git a/.github/workflows/bench_applications.yml b/.github/workflows/bench_applications.yml
@@ -0,0 +1,113 @@
+name: Benchmark Applications
+
+on:
+  push:
+    branches: [ main ]
+    # Only run benchmarks on changes to following paths:
+    paths:
+      - 'risc0/**'
+      - '.github/workflows/bench_*'
+
+  # # uncomment for only for testing changes to this workflow while in a PR
+  # pull_request:
+  #   branches: [ main ]
+  #   # Only run benchmarks on changes to following paths:
+  #   paths:
+  #     - 'risc0/**'
+  #     - '.github/workflows/bench_*'
+
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  bench:
+    runs-on: [self-hosted, prod, bench, "${{ matrix.os }}", "${{ matrix.device }}"]
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: Linux
+            feature: cuda
+            device: nvidia_rtx_a5000
+          - os: macOS
+            feature: metal
+            device: apple_m2_pro
+          - os: macOS
+            feature: default
+            device: cpu
+          - os: Linux
+            feature: default
+            device: cpu
+
+    env:
+      CUDA_LIBRARY_PATH: /usr/lib/cuda
+      FEATURE: ${{ matrix.feature }}
+      RUSTC_WRAPPER: sccache
+
+    steps:
+      # This is a workaround from: https://github.com/actions/checkout/issues/590#issuecomment-970586842
+      - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/rustup
+      - uses: ./.github/actions/sccache
+        with:
+          key: ${{ matrix.os }}-${{ matrix.feature }}
+      - run: cargo run --bin cargo-risczero -- risczero install
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: cargo run --release --bin risc0-benchmark -F $FEATURE -- --out ${{ matrix.os }}-${{ matrix.device }}.csv all
+        working-directory: benchmarks
+      - run: cargo run --release --bin average -F $FEATURE -- --out ${{ matrix.os }}-${{ matrix.device }}.average.csv all
+        working-directory: benchmarks
+      - name: Save commit hash to a file
+        run: echo "${{ github.sha }}" > COMMIT_HASH.txt
+        working-directory: benchmarks
+      - name: Upload benchmark result
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmarks
+          path: |
+            benchmarks/${{ matrix.os }}-${{ matrix.device }}.csv
+            benchmarks/${{ matrix.os }}-${{ matrix.device }}.average.csv
+            benchmarks/COMMIT_HASH.txt
+
+  push_to_ghpages:
+    needs: bench
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout gh-pages repository
+        uses: actions/checkout@v3
+        with:
+          repository: risc0/ghpages
+          ref: dev
+          token: ${{ secrets.BENCHMARK_TOKEN }}
+      - name: Download benchmark
+        uses: actions/download-artifact@v2
+        with:
+          name: benchmarks
+          path: dev/benchmarks
+      - name: Push benchmark result
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "[email protected]"
+          git add dev/benchmarks/macOS-cpu.csv
+          git add dev/benchmarks/macOS-apple_m2_pro.csv
+          git add dev/benchmarks/Linux-nvidia_rtx_a5000.csv
+          git add dev/benchmarks/Linux-cpu.csv
+          git add dev/benchmarks/macOS-cpu.average.csv
+          git add dev/benchmarks/macOS-apple_m2_pro.average.csv
+          git add dev/benchmarks/Linux-nvidia_rtx_a5000.average.csv
+          git add dev/benchmarks/Linux-cpu.average.csv
+          git add dev/benchmarks/COMMIT_HASH.txt
+          if git diff --cached --exit-code; then
+            echo "No changes to commit"
+          else
+            git commit -m "Add build artifacts"
+            git remote set-url origin https://${{ secrets.BENCHMARK_TOKEN }}@github.com/risc0/ghpages.git
+            git push -f origin dev
+          fi
diff --git a/.github/workflows/bench_datasheet.yml b/.github/workflows/bench_datasheet.yml
@@ -64,6 +64,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: cargo run --release -F $FEATURE --example loop -- --json | tee ${{ matrix.os }}-${{ matrix.device }}.json
         working-directory: risc0/zkvm
+      - run: cargo run --release -- --out ../../risc0/zkvm/cycle_count.csv
+        working-directory: benchmarks/cycle_count
+        if: matrix.feature == 'metal'
       - name: Save commit hash to a file
         run: echo "${{ github.sha }}" > COMMIT_HASH.txt
         working-directory: risc0/zkvm
@@ -74,6 +77,7 @@ jobs:
           path: |
             risc0/zkvm/${{ matrix.os }}-${{ matrix.device }}.json
             risc0/zkvm/COMMIT_HASH.txt
+            risc0/zkvm/cycle_count.csv
 
   push_to_ghpages:
     needs: bench
@@ -100,6 +104,7 @@ jobs:
           git add dev/datasheet/Linux-nvidia_rtx_a5000.json
           git add dev/datasheet/Linux-cpu.json
           git add dev/datasheet/COMMIT_HASH.txt
+          git add dev/datasheet/cycle_count.csv
           if git diff --cached --exit-code; then
             echo "No changes to commit"
           else

diff --git a/.github/workflows/ghpages.yml b/.github/workflows/ghpages.yml
@@ -61,6 +61,7 @@ jobs:
           git add dev/datasheet/static/RISC_Zero_Logo.jpeg
           git add dev/datasheet/static/risczero_qrcode.png
           git add dev/datasheet/static/zkvm_examples_qrcode.png
+          git add dev/benchmarks/index.html
           git add dev/bench/index.html
           git add dev/crate-validation/index.html
           git add README.md

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -31,10 +31,12 @@ jobs:
           node-version: 18
       - run: cargo fmt --all --check
       - run: cargo fmt --all --check --manifest-path examples/Cargo.toml
+      - run: cargo fmt --all --check --manifest-path benchmarks/Cargo.toml
       - run: cargo fmt --all --check --manifest-path bonsai/Cargo.toml
       - run: cargo fmt --all --check --manifest-path bonsai/examples/governance/Cargo.toml
       - run: cargo sort --workspace --check
       - run: cargo sort --workspace --check examples
+      - run: cargo sort --workspace --check benchmarks
       - run: cargo sort --workspace --check bonsai
       - run: cargo sort --workspace --check bonsai/examples/governance
       - uses: actions/setup-python@v4
@@ -169,6 +171,24 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: cargo doc --no-deps --exclude=risc0-zkvm-methods --workspace
 
+  check-benchmarks:
+    runs-on: [self-hosted, prod, Linux, X64]
+    steps:
+      # This is a workaround from: https://github.com/actions/checkout/issues/590#issuecomment-970586842
+      - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/rustup
+      - uses: ./.github/actions/sccache
+        with:
+          key: Linux
+      - run: cargo run --bin cargo-risczero --no-default-features -- risczero install
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: cargo check
+        working-directory: benchmarks
+      - run: cargo check -p risc0-examples-cycle-counter
+        working-directory: benchmarks
+
   check_template:
     runs-on: [self-hosted, prod, cpu, "${{ matrix.os }}"]
     strategy:

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -37,10 +37,11 @@
   "rust-analyzer.showUnlinkedFileNotification": false,
   "rust-analyzer.linkedProjects": [
     "Cargo.toml",
+    "benchmarks/Cargo.toml",
     "bonsai/examples/governance/Cargo.toml",
     "examples/Cargo.toml",
     "risc0/fault/guest/Cargo.toml",
     "risc0/zkvm/methods/guest/Cargo.toml",
-    "risc0/zkvm/methods/std/Cargo.toml"
+    "risc0/zkvm/methods/std/Cargo.toml",
   ]
 }
diff --git a/Cargo.toml b/Cargo.toml
@@ -37,6 +37,7 @@ bonsai-ethereum-contracts = { version = "0.1.0", path = "bonsai/ethereum" }
 bonsai-ethereum-relay = { version = "0.1.0", default-features = false, path = "bonsai/ethereum-relay" }
 bonsai-rest-api-mock = { version = "0.1.0", default-features = false, path = "bonsai/rest-api-mock" }
 bonsai-sdk = { version = "0.5.0-alpha.1", default-features = false, path = "bonsai/sdk" }
+risc0-benchmark = { version = "0.19.0-alpha.1", default-features = false, path = "benchmarks" }
 risc0-binfmt = { version = "0.19.0-alpha.1", default-features = false, path = "risc0/binfmt" }
 risc0-build = { version = "0.19.0-alpha.1", default-features = false, path = "risc0/build" }
 risc0-build-kernel = { version = "0.19.0-alpha.1", default-features = false, path = "risc0/build_kernel" }

diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore
@@ -0,0 +1,2 @@
+*.csv
+!Cargo.lock