Benchmark / performance tracking (FuelLabs#4564)

## Description

Adds an initial benchmarking setup to CI, where a collection of
libraries from https://github.com/FuelLabs/sway-libs are compiled,
performance metrics generated and later pushed into a new repo:
https://github.com/FuelLabs/sway-performance-data

Closes FuelLabs#318.

## Checklist

- [x] I have linked to any relevant issues.
- [x] I have commented my code, particularly in hard-to-understand
areas.
- [x] I have updated the documentation where relevant (API docs, the
reference, and the Sway book).
- [x] I have added tests that prove my fix is effective or that my
feature works.
- [x] I have added (or requested a maintainer to add) the necessary
`Breaking*` or `New Feature` labels where relevant.
- [x] I have done my best to ensure that my PR adheres to [the Fuel Labs
Code Review
Standards](https://github.com/FuelLabs/rfcs/blob/master/text/code-standards/external-contributors.md).
- [x] I have requested a review from the relevant team or maintainers.

.github/workflows/ci.yml

@@ -292,6 +292,36 @@ jobs:
       - name: Cargo Test sway-lib-std
         run: cargo test --locked --manifest-path ./test/src/sdk-harness/Cargo.toml -- --nocapture
 
+  forc-run-benchmarks:
+    runs-on: ubuntu-latest
+    permissions:
+        contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+      - uses: Swatinem/rust-cache@v2
+      - name: Install Forc
+        run: cargo install --locked --debug --path ./forc
+      - name: Run benchmarks
+        run: ./benchmark.sh
+      - name: Checkout benchmark data
+        uses: actions/checkout@v3
+        with:
+          repository: FuelLabs/sway-performance-data
+          path: performance-data
+          token: ${{ secrets.CI_ACCOUNT_PAT }}
+      - name: Prepare benchmarks data for commit
+        if: github.event_name == 'push'
+        run: ./benchmark.sh --prepare-for-commit
+      - uses: EndBug/add-and-commit@v9
+        with:
+          cwd: './performance-data'
+          message: 'Updated benchmark data'
+          default_author: github_actions
+
   forc-unit-tests:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -15,3 +15,6 @@ sway-lib-std/Forc.lock
 
 # Forc's build directory
 out
+
+# Benchmarks data output directory
+benchmarks/

benchmark.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+PREPARE_FOR_COMMIT=false
+while [[ $# -gt 0 ]]
+do
+    key="$1"
+    case $key in
+        --prepare-for-commit)
+            PREPARE_FOR_COMMIT=true
+            shift # past argument
+            ;;
+        *)
+        shift # past argument
+        ;;
+    esac
+done
+
+benchmarks_dir="benchmarks"
+if [ ! -d "$benchmarks_dir" ]; then
+    mkdir -p "$benchmarks_dir"
+fi
+
+if [ -d "$SCRIPT_DIR/target/release" ]; then
+    build_type="release"
+elif [ -d "$SCRIPT_DIR/target/debug" ]; then
+    build_type="debug"
+else
+    echo "Neither target/release nor target/debug directories found. Exiting..."
+    exit 1
+fi
+
+forc_path="$SCRIPT_DIR/target/$build_type/forc"
+
+# prepare the benchmark data for commit if requested
+if $PREPARE_FOR_COMMIT; then
+    sway_peformance_data_dir=performance-data
+    [email protected]:FuelLabs/sway-performance-data.git
+
+    if [ ! -d "$SCRIPT_DIR/$sway_peformance_data_dir" ]; then
+        echo "Directory $sway_peformance_data_dir not found. Cloning the repository..."
+        git clone "$sway_performance_data_repo_url" "$sway_peformance_data_dir"
+        echo "Repository cloned into $sway_peformance_data_dir."
+    else
+        echo "Updating sway-performance-data repository..."
+        git -C "$SCRIPT_DIR/$sway_peformance_data_dir" pull
+    fi
+
+    mkdir -p "$SCRIPT_DIR/$sway_peformance_data_dir/$GITHUB_SHA"
+    cp -r $benchmarks_dir/* "$SCRIPT_DIR/$sway_peformance_data_dir/$GITHUB_SHA"
+else
+    sway_libs_dir=sway-libs
+    sway_libs_repo_url=https://github.com/FuelLabs/sway-libs.git
+    sway_libs_branch_name="benchmarks"
+
+    if [ ! -d "$SCRIPT_DIR/$sway_libs_dir" ]; then
+        echo "Directory $sway_libs_dir not found. Cloning the repository..."
+        git clone -b "$sway_libs_branch_name" "$sway_libs_repo_url" "$sway_libs_dir"
+        echo "Repository cloned with branch $sway_libs_branch_name into $sway_libs_dir."
+    fi
+
+    libs=(
+        "fixed_point"
+        "merkle_proof"
+        "nft"
+        "ownership"
+        "reentrancy"
+        "signed_integers"
+        "storagemapvec"
+        "strings/storage_string"
+        "strings/string"
+    )
+
+    sway_apps_dir=sway-applications
+    sway_apps_repo_url=https://github.com/FuelLabs/sway-applications.git
+    sway_apps_branch_name="master"
+
+    if [ ! -d "$SCRIPT_DIR/$sway_apps_dir" ]; then
+        echo "Directory $sway_apps_dir not found. Cloning the repository..."
+        git clone -b "$sway_apps_branch_name" "$sway_apps_repo_url" "$sway_apps_dir"
+        echo "Repository cloned with branch $sway_apps_branch_name into $sway_apps_dir."
+    fi
+
+    sway_libs_revision=$(git -C $SCRIPT_DIR/$sway_libs_dir rev-parse HEAD)
+    sway_apps_revision=$(git -C $SCRIPT_DIR/$sway_apps_dir rev-parse HEAD)
+    sway_git_revision=$(git rev-parse HEAD)
+
+    for lib in "${libs[@]}"; do
+        echo "Benchmarking $lib..."
+        project_name=$(basename "$lib")
+        metrics_json_file="$benchmarks_dir/$project_name.json"
+        output=$(/usr/bin/time -f '{"elapsed": "%e", "cpu_usage": "%P", "memory": "%MKB"}' \
+            $forc_path build --path "$SCRIPT_DIR/sway-libs/libs/$lib" \
+            --metrics-outfile="$metrics_json_file" 2>&1)
+
+        exit_status=$?
+        if [ $exit_status -ne 0 ]; then
+            echo "  Failed, ignoring."
+            continue
+        fi
+
+        # filter out forc warnings by only matching on the JSON metrics data
+        json_stats=$(echo "$output" | grep -Eo '^\s*{[^}]*}')
+
+        metrics_json=$(cat "$metrics_json_file" | jq '{phases: .}')
+        merged_json=$(jq -s '.[0] * .[1]' <(echo "$metrics_json") <(echo "$json_stats"))
+        merged_json=$(jq --arg bt "$build_type" '. + {build_type: $bt}' <<< "$merged_json")
+        merged_json=$(jq --arg gr "$sway_apps_revision" '. + {sway_apps_revision: $gr}' <<< "$merged_json")
+        merged_json=$(jq --arg gr "$sway_libs_revision" '. + {sway_libs_revision: $gr}' <<< "$merged_json")
+        merged_json=$(jq --arg gr "$sway_git_revision" '. + {sway_git_revision: $gr}' <<< "$merged_json")
+
+        echo "$merged_json" > $metrics_json_file
+    done
+fi

forc-pkg/Cargo.toml

@@ -27,6 +27,7 @@ sway-core = { version = "0.39.1", path = "../sway-core" }
 sway-error = { version = "0.39.1", path = "../sway-error" }
 sway-types = { version = "0.39.1", path = "../sway-types" }
 sway-utils = { version = "0.39.1", path = "../sway-utils" }
+sysinfo = "0.29.0"
 toml = "0.5"
 tracing = "0.1"
 url = { version = "2.2", features = ["serde"] }

forc-pkg/src/manifest.rs

@@ -218,6 +218,8 @@ pub struct BuildProfile {
     #[serde(default)]
     pub time_phases: bool,
     #[serde(default)]
+    pub metrics_outfile: Option<String>,
+    #[serde(default)]
     pub include_tests: bool,
     #[serde(default)]
     pub json_abi_with_callpaths: bool,
@@ -663,6 +665,7 @@ impl BuildProfile {
             print_intermediate_asm: false,
             terse: false,
             time_phases: false,
+            metrics_outfile: None,
             include_tests: false,
             json_abi_with_callpaths: false,
             error_on_warnings: false,
@@ -679,6 +682,7 @@ impl BuildProfile {
             print_intermediate_asm: false,
             terse: false,
             time_phases: false,
+            metrics_outfile: None,
             include_tests: false,
             json_abi_with_callpaths: false,
             error_on_warnings: false,