ci: introduce benchmarks for CI

.github/actions/install-wasm-utils/action.yml

@@ -4,7 +4,7 @@ inputs:
   binaryen_version:
     description: "Binaryen Version"
     required: false
-    default: "111"
+    default: "123"
 
 runs:
   using: composite

.github/workflows/rs-bench.yml

@@ -0,0 +1,107 @@
+name: '[rs] Benchmarks'
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - 'benchmarks/**'
+      - 'rs/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - '.github/workflows/rs-bench.yml'
+      - 'examples/**'
+  push:
+    branches: [master]
+    paths:
+      - 'benchmarks/**'
+      - 'rs/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - '.github/workflows/rs-bench.yml'
+      - 'examples/**'
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  benchmark:
+    if: (github.event_name == 'push') ||
+        (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+
+      - name: Free Disk Space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install wasm-opt
+        uses: ./.github/actions/install-wasm-utils
+
+      - name: Build bench-analyzer
+        run: |
+          make build-bench-analyzer
+
+      - name: Copy current benchmarks for a diff test
+        run: |
+          cp benchmarks/bench_data.json benchmarks/bench_data_before_bench_run.json
+
+      - name: Run benchmarks
+        run: |
+          make bench
+
+      # The check is done with a threshold test
+      - name: Check current branch has actual benchmark data
+        if: github.event_name == 'pull_request'
+        run: |
+          ./target/debug/bench-analyzer --current=benchmarks/bench_data.json --other=benchmarks/bench_data_before_bench_run.json --threshold=1
+
+      # If diff check passes, compare with master baseline
+      # First copy baseline JSON from master branch
+      - name: Copy baseline benchmarks from master branch
+        if: github.event_name == 'pull_request'
+        run: |
+          git fetch origin master:refs/remotes/origin/master
+          git show origin/master:benchmarks/bench_data.json > benchmarks/baseline.json
+
+      # Now compare benchmarks and generate markdown table
+      - name: Compare Benchmarks vs Master
+        if: github.event_name == 'pull_request'
+        run: |
+          ./target/debug/bench-analyzer --current=benchmarks/bench_data.json --other=benchmarks/baseline.json --output=benchmarks/comparison.md
+
+      # Read the comparison markdown for the comment
+      - name: Read Comparison Result
+        if: github.event_name == 'pull_request'
+        id: comparison
+        run: |
+          echo 'COMPARISON_TABLE<<EOF' >> $GITHUB_OUTPUT
+          cat benchmarks/comparison.md >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+
+      # Comment the comparison table on the PR
+      - name: Comment PR with Benchmark Comparison
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`;
+
+            const commentBody = `${comparisonTable}
+
+            ---
+            <sub>🤖 This comment was automatically generated by the benchmark comparison workflow at ${{ github.sha }}.</sub>`;
+
+            // Always create a new comment
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: commentBody
+            });

Makefile

@@ -18,6 +18,9 @@ clippy:
 bench:
 	@__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo test --release --manifest-path=benchmarks/Cargo.toml
 
+build-bench-analyzer:
+	@__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo build --bin bench-analyzer
+
 build-parser:
 	@echo "Building idlparser"
 	@cargo build -p sails-idl-parser --target=wasm32-unknown-unknown --release

benchmarks/Cargo.toml

@@ -6,12 +6,18 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[[bin]]
+name = "bench-analyzer"
+path = "src/bin/bench_analyzer.rs"
+
 [dependencies]
 anyhow.workspace = true
+clap = { version = "4.0", features = ["derive"] }
 serde = { workspace = true, features = ["derive"] }
 serde-json.workspace = true
 fs2.workspace = true
 sails-rs.workspace = true
+itertools.workspace = true
 
 [build-dependencies]
 sails-rs = { workspace = true, features = ["build"] }
@@ -33,4 +39,3 @@ redirect-proxy = { path = "../examples/redirect/proxy" }
 redirect-proxy-client = { path = "../examples/redirect/proxy-client" }
 tokio = { workspace = true, features = ["rt", "macros"] }
 tempfile.workspace = true
-itertools.workspace = true

benchmarks/src/benchmarks.rs

@@ -169,7 +169,7 @@ async fn alloc_stress_bench() {
 
     for (len, gas_benches) in benches {
         crate::store_bench_data(|bench_data| {
-            bench_data.alloc.insert(len, median(gas_benches));
+            bench_data.update_alloc_bench(len, median(gas_benches));
         })
         .unwrap();
     }
@@ -195,7 +195,7 @@ async fn compute_stress_bench() {
     gas_benches.sort_unstable();
 
     crate::store_bench_data(|bench_data| {
-        bench_data.compute = median(gas_benches);
+        bench_data.update_compute_bench(median(gas_benches));
     })
     .unwrap();
 }
@@ -240,8 +240,8 @@ async fn counter_bench() {
     gas_benches_async.sort_unstable();
 
     crate::store_bench_data(|bench_data| {
-        bench_data.counter.sync_call = median(gas_benches_sync);
-        bench_data.counter.async_call = median(gas_benches_async);
+        bench_data.update_counter_bench(false, median(gas_benches_sync));
+        bench_data.update_counter_bench(true, median(gas_benches_async));
     })
     .unwrap();
 }
@@ -269,7 +269,7 @@ async fn cross_program_bench() {
     gas_benches.sort_unstable();
 
     crate::store_bench_data(|bench_data| {
-        bench_data.cross_program = median(gas_benches);
+        bench_data.update_cross_program_bench(median(gas_benches));
     })
     .unwrap();
 }
@@ -321,7 +321,7 @@ async fn redirect_bench() {
         .collect::<Vec<_>>();
 
     crate::store_bench_data(|bench_data| {
-        bench_data.redirect = median(gas_benches);
+        bench_data.update_redirect_bench(median(gas_benches));
     })
     .unwrap();
 }

benchmarks/src/bin/bench_analyzer.rs

@@ -0,0 +1,158 @@
+use anyhow::{Context, Result, anyhow};
+use benchmarks::{
+    BenchCategoryComparison, BenchCategoryComparisonReport, BenchData, BenchDataFile,
+};
+use clap::Parser;
+use std::{fs, path::PathBuf};
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+#[command(name = "bench-analyzer")]
+#[command(
+    about = "A tool for analyzing benchmark data by comparing current and previous benchmark results."
+)]
+struct Cli {
+    /// Current benchmark data file
+    #[arg(long)]
+    current: PathBuf,
+
+    /// Other benchmark data file
+    #[arg(long)]
+    other: PathBuf,
+
+    /// Threshold percentage for failure
+    #[arg(long)]
+    threshold: Option<f64>,
+
+    /// Report markdown file
+    #[arg(long)]
+    output: Option<PathBuf>,
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    analyze_benches(cli.current, cli.other, cli.output, cli.threshold)
+}
+
+fn analyze_benches(
+    current: PathBuf,
+    other: PathBuf,
+    report_output: Option<PathBuf>,
+    threshold: Option<f64>,
+) -> Result<()> {
+    // Get benches data from the provided files.
+    let (current_data, other_data) = get_bench_data(current, other)?;
+
+    // Flag to track if any benchmarks fail the threshold check.
+    let mut threshold_failed = threshold.map(|_| false);
+
+    // Create unfinished report.
+    let mut report = current_data
+        .into_iter()
+        .zip(other_data)
+        .map(
+            // Create a comparison entity for each benchmark category.
+            |((current_category, current_value), (other_category, other_value))| {
+                assert_eq!(current_category, other_category, "Categories do not match");
+
+                let comparison = BenchCategoryComparison::new(
+                    current_category,
+                    current_value,
+                    other_value,
+                    threshold,
+                );
+
+                if matches!(threshold_failed, Some(false)) && comparison.has_failed_threshold() {
+                    let _ = threshold_failed.insert(true);
+                }
+
+                comparison
+            },
+        )
+        .fold(initialize_report(), |mut report, comparison| {
+            // Add each comparison to the report.
+            add_comparison_to_report(&mut report, comparison);
+            report
+        });
+
+    // Finish the report.
+    add_report_conclusion(&mut report, threshold, threshold_failed);
+
+    // Printing the finalized report.
+    println!("{report}");
+
+    // If any benchmarks failed the threshold check, return an error.
+    if matches!(threshold_failed, Some(true)) {
+        return Err(anyhow!("Benchmark contains tests failing the threshold."));
+    }
+
+    // If an output path is provided, write the report to that file.
+    if let Some(report_output) = report_output {
+        fs::write(&report_output, &report).context("Failed to write report output")?;
+
+        println!(
+            "\nComparison table written to '{}'",
+            report_output.display()
+        );
+    }
+
+    Ok(())
+}
+
+fn get_bench_data(current: PathBuf, previous: PathBuf) -> Result<(BenchData, BenchData)> {
+    let mut current_file =
+        BenchDataFile::open(current).context("Failed to open current benchmark data file")?;
+    let mut previous_file =
+        BenchDataFile::open(previous).context("Failed to open previous benchmark data file")?;
+
+    let current_data = current_file.read_bench_data()?;
+    let previous_data = previous_file.read_bench_data()?;
+
+    Ok((current_data, previous_data))
+}
+
+fn initialize_report() -> String {
+    let mut report = String::new();
+    report.push_str("## 🔬 Benchmark Comparison\n\n");
+    report.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n");
+    report.push_str("|-----------|---------|----------|---------|----------|--------|\n");
+
+    report
+}
+
+fn add_comparison_to_report(report: &mut String, comparison: BenchCategoryComparison) {
+    let BenchCategoryComparisonReport {
+        category,
+        current,
+        other,
+        diff_sign,
+        diff,
+        diff_percent_sign,
+        diff_percent,
+        status,
+    } = comparison.into();
+    report.push_str(&format!(
+        "| {category} | {current} | {other} | {diff_sign}{diff} | {diff_percent_sign}{diff_percent:.2}% | {status} |\n",
+    ));
+}
+
+fn add_report_conclusion(
+    report: &mut String,
+    threshold: Option<f64>,
+    threshold_failed: Option<bool>,
+) {
+    match threshold_failed {
+        Some(true) => {
+            let threshold = threshold.expect("threshold is required when threshold_failed is true");
+            let err_str = format!("\n❌ Benchmark threshold {threshold:.1}% check failed!\n");
+            report.push_str(&err_str);
+        }
+        Some(false) => {
+            report.push_str("\n✅ All benchmark differences are within acceptable thresholds.");
+        }
+        None => {
+            report.push_str("\n### Legend\n- 🚀 Significant improvement (>5% reduction)\n- 👍 Minor improvement (<5% reduction)\n- ✅ No significant change\n- ⚠️ Minor regression (<5% increase)\n- ❌ Significant regression (>5% increase)\n");
+        }
+    }
+}