From 1570270965fc0825717f8f1257f2ffa43aff1eb2 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 16:18:46 +0300 Subject: [PATCH 01/50] introduce ci job & also a converion script --- .github/workflows/bench.yml | 84 +++++++++++++++++ benchmarks/Cargo.toml | 4 + benchmarks/bench_data_converted.json | 67 ++++++++++++++ benchmarks/src/bin/convert_bench_data.rs | 109 +++++++++++++++++++++++ 4 files changed, 264 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 benchmarks/bench_data_converted.json create mode 100644 benchmarks/src/bin/convert_bench_data.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..f563e8508 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,84 @@ +# name: '[rs] Benchmarks' + +# on: +# pull_request: +# types: [labeled] +# paths: +# - 'rs/**' +# - 'Cargo.lock' +# - 'Cargo.toml' +# push: +# branches: [master] +# paths: +# - 'rs/**' +# - 'Cargo.lock' +# - 'Cargo.toml' + +# env: +# CARGO_TERM_COLOR: always +# RUST_BACKTRACE: 1 + +# jobs: +# benchmark: +# # if: contains(github.event.label.name, 'run-benchmarks') +# runs-on: ubuntu-latest +# permissions: +# contents: write +# pull-requests: write +# steps: +# # Checkout the current branch (PR branch) +# - name: Checkout PR branch +# uses: actions/checkout@v4 +# with: +# ref: ${{ github.event.pull_request.head.sha }} + +# - name: Setup Rust +# uses: actions-rust-lang/setup-rust-toolchain@v1 +# with: +# toolchain-file: rust-toolchain.toml +# cache: false + +# # Run benchmarks on PR branch +# - name: Run Benchmarks +# run: | +# make bench + +# # Checkout master branch to get baseline JSON +# - name: Checkout master branch +# uses: actions/checkout@v4 +# with: +# ref: master +# path: master-branch + +# # Copy baseline JSON from master +# - name: Copy Baseline JSON +# run: | +# cp master-branch/benchmarks/bench_data.json baseline.json + +# - name: Pwd +# run: ls benchmarks/src/bin + +# # Update the bench files formats +# - name: Convert PR Bench Data +# run: | +# cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json + +# - name: Convert Baseline Bench Data +# run: | +# cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_baseline_converted.json + +# # Compare benchmarks using github-action-benchmark +# - name: Compare Benchmarks +# uses: benchmark-action/github-action-benchmark@v1 +# with: +# tool: 'customSmallerIsBetter' +# # Path to the benchmark JSON from the PR branch +# benchmark-data-dir-path: . +# # Name of the benchmark JSON file +# output-file-path: bench_data_pr_converted.json +# # Path to the baseline JSON (from master) +# external-data-json-path: bench_data_baseline_converted.json +# # Comment on PR with results +# # comment-always: true +# # Threshold for alerting on regression (e.g., 10% worse) +# alert-threshold: '1%' \ No newline at end of file diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index bf8a60a51..eca9c4ca2 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -6,6 +6,10 @@ edition.workspace = true license.workspace = true repository.workspace = true +[[bin]] +name = "convert-bench-data" +path = "src/bin/convert_bench_data.rs" + [dependencies] anyhow.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/benchmarks/bench_data_converted.json b/benchmarks/bench_data_converted.json new file mode 100644 index 000000000..8944d596d --- /dev/null +++ b/benchmarks/bench_data_converted.json @@ -0,0 +1,67 @@ +[ + { + "name": "Compute", + "unit": "gas", + "value": 450513691329 + }, + { + "name": "alloc - 0", + "unit": "gas", + "value": 563795739 + }, + { + "name": "alloc - 12", + "unit": "gas", + "value": 567302331 + }, + { + "name": "alloc - 143", + "unit": "gas", + "value": 726670926 + }, + { + "name": "alloc - 986", + "unit": "gas", + "value": 827656693 + }, + { + "name": "alloc - 10945", + "unit": "gas", + "value": 2018421219 + }, + { + "name": "alloc - 46367", + "unit": "gas", + "value": 6403682702 + }, + { + "name": "alloc - 121392", + "unit": "gas", + "value": 16858374460 + }, + { + "name": "alloc - 317810", + "unit": "gas", + "value": 43146693797 + }, + { + "name": "counter - async_call", + "unit": "gas", + "value": 850482107 + }, + { + "name": "counter - sync_call", + "unit": "gas", + "value": 677860234 + }, + { + "name": "cross_program", + "unit": "gas", + "value": 2511573058 + }, + { + "name": "redirect", + "unit": "gas", + "value": 3612969613 + } +] \ No newline at end of file diff --git a/benchmarks/src/bin/convert_bench_data.rs b/benchmarks/src/bin/convert_bench_data.rs new file mode 100644 index 000000000..290d15156 --- /dev/null +++ b/benchmarks/src/bin/convert_bench_data.rs @@ -0,0 +1,109 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::Path; + +#[derive(Deserialize)] +struct BenchData { + compute: u64, + alloc: HashMap, + counter: HashMap, + cross_program: u64, + redirect: u64, +} + +#[derive(Serialize)] +struct BenchmarkEntry { + name: String, + unit: String, + value: u64, +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + + if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { + println!("Benchmark Data Converter"); + println!(); + println!("USAGE:"); + println!(" {} [INPUT_FILE] [OUTPUT_FILE]", args[0]); + println!(); + println!("ARGUMENTS:"); + println!(" INPUT_FILE Input JSON file (default: bench_data.json)"); + println!(" OUTPUT_FILE Output JSON file (default: bench_data_converted.json)"); + println!(); + println!("DESCRIPTION:"); + println!(" Converts benchmark data from the original nested JSON format"); + println!(" to the GitHub Actions benchmark format with name, unit, and value fields."); + return Ok(()); + } + + let input_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); + let output_file = args.get(2).unwrap_or(&"bench_data_converted.json".to_string()).clone(); + + // Read the input file + let input_path = Path::new(&input_file); + if !input_path.exists() { + eprintln!("Input file '{}' does not exist", input_file); + std::process::exit(1); + } + + let content = fs::read_to_string(input_path)?; + let bench_data: BenchData = serde_json::from_str(&content)?; + + let mut entries = Vec::new(); + + // Add compute benchmark + entries.push(BenchmarkEntry { + name: "Compute".to_string(), + unit: "gas".to_string(), + value: bench_data.compute, + }); + + // Add alloc benchmarks (sorted by key for consistent ordering) + let mut alloc_keys: Vec<_> = bench_data.alloc.keys().collect(); + alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); + + for key in alloc_keys { + let value = bench_data.alloc[key]; + entries.push(BenchmarkEntry { + name: format!("alloc - {}", key), + unit: "gas".to_string(), + value, + }); + } + + // Add counter benchmarks + for (key, value) in &bench_data.counter { + entries.push(BenchmarkEntry { + name: format!("counter - {}", key), + unit: "gas".to_string(), + value: *value, + }); + } + + // Add cross_program benchmark + entries.push(BenchmarkEntry { + name: "cross_program".to_string(), + unit: "gas".to_string(), + value: bench_data.cross_program, + }); + + // Add redirect benchmark + entries.push(BenchmarkEntry { + name: "redirect".to_string(), + unit: "gas".to_string(), + value: bench_data.redirect, + }); + + // Write the output file + let output_json = serde_json::to_string_pretty(&entries)?; + fs::write(&output_file, output_json)?; + + println!("Successfully converted '{}' to '{}'", input_file, output_file); + println!("Generated {} benchmark entries", entries.len()); + + Ok(()) +} From 38ed5179d96ec6867c17f00c0237eb69a188514d Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 19:44:13 +0300 Subject: [PATCH 02/50] add tested workflow --- .github/workflows/bench.yml | 147 ++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 74 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f563e8508..3c66cf005 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,84 +1,83 @@ -# name: '[rs] Benchmarks' +name: '[rs] Benchmarks' -# on: -# pull_request: -# types: [labeled] -# paths: -# - 'rs/**' -# - 'Cargo.lock' -# - 'Cargo.toml' -# push: -# branches: [master] -# paths: -# - 'rs/**' -# - 'Cargo.lock' -# - 'Cargo.toml' +on: + pull_request: + types: [labeled] + paths: + - 'rs/**' + - 'Cargo.lock' + - 'Cargo.toml' + push: + branches: [master] + paths: + - 'rs/**' + - 'Cargo.lock' + - 'Cargo.toml' -# env: -# CARGO_TERM_COLOR: always -# RUST_BACKTRACE: 1 +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 -# jobs: -# benchmark: -# # if: contains(github.event.label.name, 'run-benchmarks') -# runs-on: ubuntu-latest -# permissions: -# contents: write -# pull-requests: write -# steps: -# # Checkout the current branch (PR branch) -# - name: Checkout PR branch -# uses: actions/checkout@v4 -# with: -# ref: ${{ github.event.pull_request.head.sha }} +jobs: + benchmark: + # if: contains(github.event.label.name, 'run-benchmarks') + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + # Checkout the current branch (PR branch) + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} -# - name: Setup Rust -# uses: actions-rust-lang/setup-rust-toolchain@v1 -# with: -# toolchain-file: rust-toolchain.toml -# cache: false + # - name: Setup Rust + # uses: actions-rust-lang/setup-rust-toolchain@v1 + # with: + # toolchain-file: rust-toolchain.toml + # cache: false -# # Run benchmarks on PR branch -# - name: Run Benchmarks -# run: | -# make bench + # Run benchmarks on PR branch + - name: Run Benchmarks + run: | + make bench -# # Checkout master branch to get baseline JSON -# - name: Checkout master branch -# uses: actions/checkout@v4 -# with: -# ref: master -# path: master-branch + # Checkout master branch to get baseline JSON + - name: Checkout master branch + uses: actions/checkout@v4 + with: + ref: master + path: master-branch -# # Copy baseline JSON from master -# - name: Copy Baseline JSON -# run: | -# cp master-branch/benchmarks/bench_data.json baseline.json + # Copy baseline JSON from master + - name: Copy Baseline JSON + run: | + cp master-branch/benchmarks/bench_data.json baseline.json -# - name: Pwd -# run: ls benchmarks/src/bin + # Update the bench files formats + - name: Convert PR Bench Data + run: | + cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json -# # Update the bench files formats -# - name: Convert PR Bench Data -# run: | -# cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json + - name: Convert Baseline Bench Data + run: | + cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json -# - name: Convert Baseline Bench Data -# run: | -# cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_baseline_converted.json - -# # Compare benchmarks using github-action-benchmark -# - name: Compare Benchmarks -# uses: benchmark-action/github-action-benchmark@v1 -# with: -# tool: 'customSmallerIsBetter' -# # Path to the benchmark JSON from the PR branch -# benchmark-data-dir-path: . -# # Name of the benchmark JSON file -# output-file-path: bench_data_pr_converted.json -# # Path to the baseline JSON (from master) -# external-data-json-path: bench_data_baseline_converted.json -# # Comment on PR with results -# # comment-always: true -# # Threshold for alerting on regression (e.g., 10% worse) -# alert-threshold: '1%' \ No newline at end of file + # Compare benchmarks using github-action-benchmark + - name: Compare Benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'customSmallerIsBetter' + # Path to the benchmark JSON from the PR branch + benchmark-data-dir-path: . + # Name of the benchmark JSON file + output-file-path: bench_data_pr_converted.json + # Path to the baseline JSON (from master) + external-data-json-path: bench_data_baseline_converted.json + # Comment on PR with results + comment-always: true + # Threshold for alerting on regression (e.g., 10% worse) + alert-threshold: '0%' + auto-push: false + github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 0b520c4a41ddbaa4e2310ff4bcc1907c18f68d0a Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 19:46:21 +0300 Subject: [PATCH 03/50] remove file --- benchmarks/bench_data_converted.json | 67 ---------------------------- 1 file changed, 67 deletions(-) delete mode 100644 benchmarks/bench_data_converted.json diff --git a/benchmarks/bench_data_converted.json b/benchmarks/bench_data_converted.json deleted file mode 100644 index 8944d596d..000000000 --- a/benchmarks/bench_data_converted.json +++ /dev/null @@ -1,67 +0,0 @@ -[ - { - "name": "Compute", - "unit": "gas", - "value": 450513691329 - }, - { - "name": "alloc - 0", - "unit": "gas", - "value": 563795739 - }, - { - "name": "alloc - 12", - "unit": "gas", - "value": 567302331 - }, - { - "name": "alloc - 143", - "unit": "gas", - "value": 726670926 - }, - { - "name": "alloc - 986", - "unit": "gas", - "value": 827656693 - }, - { - "name": "alloc - 10945", - "unit": "gas", - "value": 2018421219 - }, - { - "name": "alloc - 46367", - "unit": "gas", - "value": 6403682702 - }, - { - "name": "alloc - 121392", - "unit": "gas", - "value": 16858374460 - }, - { - "name": "alloc - 317810", - "unit": "gas", - "value": 43146693797 - }, - { - "name": "counter - async_call", - "unit": "gas", - "value": 850482107 - }, - { - "name": "counter - sync_call", - "unit": "gas", - "value": 677860234 - }, - { - "name": "cross_program", - "unit": "gas", - "value": 2511573058 - }, - { - "name": "redirect", - "unit": "gas", - "value": 3612969613 - } -] \ No newline at end of file From 9edf8abea0f11733601c6eec9a5aebb31ac8f71e Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 19:48:01 +0300 Subject: [PATCH 04/50] try trigger CI --- .github/workflows/bench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 3c66cf005..86b3fb7e8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,11 +2,11 @@ name: '[rs] Benchmarks' on: pull_request: - types: [labeled] - paths: - - 'rs/**' - - 'Cargo.lock' - - 'Cargo.toml' + # types: [labeled] + # paths: + # - 'rs/**' + # - 'Cargo.lock' + # - 'Cargo.toml' push: branches: [master] paths: From ed4eaad5a9e35bc5ce97259d4677aa677715c18e Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 20:07:54 +0300 Subject: [PATCH 05/50] adjust CI job --- .github/workflows/bench.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 86b3fb7e8..ae5999952 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -77,7 +77,4 @@ jobs: external-data-json-path: bench_data_baseline_converted.json # Comment on PR with results comment-always: true - # Threshold for alerting on regression (e.g., 10% worse) - alert-threshold: '0%' - auto-push: false github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 1bd9b60cafcc14b6e84beea839bf68f9f6ca74af Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 5 Aug 2025 20:28:41 +0300 Subject: [PATCH 06/50] add summary CI --- .github/workflows/bench.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ae5999952..f99e05314 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -32,11 +32,11 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup Rust - # uses: actions-rust-lang/setup-rust-toolchain@v1 - # with: - # toolchain-file: rust-toolchain.toml - # cache: false + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain-file: rust-toolchain.toml + cache: false # Run benchmarks on PR branch - name: Run Benchmarks @@ -77,4 +77,5 @@ jobs: external-data-json-path: bench_data_baseline_converted.json # Comment on PR with results comment-always: true + summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 8442c68e5d24beee74ba838f46bce7ecdeef64d3 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 14:17:57 +0300 Subject: [PATCH 07/50] try include issues --- .github/workflows/bench.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f99e05314..6a51c9c70 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -25,6 +25,7 @@ jobs: permissions: contents: write pull-requests: write + issues: write steps: # Checkout the current branch (PR branch) - name: Checkout PR branch From 6b07ebcd1fe07ba7dbee88422829296e8be19e51 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 15:25:44 +0300 Subject: [PATCH 08/50] test comments --- .github/workflows/bench.yml | 95 +++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6a51c9c70..92a433f8d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -32,51 +32,64 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Test Comment + # if: github.event_name == 'pull_request' + uses: actions/github-script@v6 with: - toolchain-file: rust-toolchain.toml - cache: false + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: 'Test comment from GitHub Actions' + }); + + # - name: Setup Rust + # uses: actions-rust-lang/setup-rust-toolchain@v1 + # with: + # toolchain-file: rust-toolchain.toml + # cache: false - # Run benchmarks on PR branch - - name: Run Benchmarks - run: | - make bench + # # Run benchmarks on PR branch + # - name: Run Benchmarks + # run: | + # make bench - # Checkout master branch to get baseline JSON - - name: Checkout master branch - uses: actions/checkout@v4 - with: - ref: master - path: master-branch + # # Checkout master branch to get baseline JSON + # - name: Checkout master branch + # uses: actions/checkout@v4 + # with: + # ref: master + # path: master-branch - # Copy baseline JSON from master - - name: Copy Baseline JSON - run: | - cp master-branch/benchmarks/bench_data.json baseline.json + # # Copy baseline JSON from master + # - name: Copy Baseline JSON + # run: | + # cp master-branch/benchmarks/bench_data.json baseline.json - # Update the bench files formats - - name: Convert PR Bench Data - run: | - cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json + # # Update the bench files formats + # - name: Convert PR Bench Data + # run: | + # cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json - - name: Convert Baseline Bench Data - run: | - cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json + # - name: Convert Baseline Bench Data + # run: | + # cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json - # Compare benchmarks using github-action-benchmark - - name: Compare Benchmarks - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'customSmallerIsBetter' - # Path to the benchmark JSON from the PR branch - benchmark-data-dir-path: . - # Name of the benchmark JSON file - output-file-path: bench_data_pr_converted.json - # Path to the baseline JSON (from master) - external-data-json-path: bench_data_baseline_converted.json - # Comment on PR with results - comment-always: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + # # Compare benchmarks using github-action-benchmark + # - name: Compare Benchmarks + # uses: benchmark-action/github-action-benchmark@v1 + # with: + # tool: 'customSmallerIsBetter' + # # Path to the benchmark JSON from the PR branch + # benchmark-data-dir-path: . + # # Name of the benchmark JSON file + # output-file-path: bench_data_pr_converted.json + # # Path to the baseline JSON (from master) + # external-data-json-path: bench_data_baseline_converted.json + # # Comment on PR with results + # comment-always: true + # summary-always: true + # github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From a51e3a210cae7746821aa16eb85f07b943e2e69e Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 15:30:40 +0300 Subject: [PATCH 09/50] turn back to usual job config --- .github/workflows/bench.yml | 96 ++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 92a433f8d..bef81693f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -33,18 +33,18 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: Test Comment - # if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: 'Test comment from GitHub Actions' - }); + # - name: Test Comment + # if: github.event_name == 'pull_request' + # uses: actions/github-script@v6 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # await github.rest.issues.createComment({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # issue_number: context.issue.number, + # body: 'Test comment from GitHub Actions' + # }); # - name: Setup Rust # uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -52,44 +52,44 @@ jobs: # toolchain-file: rust-toolchain.toml # cache: false - # # Run benchmarks on PR branch - # - name: Run Benchmarks - # run: | - # make bench + # Run benchmarks on PR branch + - name: Run Benchmarks + run: | + make bench - # # Checkout master branch to get baseline JSON - # - name: Checkout master branch - # uses: actions/checkout@v4 - # with: - # ref: master - # path: master-branch + # Checkout master branch to get baseline JSON + - name: Checkout master branch + uses: actions/checkout@v4 + with: + ref: master + path: master-branch - # # Copy baseline JSON from master - # - name: Copy Baseline JSON - # run: | - # cp master-branch/benchmarks/bench_data.json baseline.json + # Copy baseline JSON from master + - name: Copy Baseline JSON + run: | + cp master-branch/benchmarks/bench_data.json baseline.json - # # Update the bench files formats - # - name: Convert PR Bench Data - # run: | - # cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json + # Update the bench files formats + - name: Convert PR Bench Data + run: | + cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json - # - name: Convert Baseline Bench Data - # run: | - # cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json + - name: Convert Baseline Bench Data + run: | + cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json - # # Compare benchmarks using github-action-benchmark - # - name: Compare Benchmarks - # uses: benchmark-action/github-action-benchmark@v1 - # with: - # tool: 'customSmallerIsBetter' - # # Path to the benchmark JSON from the PR branch - # benchmark-data-dir-path: . - # # Name of the benchmark JSON file - # output-file-path: bench_data_pr_converted.json - # # Path to the baseline JSON (from master) - # external-data-json-path: bench_data_baseline_converted.json - # # Comment on PR with results - # comment-always: true - # summary-always: true - # github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + # Compare benchmarks using github-action-benchmark + - name: Compare Benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'customSmallerIsBetter' + # Path to the benchmark JSON from the PR branch + benchmark-data-dir-path: . + # Name of the benchmark JSON file + output-file-path: bench_data_pr_converted.json + # Path to the baseline JSON (from master) + external-data-json-path: bench_data_baseline_converted.json + # Comment on PR with results + comment-always: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 97510d775767fe6b042469dc9d7e725c015ef7f3 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 15:45:16 +0300 Subject: [PATCH 10/50] try again alert threshold 0% --- .github/workflows/bench.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index bef81693f..5101a963a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -92,4 +92,5 @@ jobs: # Comment on PR with results comment-always: true summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: 0% \ No newline at end of file From 3191f42914c116b5c29addd035ee9c0207bf97c8 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 15:58:28 +0300 Subject: [PATCH 11/50] define comment-on-alert --- .github/workflows/bench.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 5101a963a..d43a585ad 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -17,6 +17,7 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + ACTIONS_STEP_DEBUG: true jobs: benchmark: @@ -46,11 +47,11 @@ jobs: # body: 'Test comment from GitHub Actions' # }); - # - name: Setup Rust - # uses: actions-rust-lang/setup-rust-toolchain@v1 - # with: - # toolchain-file: rust-toolchain.toml - # cache: false + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain-file: rust-toolchain.toml + cache: false # Run benchmarks on PR branch - name: Run Benchmarks @@ -92,5 +93,6 @@ jobs: # Comment on PR with results comment-always: true summary-always: true + comment-on-alert: true github-token: ${{ secrets.GITHUB_TOKEN }} alert-threshold: 0% \ No newline at end of file From 18141b2581a47b272421f54151b679ab9937d441 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 16:55:49 +0300 Subject: [PATCH 12/50] add possible fix --- .github/workflows/bench.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d43a585ad..d9f81cb5a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -81,7 +81,7 @@ jobs: # Compare benchmarks using github-action-benchmark - name: Compare Benchmarks - uses: benchmark-action/github-action-benchmark@v1 + uses: benchmark-action/github-action-benchmark@v1.20.4 with: tool: 'customSmallerIsBetter' # Path to the benchmark JSON from the PR branch @@ -92,7 +92,7 @@ jobs: external-data-json-path: bench_data_baseline_converted.json # Comment on PR with results comment-always: true - summary-always: true comment-on-alert: true - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: 0% \ No newline at end of file + save-data-file: false + ref: ${{ github.event.pull_request.head.sha }} + alert-threshold: "0%" \ No newline at end of file From 392c375f4ae628a2d67d9d3de9853915e2f6bfe4 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 17:16:51 +0300 Subject: [PATCH 13/50] add gh token --- .github/workflows/bench.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d9f81cb5a..191342fea 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -92,6 +92,7 @@ jobs: external-data-json-path: bench_data_baseline_converted.json # Comment on PR with results comment-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} comment-on-alert: true save-data-file: false ref: ${{ github.event.pull_request.head.sha }} From 1309e61255919be6596f2225204c57c86c111ede Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 18:16:15 +0300 Subject: [PATCH 14/50] try bench --- .github/workflows/bench.yml | 2 +- benchmarks/compute-stress/src/lib.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 191342fea..52f618835 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -96,4 +96,4 @@ jobs: comment-on-alert: true save-data-file: false ref: ${{ github.event.pull_request.head.sha }} - alert-threshold: "0%" \ No newline at end of file + alert-threshold: "1%" \ No newline at end of file diff --git a/benchmarks/compute-stress/src/lib.rs b/benchmarks/compute-stress/src/lib.rs index 6319632de..751d4ad73 100644 --- a/benchmarks/compute-stress/src/lib.rs +++ b/benchmarks/compute-stress/src/lib.rs @@ -8,6 +8,10 @@ struct ComputeStressService; impl ComputeStressService { #[export] pub fn compute_stress(&mut self, n: u32) -> ComputeStressResult { + let mut vec1 = vec![0u32; 1000]; + for i in 0..vec1.len() { + vec1[i] = i as u32; + } let res = sum_of_fib(n); ComputeStressResult { res } From cb82b74692d169f0aa76801657a6438681cc49a2 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 18:33:38 +0300 Subject: [PATCH 15/50] cat bench files --- .github/workflows/bench.yml | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 52f618835..7011d0530 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,11 +2,11 @@ name: '[rs] Benchmarks' on: pull_request: - # types: [labeled] - # paths: - # - 'rs/**' - # - 'Cargo.lock' - # - 'Cargo.toml' + types: [labeled] + paths: + - 'rs/**' + - 'Cargo.lock' + - 'Cargo.toml' push: branches: [master] paths: @@ -17,11 +17,10 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - ACTIONS_STEP_DEBUG: true jobs: benchmark: - # if: contains(github.event.label.name, 'run-benchmarks') + if: contains(github.event.label.name, 'run-benchmarks') runs-on: ubuntu-latest permissions: contents: write @@ -33,19 +32,6 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - # - name: Test Comment - # if: github.event_name == 'pull_request' - # uses: actions/github-script@v6 - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # script: | - # await github.rest.issues.createComment({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # issue_number: context.issue.number, - # body: 'Test comment from GitHub Actions' - # }); - name: Setup Rust uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -79,18 +65,21 @@ jobs: run: | cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json + - name: Display Bench Data + run: | + echo "PR Bench Data:" + cat bench_data_pr_converted.json + echo "Baseline Bench Data:" + cat bench_data_baseline_converted.json + # Compare benchmarks using github-action-benchmark - name: Compare Benchmarks uses: benchmark-action/github-action-benchmark@v1.20.4 with: tool: 'customSmallerIsBetter' - # Path to the benchmark JSON from the PR branch benchmark-data-dir-path: . - # Name of the benchmark JSON file output-file-path: bench_data_pr_converted.json - # Path to the baseline JSON (from master) external-data-json-path: bench_data_baseline_converted.json - # Comment on PR with results comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} comment-on-alert: true From 974860b86767c6ba1269c406980a403fe3e8500f Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 18:35:34 +0300 Subject: [PATCH 16/50] Trigger CI From 28a0bed33e23bdee2b180c5e1c688a12f2688789 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 6 Aug 2025 18:36:27 +0300 Subject: [PATCH 17/50] remove filters --- .github/workflows/bench.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7011d0530..225134d4f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,11 +2,11 @@ name: '[rs] Benchmarks' on: pull_request: - types: [labeled] - paths: - - 'rs/**' - - 'Cargo.lock' - - 'Cargo.toml' + # types: [labeled] + # paths: + # - 'rs/**' + # - 'Cargo.lock' + # - 'Cargo.toml' push: branches: [master] paths: @@ -20,7 +20,7 @@ env: jobs: benchmark: - if: contains(github.event.label.name, 'run-benchmarks') + # if: contains(github.event.label.name, 'run-benchmarks') runs-on: ubuntu-latest permissions: contents: write From c7bfcc8e1b2f0a6393ce3a6e8de13de039d2b8ed Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 7 Aug 2025 14:29:25 +0300 Subject: [PATCH 18/50] include wasm-opt --- .github/workflows/bench.yml | 11 +++-------- benchmarks/compute-stress/src/lib.rs | 4 ---- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 225134d4f..6a4fc6bd8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -30,14 +30,9 @@ jobs: # Checkout the current branch (PR branch) - name: Checkout PR branch uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain-file: rust-toolchain.toml - cache: false + + - name: Install wasm-opt + uses: ./.github/actions/install-wasm-utils # Run benchmarks on PR branch - name: Run Benchmarks diff --git a/benchmarks/compute-stress/src/lib.rs b/benchmarks/compute-stress/src/lib.rs index 751d4ad73..6319632de 100644 --- a/benchmarks/compute-stress/src/lib.rs +++ b/benchmarks/compute-stress/src/lib.rs @@ -8,10 +8,6 @@ struct ComputeStressService; impl ComputeStressService { #[export] pub fn compute_stress(&mut self, n: u32) -> ComputeStressResult { - let mut vec1 = vec![0u32; 1000]; - for i in 0..vec1.len() { - vec1[i] = i as u32; - } let res = sum_of_fib(n); ComputeStressResult { res } From a4570205310e8154fe8e20175f350c38398ab407 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 7 Aug 2025 14:47:11 +0300 Subject: [PATCH 19/50] add clean-up --- .github/workflows/bench.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6a4fc6bd8..0637fd241 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -31,6 +31,9 @@ jobs: - name: Checkout PR branch uses: actions/checkout@v4 + - name: Free Disk Space + uses: ./.github/actions/free-disk-space + - name: Install wasm-opt uses: ./.github/actions/install-wasm-utils @@ -39,6 +42,7 @@ jobs: run: | make bench + # todo [sab] check if you can download only one file # Checkout master branch to get baseline JSON - name: Checkout master branch uses: actions/checkout@v4 From 0f91cb8b658a640bcb89d1bb642c7358411914e9 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Mon, 11 Aug 2025 16:13:04 +0300 Subject: [PATCH 20/50] adjust solution, remove conversion script, check weights --- .github/workflows/{bench.yml => rs-bench.yml} | 28 +---- benchmarks/Cargo.toml | 4 - benchmarks/src/bin/convert_bench_data.rs | 109 ------------------ rust-toolchain.toml | 4 +- 4 files changed, 4 insertions(+), 141 deletions(-) rename .github/workflows/{bench.yml => rs-bench.yml} (55%) delete mode 100644 benchmarks/src/bin/convert_bench_data.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/rs-bench.yml similarity index 55% rename from .github/workflows/bench.yml rename to .github/workflows/rs-bench.yml index 0637fd241..14a601c0b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/rs-bench.yml @@ -55,33 +55,9 @@ jobs: run: | cp master-branch/benchmarks/bench_data.json baseline.json - # Update the bench files formats - - name: Convert PR Bench Data - run: | - cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- benchmarks/bench_data.json bench_data_pr_converted.json - - - name: Convert Baseline Bench Data - run: | - cargo run --bin convert-bench-data --manifest-path=benchmarks/Cargo.toml -- baseline.json bench_data_baseline_converted.json - - name: Display Bench Data run: | echo "PR Bench Data:" - cat bench_data_pr_converted.json + cat benchmarks/bench_data.json echo "Baseline Bench Data:" - cat bench_data_baseline_converted.json - - # Compare benchmarks using github-action-benchmark - - name: Compare Benchmarks - uses: benchmark-action/github-action-benchmark@v1.20.4 - with: - tool: 'customSmallerIsBetter' - benchmark-data-dir-path: . - output-file-path: bench_data_pr_converted.json - external-data-json-path: bench_data_baseline_converted.json - comment-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - save-data-file: false - ref: ${{ github.event.pull_request.head.sha }} - alert-threshold: "1%" \ No newline at end of file + cat baseline.json diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index eca9c4ca2..bf8a60a51 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -6,10 +6,6 @@ edition.workspace = true license.workspace = true repository.workspace = true -[[bin]] -name = "convert-bench-data" -path = "src/bin/convert_bench_data.rs" - [dependencies] anyhow.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/benchmarks/src/bin/convert_bench_data.rs b/benchmarks/src/bin/convert_bench_data.rs deleted file mode 100644 index 290d15156..000000000 --- a/benchmarks/src/bin/convert_bench_data.rs +++ /dev/null @@ -1,109 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::fs; -use std::path::Path; - -#[derive(Deserialize)] -struct BenchData { - compute: u64, - alloc: HashMap, - counter: HashMap, - cross_program: u64, - redirect: u64, -} - -#[derive(Serialize)] -struct BenchmarkEntry { - name: String, - unit: String, - value: u64, -} - -fn main() -> Result<()> { - let args: Vec = env::args().collect(); - - if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { - println!("Benchmark Data Converter"); - println!(); - println!("USAGE:"); - println!(" {} [INPUT_FILE] [OUTPUT_FILE]", args[0]); - println!(); - println!("ARGUMENTS:"); - println!(" INPUT_FILE Input JSON file (default: bench_data.json)"); - println!(" OUTPUT_FILE Output JSON file (default: bench_data_converted.json)"); - println!(); - println!("DESCRIPTION:"); - println!(" Converts benchmark data from the original nested JSON format"); - println!(" to the GitHub Actions benchmark format with name, unit, and value fields."); - return Ok(()); - } - - let input_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); - let output_file = args.get(2).unwrap_or(&"bench_data_converted.json".to_string()).clone(); - - // Read the input file - let input_path = Path::new(&input_file); - if !input_path.exists() { - eprintln!("Input file '{}' does not exist", input_file); - std::process::exit(1); - } - - let content = fs::read_to_string(input_path)?; - let bench_data: BenchData = serde_json::from_str(&content)?; - - let mut entries = Vec::new(); - - // Add compute benchmark - entries.push(BenchmarkEntry { - name: "Compute".to_string(), - unit: "gas".to_string(), - value: bench_data.compute, - }); - - // Add alloc benchmarks (sorted by key for consistent ordering) - let mut alloc_keys: Vec<_> = bench_data.alloc.keys().collect(); - alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); - - for key in alloc_keys { - let value = bench_data.alloc[key]; - entries.push(BenchmarkEntry { - name: format!("alloc - {}", key), - unit: "gas".to_string(), - value, - }); - } - - // Add counter benchmarks - for (key, value) in &bench_data.counter { - entries.push(BenchmarkEntry { - name: format!("counter - {}", key), - unit: "gas".to_string(), - value: *value, - }); - } - - // Add cross_program benchmark - entries.push(BenchmarkEntry { - name: "cross_program".to_string(), - unit: "gas".to_string(), - value: bench_data.cross_program, - }); - - // Add redirect benchmark - entries.push(BenchmarkEntry { - name: "redirect".to_string(), - unit: "gas".to_string(), - value: bench_data.redirect, - }); - - // Write the output file - let output_json = serde_json::to_string_pretty(&entries)?; - fs::write(&output_file, output_json)?; - - println!("Successfully converted '{}' to '{}'", input_file, output_file); - println!("Generated {} benchmark entries", entries.len()); - - Ok(()) -} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4fa2844cf..11572f7b3 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "stable" +channel = "1.88" targets = ["wasm32-unknown-unknown", "wasm32v1-none"] -components = [ "clippy", "rustfmt", "llvm-tools" ] +components = ["clippy", "rustfmt", "llvm-tools"] From 21e75d8b087472de48c9d497304305b0a56e4f56 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Mon, 11 Aug 2025 16:25:09 +0300 Subject: [PATCH 21/50] bump wasm-opt version --- .github/actions/install-wasm-utils/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-wasm-utils/action.yml b/.github/actions/install-wasm-utils/action.yml index 91c44f66a..fdb8e8892 100644 --- a/.github/actions/install-wasm-utils/action.yml +++ b/.github/actions/install-wasm-utils/action.yml @@ -4,7 +4,7 @@ inputs: binaryen_version: description: "Binaryen Version" required: false - default: "111" + default: "123" runs: using: composite From faa90bc2ee3ec1bd272a6a16d0fbca35ccd8bc79 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Mon, 11 Aug 2025 16:38:21 +0300 Subject: [PATCH 22/50] introduce first iter solution --- .github/workflows/rs-bench.yml | 63 +++++++ benchmarks/Cargo.toml | 4 + benchmarks/baseline.json | 19 ++ benchmarks/comparison.md | 23 +++ benchmarks/src/bin/compare_benchmarks.rs | 221 +++++++++++++++++++++++ 5 files changed, 330 insertions(+) create mode 100644 benchmarks/baseline.json create mode 100644 benchmarks/comparison.md create mode 100644 benchmarks/src/bin/compare_benchmarks.rs diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 14a601c0b..204a85f2c 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -61,3 +61,66 @@ jobs: cat benchmarks/bench_data.json echo "Baseline Bench Data:" cat baseline.json + + # Compare benchmarks and generate markdown table + - name: Compare Benchmarks + if: github.event_name == 'pull_request' + run: | + cd benchmarks + cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md + env: + CARGO_TERM_COLOR: always + + # Read the comparison markdown for the comment + - name: Read Comparison Result + if: github.event_name == 'pull_request' + id: comparison + run: | + cd benchmarks + echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT + cat comparison.md >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + # Comment the comparison table on the PR + - name: Comment PR with Benchmark Comparison + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; + + // Find existing benchmark comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.data.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ”¬ Benchmark Comparison') + ); + + const commentBody = `${comparisonTable} + + --- + šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody + }); + } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index bf8a60a51..47942c14c 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -6,6 +6,10 @@ edition.workspace = true license.workspace = true repository.workspace = true +[[bin]] +name = "compare-benchmarks" +path = "src/bin/compare_benchmarks.rs" + [dependencies] anyhow.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/benchmarks/baseline.json b/benchmarks/baseline.json new file mode 100644 index 000000000..1c214476d --- /dev/null +++ b/benchmarks/baseline.json @@ -0,0 +1,19 @@ +{ + "compute": 450513691329, + "alloc": { + "0": 563795739, + "12": 567302331, + "143": 726670926, + "986": 827656693, + "10945": 2018421219, + "46367": 6403682702, + "121392": 16858374460, + "317810": 43146693797 + }, + "counter": { + "async_call": 850482107, + "sync_call": 677860234 + }, + "cross_program": 2511573058, + "redirect": 3612969613 +} \ No newline at end of file diff --git a/benchmarks/comparison.md b/benchmarks/comparison.md new file mode 100644 index 000000000..e0fc5928f --- /dev/null +++ b/benchmarks/comparison.md @@ -0,0 +1,23 @@ +## šŸ”¬ Benchmark Comparison + +| Benchmark | Current | Baseline | Change | Change % | Status | +|-----------|---------|----------|---------|----------|--------| +| Compute | 450_513_691_329 | 450_513_691_329 | +0 | +0.00% | āœ… | +| alloc - 0 | 563_795_739 | 563_795_739 | +0 | +0.00% | āœ… | +| alloc - 12 | 567_302_331 | 567_302_331 | +0 | +0.00% | āœ… | +| alloc - 143 | 726_670_926 | 726_670_926 | +0 | +0.00% | āœ… | +| alloc - 986 | 827_656_693 | 827_656_693 | +0 | +0.00% | āœ… | +| alloc - 10945 | 2_018_421_219 | 2_018_421_219 | +0 | +0.00% | āœ… | +| alloc - 46367 | 6_403_682_702 | 6_403_682_702 | +0 | +0.00% | āœ… | +| alloc - 121392 | 16_858_374_460 | 16_858_374_460 | +0 | +0.00% | āœ… | +| alloc - 317810 | 43_146_693_797 | 43_146_693_797 | +0 | +0.00% | āœ… | +| counter - sync_call | 677_860_234 | 677_860_234 | +0 | +0.00% | āœ… | +| counter - async_call | 850_482_107 | 850_482_107 | +0 | +0.00% | āœ… | +| cross_program | 2_511_573_058 | 2_511_573_058 | +0 | +0.00% | āœ… | +| redirect | 3_612_969_613 | 3_612_969_613 | +0 | +0.00% | āœ… | + +### Legend +- šŸš€ Significant improvement (>5% reduction) +- āœ… No significant change or minor improvement +- āš ļø Minor regression (<5% increase) +- āŒ Significant regression (>5% increase) diff --git a/benchmarks/src/bin/compare_benchmarks.rs b/benchmarks/src/bin/compare_benchmarks.rs new file mode 100644 index 000000000..7308de40e --- /dev/null +++ b/benchmarks/src/bin/compare_benchmarks.rs @@ -0,0 +1,221 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::Path; + +#[derive(Deserialize, Clone)] +struct BenchData { + compute: u64, + alloc: HashMap, + counter: HashMap, + cross_program: u64, + redirect: u64, +} + +#[derive(Serialize)] +struct ComparisonResult { + name: String, + current: u64, + baseline: u64, + change: i64, + change_percent: f64, + status: String, +} + +fn format_gas(gas: u64) -> String { + let gas_str = gas.to_string(); + let mut result = String::new(); + + for (i, ch) in gas_str.chars().rev().enumerate() { + if i > 0 && i % 3 == 0 { + result.push('_'); + } + result.push(ch); + } + + result.chars().rev().collect() +} + +fn calculate_change_status(change_percent: f64) -> String { + if change_percent.abs() < 1.0 { + "āœ…".to_string() // No significant change + } else if change_percent < -5.0 { + "šŸš€".to_string() // Significant improvement + } else if change_percent < 0.0 { + "āœ…".to_string() // Minor improvement + } else if change_percent < 5.0 { + "āš ļø".to_string() // Minor regression + } else { + "āŒ".to_string() // Significant regression + } +} + +fn compare_values(name: String, current: u64, baseline: u64) -> ComparisonResult { + let change = current as i64 - baseline as i64; + let change_percent = if baseline > 0 { + (change as f64 / baseline as f64) * 100.0 + } else { + 0.0 + }; + let status = calculate_change_status(change_percent); + + ComparisonResult { + name, + current, + baseline, + change, + change_percent, + status, + } +} + +fn generate_markdown_table(comparisons: &[ComparisonResult]) -> String { + let mut markdown = String::new(); + + markdown.push_str("## šŸ”¬ Benchmark Comparison\n\n"); + markdown.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n"); + markdown.push_str("|-----------|---------|----------|---------|----------|--------|\n"); + + for comp in comparisons { + let change_sign = if comp.change >= 0 { "+" } else { "" }; + markdown.push_str(&format!( + "| {} | {} | {} | {}{} | {}{:.2}% | {} |\n", + comp.name, + format_gas(comp.current), + format_gas(comp.baseline), + change_sign, + format_gas(comp.change.abs() as u64), + if comp.change_percent >= 0.0 { "+" } else { "" }, + comp.change_percent, + comp.status + )); + } + + markdown.push_str("\n### Legend\n"); + markdown.push_str("- šŸš€ Significant improvement (>5% reduction)\n"); + markdown.push_str("- āœ… No significant change or minor improvement\n"); + markdown.push_str("- āš ļø Minor regression (<5% increase)\n"); + markdown.push_str("- āŒ Significant regression (>5% increase)\n"); + + markdown +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + + if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { + println!("Benchmark Comparison Tool"); + println!(); + println!("USAGE:"); + println!(" {} [CURRENT_FILE] [BASELINE_FILE] [OUTPUT_FILE]", args[0]); + println!(); + println!("ARGUMENTS:"); + println!(" CURRENT_FILE Current benchmark data (default: bench_data.json)"); + println!(" BASELINE_FILE Baseline benchmark data (default: baseline.json)"); + println!(" OUTPUT_FILE Output markdown file (default: comparison.md)"); + println!(); + println!("DESCRIPTION:"); + println!(" Compares two benchmark JSON files and generates a markdown table"); + println!(" showing the differences with status indicators."); + return Ok(()); + } + + let current_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); + let baseline_file = args.get(2).unwrap_or(&"baseline.json".to_string()).clone(); + let output_file = args.get(3).unwrap_or(&"comparison.md".to_string()).clone(); + + // Read the files + if !Path::new(¤t_file).exists() { + eprintln!("Current file '{}' does not exist", current_file); + std::process::exit(1); + } + + if !Path::new(&baseline_file).exists() { + eprintln!("Baseline file '{}' does not exist", baseline_file); + std::process::exit(1); + } + + let current_content = fs::read_to_string(¤t_file)?; + let baseline_content = fs::read_to_string(&baseline_file)?; + + let current_data: BenchData = serde_json::from_str(¤t_content)?; + let baseline_data: BenchData = serde_json::from_str(&baseline_content)?; + + let mut comparisons = Vec::new(); + + // Compare compute + comparisons.push(compare_values( + "Compute".to_string(), + current_data.compute, + baseline_data.compute, + )); + + // Compare alloc benchmarks (get all keys from both datasets) + let mut alloc_keys: std::collections::HashSet = current_data.alloc.keys().cloned().collect(); + alloc_keys.extend(baseline_data.alloc.keys().cloned()); + let mut alloc_keys: Vec<_> = alloc_keys.into_iter().collect(); + alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); + + for key in alloc_keys { + let current_val = current_data.alloc.get(&key).unwrap_or(&0); + let baseline_val = baseline_data.alloc.get(&key).unwrap_or(&0); + comparisons.push(compare_values( + format!("alloc - {}", key), + *current_val, + *baseline_val, + )); + } + + // Compare counter benchmarks + let mut counter_keys: std::collections::HashSet = current_data.counter.keys().cloned().collect(); + counter_keys.extend(baseline_data.counter.keys().cloned()); + let counter_keys: Vec<_> = counter_keys.into_iter().collect(); + + for key in counter_keys { + let current_val = current_data.counter.get(&key).unwrap_or(&0); + let baseline_val = baseline_data.counter.get(&key).unwrap_or(&0); + comparisons.push(compare_values( + format!("counter - {}", key), + *current_val, + *baseline_val, + )); + } + + // Compare cross_program + comparisons.push(compare_values( + "cross_program".to_string(), + current_data.cross_program, + baseline_data.cross_program, + )); + + // Compare redirect + comparisons.push(compare_values( + "redirect".to_string(), + current_data.redirect, + baseline_data.redirect, + )); + + // Generate markdown table + let markdown = generate_markdown_table(&comparisons); + + // Write to file + fs::write(&output_file, &markdown)?; + + // Also output to stdout for GitHub Actions + println!("{}", markdown); + + // Summary + let total_benchmarks = comparisons.len(); + let improvements = comparisons.iter().filter(|c| c.change_percent < -1.0).count(); + let regressions = comparisons.iter().filter(|c| c.change_percent > 1.0).count(); + let no_change = total_benchmarks - improvements - regressions; + + println!("šŸ“Š **Summary**: {} total benchmarks - {} improvements, {} no significant change, {} regressions", + total_benchmarks, improvements, no_change, regressions); + + println!("\nComparison table written to '{}'", output_file); + + Ok(()) +} From 582c8689a2fe2a587d7ec2045a38c58af2cd1d6c Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 14:51:22 +0300 Subject: [PATCH 23/50] introduce pre-check to find if benches were forgotten to be updated --- .github/workflows/rs-bench.yml | 33 ++- benchmarks/Cargo.toml | 8 + benchmarks/src/bin/check_benchmark_diff.rs | 233 +++++++++++++++++++++ benchmarks/src/bin/convert_bench_data.rs | 0 4 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 benchmarks/src/bin/check_benchmark_diff.rs create mode 100644 benchmarks/src/bin/convert_bench_data.rs diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 204a85f2c..582f3f828 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -42,28 +42,47 @@ jobs: run: | make bench - # todo [sab] check if you can download only one file - # Checkout master branch to get baseline JSON - - name: Checkout master branch + # Step 1: Check if current benchmarks differ significantly from previous ones in current branch + - name: Check Benchmark Diff vs Current Branch + if: github.event_name == 'pull_request' + run: | + # Get previous bench_data.json from current branch (before the latest commit) + git fetch origin ${{ github.head_ref }} + if git show HEAD~1:benchmarks/bench_data.json > bench_data_previous.json 2>/dev/null; then + echo "Found previous benchmark data in current branch" + cd benchmarks + cargo run --bin check-benchmark-diff -- bench_data.json ../bench_data_previous.json 1.0 + else + echo "No previous benchmark data found in current branch - treating as first run" + fi + env: + CARGO_TERM_COLOR: always + + # Step 2: If diff check passes, compare with master baseline + - name: Checkout master branch for baseline + if: github.event_name == 'pull_request' uses: actions/checkout@v4 with: ref: master path: master-branch # Copy baseline JSON from master - - name: Copy Baseline JSON + - name: Copy Baseline JSON from Master + if: github.event_name == 'pull_request' run: | cp master-branch/benchmarks/bench_data.json baseline.json - name: Display Bench Data + if: github.event_name == 'pull_request' run: | - echo "PR Bench Data:" + echo "=== Current PR Bench Data ===" cat benchmarks/bench_data.json - echo "Baseline Bench Data:" + echo "" + echo "=== Master Baseline Bench Data ===" cat baseline.json # Compare benchmarks and generate markdown table - - name: Compare Benchmarks + - name: Compare Benchmarks vs Master if: github.event_name == 'pull_request' run: | cd benchmarks diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 47942c14c..a51410c71 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -6,10 +6,18 @@ edition.workspace = true license.workspace = true repository.workspace = true +[[bin]] +name = "convert-bench-data" +path = "src/bin/convert_bench_data.rs" + [[bin]] name = "compare-benchmarks" path = "src/bin/compare_benchmarks.rs" +[[bin]] +name = "check-benchmark-diff" +path = "src/bin/check_benchmark_diff.rs" + [dependencies] anyhow.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/benchmarks/src/bin/check_benchmark_diff.rs b/benchmarks/src/bin/check_benchmark_diff.rs new file mode 100644 index 000000000..9e8ca8f1e --- /dev/null +++ b/benchmarks/src/bin/check_benchmark_diff.rs @@ -0,0 +1,233 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::Path; +use std::process; + +#[derive(Deserialize, Clone)] +struct BenchData { + compute: u64, + alloc: HashMap, + counter: HashMap, + cross_program: u64, + redirect: u64, +} + +#[derive(Serialize)] +struct DiffResult { + benchmark: String, + current: u64, + previous: u64, + diff_percent: f64, + exceeds_threshold: bool, +} + +fn calculate_diff_percent(current: u64, previous: u64) -> f64 { + if previous == 0 { + if current == 0 { + 0.0 + } else { + 100.0 // Consider any non-zero value as 100% increase from zero + } + } else { + ((current as f64 - previous as f64) / previous as f64) * 100.0 + } +} + +fn check_benchmark_value(name: String, current: u64, previous: u64, threshold: f64) -> DiffResult { + let diff_percent = calculate_diff_percent(current, previous); + let exceeds_threshold = diff_percent.abs() > threshold; + + DiffResult { + benchmark: name, + current, + previous, + diff_percent, + exceeds_threshold, + } +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + + if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { + println!("Benchmark Difference Checker"); + println!(); + println!("USAGE:"); + println!(" {} [CURRENT_FILE] [PREVIOUS_FILE] [THRESHOLD]", args[0]); + println!(); + println!("ARGUMENTS:"); + println!(" CURRENT_FILE Current benchmark data (default: bench_data.json)"); + println!(" PREVIOUS_FILE Previous benchmark data (default: bench_data_previous.json)"); + println!(" THRESHOLD Threshold percentage for failure (default: 1.0)"); + println!(); + println!("DESCRIPTION:"); + println!(" Compares current vs previous benchmark data and fails if any benchmark"); + println!(" differs by more than the threshold percentage. Exit code 0 = pass, 1 = fail."); + return Ok(()); + } + + let current_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); + let previous_file = args.get(2).unwrap_or(&"bench_data_previous.json".to_string()).clone(); + let threshold: f64 = args.get(3) + .and_then(|s| s.parse().ok()) + .unwrap_or(1.0); + + // Check if files exist + if !Path::new(¤t_file).exists() { + eprintln!("āŒ Current file '{}' does not exist", current_file); + process::exit(1); + } + + if !Path::new(&previous_file).exists() { + eprintln!("āš ļø Previous file '{}' does not exist - treating as first run", previous_file); + println!("āœ… No previous benchmarks to compare against. Skipping diff check."); + process::exit(0); + } + + // Read the files + let current_content = fs::read_to_string(¤t_file)?; + let previous_content = fs::read_to_string(&previous_file)?; + + let current_data: BenchData = serde_json::from_str(¤t_content)?; + let previous_data: BenchData = serde_json::from_str(&previous_content)?; + + let mut results = Vec::new(); + let mut has_failures = false; + + // Check compute + let diff = check_benchmark_value( + "compute".to_string(), + current_data.compute, + previous_data.compute, + threshold, + ); + if diff.exceeds_threshold { + has_failures = true; + } + results.push(diff); + + // Check alloc benchmarks + let mut alloc_keys: std::collections::HashSet = current_data.alloc.keys().cloned().collect(); + alloc_keys.extend(previous_data.alloc.keys().cloned()); + let mut alloc_keys: Vec<_> = alloc_keys.into_iter().collect(); + alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); + + for key in alloc_keys { + let current_val = current_data.alloc.get(&key).unwrap_or(&0); + let previous_val = previous_data.alloc.get(&key).unwrap_or(&0); + let diff = check_benchmark_value( + format!("alloc-{}", key), + *current_val, + *previous_val, + threshold, + ); + if diff.exceeds_threshold { + has_failures = true; + } + results.push(diff); + } + + // Check counter benchmarks + let mut counter_keys: std::collections::HashSet = current_data.counter.keys().cloned().collect(); + counter_keys.extend(previous_data.counter.keys().cloned()); + let counter_keys: Vec<_> = counter_keys.into_iter().collect(); + + for key in counter_keys { + let current_val = current_data.counter.get(&key).unwrap_or(&0); + let previous_val = previous_data.counter.get(&key).unwrap_or(&0); + let diff = check_benchmark_value( + format!("counter-{}", key), + *current_val, + *previous_val, + threshold, + ); + if diff.exceeds_threshold { + has_failures = true; + } + results.push(diff); + } + + // Check cross_program + let diff = check_benchmark_value( + "cross_program".to_string(), + current_data.cross_program, + previous_data.cross_program, + threshold, + ); + if diff.exceeds_threshold { + has_failures = true; + } + results.push(diff); + + // Check redirect + let diff = check_benchmark_value( + "redirect".to_string(), + current_data.redirect, + previous_data.redirect, + threshold, + ); + if diff.exceeds_threshold { + has_failures = true; + } + results.push(diff); + + // Print results + println!("šŸ” Benchmark Difference Analysis (threshold: {:.1}%)", threshold); + println!("ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); + + let mut passed = 0; + let mut failed = 0; + + for result in &results { + let status = if result.exceeds_threshold { + failed += 1; + "āŒ FAIL" + } else { + passed += 1; + "āœ… PASS" + }; + + let sign = if result.diff_percent >= 0.0 { "+" } else { "" }; + println!( + "{} | {:20} | {:>15} ā†’ {:>15} | {}{:>6.2}%", + status, + result.benchmark, + format_number(result.previous), + format_number(result.current), + sign, + result.diff_percent + ); + } + + println!("ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); + println!("šŸ“Š Summary: {} passed, {} failed", passed, failed); + + if has_failures { + println!(); + println!("āŒ BENCHMARK DIFF CHECK FAILED!"); + println!(" Some benchmarks differ by more than {:.1}% from the previous run.", threshold); + println!(" This indicates significant performance changes that need investigation."); + process::exit(1); + } else { + println!(); + println!("āœ… All benchmark differences are within acceptable threshold ({:.1}%)", threshold); + process::exit(0); + } +} + +fn format_number(num: u64) -> String { + let num_str = num.to_string(); + let mut result = String::new(); + + for (i, ch) in num_str.chars().rev().enumerate() { + if i > 0 && i % 3 == 0 { + result.push('_'); + } + result.push(ch); + } + + result.chars().rev().collect() +} diff --git a/benchmarks/src/bin/convert_bench_data.rs b/benchmarks/src/bin/convert_bench_data.rs new file mode 100644 index 000000000..e69de29bb From 137b2dc932a7b2e556054c52c613b162707b86ec Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 15:15:15 +0300 Subject: [PATCH 24/50] test git show --- .github/workflows/rs-bench.yml | 155 +++++++++++++++++---------------- 1 file changed, 80 insertions(+), 75 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 582f3f828..2f9fcf60b 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -42,6 +42,11 @@ jobs: run: | make bench + - name: Check bench data on previous commit + run: | + git fetch origin ${{ github.head_ref }} + git show HEAD~1:benchmarks/bench_data.json + # Step 1: Check if current benchmarks differ significantly from previous ones in current branch - name: Check Benchmark Diff vs Current Branch if: github.event_name == 'pull_request' @@ -58,88 +63,88 @@ jobs: env: CARGO_TERM_COLOR: always - # Step 2: If diff check passes, compare with master baseline - - name: Checkout master branch for baseline - if: github.event_name == 'pull_request' - uses: actions/checkout@v4 - with: - ref: master - path: master-branch + # # Step 2: If diff check passes, compare with master baseline + # - name: Checkout master branch for baseline + # if: github.event_name == 'pull_request' + # uses: actions/checkout@v4 + # with: + # ref: master + # path: master-branch - # Copy baseline JSON from master - - name: Copy Baseline JSON from Master - if: github.event_name == 'pull_request' - run: | - cp master-branch/benchmarks/bench_data.json baseline.json + # # Copy baseline JSON from master + # - name: Copy Baseline JSON from Master + # if: github.event_name == 'pull_request' + # run: | + # cp master-branch/benchmarks/bench_data.json baseline.json - - name: Display Bench Data - if: github.event_name == 'pull_request' - run: | - echo "=== Current PR Bench Data ===" - cat benchmarks/bench_data.json - echo "" - echo "=== Master Baseline Bench Data ===" - cat baseline.json + # - name: Display Bench Data + # if: github.event_name == 'pull_request' + # run: | + # echo "=== Current PR Bench Data ===" + # cat benchmarks/bench_data.json + # echo "" + # echo "=== Master Baseline Bench Data ===" + # cat baseline.json - # Compare benchmarks and generate markdown table - - name: Compare Benchmarks vs Master - if: github.event_name == 'pull_request' - run: | - cd benchmarks - cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md - env: - CARGO_TERM_COLOR: always + # # Compare benchmarks and generate markdown table + # - name: Compare Benchmarks vs Master + # if: github.event_name == 'pull_request' + # run: | + # cd benchmarks + # cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md + # env: + # CARGO_TERM_COLOR: always - # Read the comparison markdown for the comment - - name: Read Comparison Result - if: github.event_name == 'pull_request' - id: comparison - run: | - cd benchmarks - echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT - cat comparison.md >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT + # # Read the comparison markdown for the comment + # - name: Read Comparison Result + # if: github.event_name == 'pull_request' + # id: comparison + # run: | + # cd benchmarks + # echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT + # cat comparison.md >> $GITHUB_OUTPUT + # echo 'EOF' >> $GITHUB_OUTPUT - # Comment the comparison table on the PR - - name: Comment PR with Benchmark Comparison - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; + # # Comment the comparison table on the PR + # - name: Comment PR with Benchmark Comparison + # if: github.event_name == 'pull_request' + # uses: actions/github-script@v7 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; - // Find existing benchmark comment - const comments = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); + # // Find existing benchmark comment + # const comments = await github.rest.issues.listComments({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # issue_number: context.issue.number, + # }); - const botComment = comments.data.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('šŸ”¬ Benchmark Comparison') - ); + # const botComment = comments.data.find(comment => + # comment.user.type === 'Bot' && + # comment.body.includes('šŸ”¬ Benchmark Comparison') + # ); - const commentBody = `${comparisonTable} + # const commentBody = `${comparisonTable} - --- - šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; + # --- + # šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; - if (botComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: commentBody - }); - } else { - // Create new comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: commentBody - }); - } + # if (botComment) { + # // Update existing comment + # await github.rest.issues.updateComment({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # comment_id: botComment.id, + # body: commentBody + # }); + # } else { + # // Create new comment + # await github.rest.issues.createComment({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # issue_number: context.issue.number, + # body: commentBody + # }); + # } From e9724491e4c3ce8d8b9e019adaf4ff81bff78ab3 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 15:16:55 +0300 Subject: [PATCH 25/50] comment make bench --- .github/workflows/rs-bench.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 2f9fcf60b..d8abf2f96 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -37,10 +37,10 @@ jobs: - name: Install wasm-opt uses: ./.github/actions/install-wasm-utils - # Run benchmarks on PR branch - - name: Run Benchmarks - run: | - make bench + # # Run benchmarks on PR branch + # - name: Run Benchmarks + # run: | + # make bench - name: Check bench data on previous commit run: | From cca912c167a003d064f1b14eedcb6bfe2f0df4aa Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 15:22:25 +0300 Subject: [PATCH 26/50] add fetch-depth --- .github/workflows/rs-bench.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index d8abf2f96..b8fb6ab7a 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -30,6 +30,8 @@ jobs: # Checkout the current branch (PR branch) - name: Checkout PR branch uses: actions/checkout@v4 + with: + fetch-depth: 2 - name: Free Disk Space uses: ./.github/actions/free-disk-space @@ -77,6 +79,7 @@ jobs: # run: | # cp master-branch/benchmarks/bench_data.json baseline.json +# todo [sab] remove # - name: Display Bench Data # if: github.event_name == 'pull_request' # run: | @@ -105,6 +108,7 @@ jobs: # cat comparison.md >> $GITHUB_OUTPUT # echo 'EOF' >> $GITHUB_OUTPUT +# todo [sab] write a new comment, not updated (edited) # # Comment the comparison table on the PR # - name: Comment PR with Benchmark Comparison # if: github.event_name == 'pull_request' From 06c9345e4cf9ac65ba25cce96b58765defcf6a25 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 15:29:34 +0300 Subject: [PATCH 27/50] test diff test on a real diff --- benchmarks/bench_data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/bench_data.json b/benchmarks/bench_data.json index 1c214476d..19e245763 100644 --- a/benchmarks/bench_data.json +++ b/benchmarks/bench_data.json @@ -1,5 +1,5 @@ { - "compute": 450513691329, + "compute": 450513695329, "alloc": { "0": 563795739, "12": 567302331, From 066ad11d21c1cede62b435617692da1b8f5cc732 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Tue, 12 Aug 2025 15:31:34 +0300 Subject: [PATCH 28/50] uncomment full job --- .github/workflows/rs-bench.yml | 161 ++++++++++++++++----------------- 1 file changed, 78 insertions(+), 83 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index b8fb6ab7a..fd4d68213 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -39,15 +39,10 @@ jobs: - name: Install wasm-opt uses: ./.github/actions/install-wasm-utils - # # Run benchmarks on PR branch - # - name: Run Benchmarks - # run: | - # make bench - - - name: Check bench data on previous commit + # Run benchmarks on PR branch + - name: Run Benchmarks run: | - git fetch origin ${{ github.head_ref }} - git show HEAD~1:benchmarks/bench_data.json + make bench # Step 1: Check if current benchmarks differ significantly from previous ones in current branch - name: Check Benchmark Diff vs Current Branch @@ -65,90 +60,90 @@ jobs: env: CARGO_TERM_COLOR: always - # # Step 2: If diff check passes, compare with master baseline - # - name: Checkout master branch for baseline - # if: github.event_name == 'pull_request' - # uses: actions/checkout@v4 - # with: - # ref: master - # path: master-branch + # Step 2: If diff check passes, compare with master baseline + - name: Checkout master branch for baseline + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + with: + ref: master + path: master-branch - # # Copy baseline JSON from master - # - name: Copy Baseline JSON from Master - # if: github.event_name == 'pull_request' - # run: | - # cp master-branch/benchmarks/bench_data.json baseline.json + # Copy baseline JSON from master + - name: Copy Baseline JSON from Master + if: github.event_name == 'pull_request' + run: | + cp master-branch/benchmarks/bench_data.json baseline.json # todo [sab] remove - # - name: Display Bench Data - # if: github.event_name == 'pull_request' - # run: | - # echo "=== Current PR Bench Data ===" - # cat benchmarks/bench_data.json - # echo "" - # echo "=== Master Baseline Bench Data ===" - # cat baseline.json + - name: Display Bench Data + if: github.event_name == 'pull_request' + run: | + echo "=== Current PR Bench Data ===" + cat benchmarks/bench_data.json + echo "" + echo "=== Master Baseline Bench Data ===" + cat baseline.json - # # Compare benchmarks and generate markdown table - # - name: Compare Benchmarks vs Master - # if: github.event_name == 'pull_request' - # run: | - # cd benchmarks - # cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md - # env: - # CARGO_TERM_COLOR: always + # Compare benchmarks and generate markdown table + - name: Compare Benchmarks vs Master + if: github.event_name == 'pull_request' + run: | + cd benchmarks + cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md + env: + CARGO_TERM_COLOR: always - # # Read the comparison markdown for the comment - # - name: Read Comparison Result - # if: github.event_name == 'pull_request' - # id: comparison - # run: | - # cd benchmarks - # echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT - # cat comparison.md >> $GITHUB_OUTPUT - # echo 'EOF' >> $GITHUB_OUTPUT + # Read the comparison markdown for the comment + - name: Read Comparison Result + if: github.event_name == 'pull_request' + id: comparison + run: | + cd benchmarks + echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT + cat comparison.md >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT # todo [sab] write a new comment, not updated (edited) - # # Comment the comparison table on the PR - # - name: Comment PR with Benchmark Comparison - # if: github.event_name == 'pull_request' - # uses: actions/github-script@v7 - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # script: | - # const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; + # Comment the comparison table on the PR + - name: Comment PR with Benchmark Comparison + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; - # // Find existing benchmark comment - # const comments = await github.rest.issues.listComments({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # issue_number: context.issue.number, - # }); + // Find existing benchmark comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); - # const botComment = comments.data.find(comment => - # comment.user.type === 'Bot' && - # comment.body.includes('šŸ”¬ Benchmark Comparison') - # ); + const botComment = comments.data.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ”¬ Benchmark Comparison') + ); - # const commentBody = `${comparisonTable} + const commentBody = `${comparisonTable} - # --- - # šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; + --- + šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; - # if (botComment) { - # // Update existing comment - # await github.rest.issues.updateComment({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # comment_id: botComment.id, - # body: commentBody - # }); - # } else { - # // Create new comment - # await github.rest.issues.createComment({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # issue_number: context.issue.number, - # body: commentBody - # }); - # } + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody + }); + } From dce980df23d179c7d8bf481c0c1ef202ca601a03 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 13 Aug 2025 13:37:37 +0300 Subject: [PATCH 29/50] test case when changes weren't benched --- benchmarks/Cargo.toml | 4 ---- benchmarks/counter-bench/src/lib.rs | 7 +++++-- benchmarks/src/benchmarks.rs | 10 +++++----- benchmarks/src/bin/check_benchmark_diff.rs | 2 ++ benchmarks/src/bin/convert_bench_data.rs | 0 5 files changed, 12 insertions(+), 11 deletions(-) delete mode 100644 benchmarks/src/bin/convert_bench_data.rs diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index a51410c71..2c07c654e 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -6,10 +6,6 @@ edition.workspace = true license.workspace = true repository.workspace = true -[[bin]] -name = "convert-bench-data" -path = "src/bin/convert_bench_data.rs" - [[bin]] name = "compare-benchmarks" path = "src/bin/compare_benchmarks.rs" diff --git a/benchmarks/counter-bench/src/lib.rs b/benchmarks/counter-bench/src/lib.rs index 8744a82a3..a989795f0 100644 --- a/benchmarks/counter-bench/src/lib.rs +++ b/benchmarks/counter-bench/src/lib.rs @@ -10,12 +10,15 @@ pub struct CounterBenchService; impl CounterBenchService { #[export] pub fn inc(&mut self) -> u64 { + let mut data: Vec = vec![]; + for _ in 0..317810 { + data.push(0); + } unsafe { let prev = COUNTER; COUNTER += 1; - - prev } + data.len() as u64 } #[export] diff --git a/benchmarks/src/benchmarks.rs b/benchmarks/src/benchmarks.rs index dcdfb6542..9c2fb7d41 100644 --- a/benchmarks/src/benchmarks.rs +++ b/benchmarks/src/benchmarks.rs @@ -207,21 +207,21 @@ async fn counter_bench() { let (remoting, pid) = create_program_async!((CounterBenchProgramFactory::, wasm_path)); - let mut expected_value = 0; + // let mut expected_value = 0; let (mut gas_benches_sync, mut gas_benches_async): (Vec<_>, Vec<_>) = (0..100) .enumerate() .map(|(i, _)| { let is_sync = i % 2 == 0; let gas = if is_sync { let (stress_resp, gas_sync_inc) = call_action!(remoting, pid, Inc); - assert_eq!(stress_resp, expected_value); - expected_value += 1; + // assert_eq!(stress_resp, expected_value); + // expected_value += 1; gas_sync_inc } else { let (stress_resp, gas_async_inc) = call_action!(remoting, pid, IncAsync); - assert_eq!(stress_resp, expected_value); - expected_value += 1; + // assert_eq!(stress_resp, expected_value); + // expected_value += 1; gas_async_inc }; diff --git a/benchmarks/src/bin/check_benchmark_diff.rs b/benchmarks/src/bin/check_benchmark_diff.rs index 9e8ca8f1e..59205f19a 100644 --- a/benchmarks/src/bin/check_benchmark_diff.rs +++ b/benchmarks/src/bin/check_benchmark_diff.rs @@ -6,6 +6,8 @@ use std::fs; use std::path::Path; use std::process; +// todo [sab] check diff on negative values + #[derive(Deserialize, Clone)] struct BenchData { compute: u64, diff --git a/benchmarks/src/bin/convert_bench_data.rs b/benchmarks/src/bin/convert_bench_data.rs deleted file mode 100644 index e69de29bb..000000000 From 348181a045c725d81e2fbca1d1e0febf2a26283c Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 13 Aug 2025 13:49:00 +0300 Subject: [PATCH 30/50] test case when benched with significant changes --- benchmarks/bench_data.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/bench_data.json b/benchmarks/bench_data.json index 19e245763..56ab4c433 100644 --- a/benchmarks/bench_data.json +++ b/benchmarks/bench_data.json @@ -1,5 +1,5 @@ { - "compute": 450513695329, + "compute": 450513691329, "alloc": { "0": 563795739, "12": 567302331, @@ -11,8 +11,8 @@ "317810": 43146693797 }, "counter": { - "async_call": 850482107, - "sync_call": 677860234 + "async_call": 699056814, + "sync_call": 43471993060 }, "cross_program": 2511573058, "redirect": 3612969613 From 951265f6e60f4541e3074349e290d027c1b737e1 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Wed, 13 Aug 2025 23:06:26 +0300 Subject: [PATCH 31/50] refactor benches analyzing scripts --- .github/workflows/rs-bench.yml | 185 +++++++-------- Cargo.lock | 1 + benchmarks/Cargo.toml | 11 +- benchmarks/bench_data_previous.json | 19 ++ benchmarks/comparison.md | 4 +- benchmarks/src/benchmarks.rs | 12 +- benchmarks/src/bin/bench_analyzer.rs | 257 +++++++++++++++++++++ benchmarks/src/bin/check_benchmark_diff.rs | 235 ------------------- benchmarks/src/bin/compare_benchmarks.rs | 221 ------------------ benchmarks/src/lib.rs | 201 +++++++++++++--- 10 files changed, 553 insertions(+), 593 deletions(-) create mode 100644 benchmarks/bench_data_previous.json create mode 100644 benchmarks/src/bin/bench_analyzer.rs delete mode 100644 benchmarks/src/bin/check_benchmark_diff.rs delete mode 100644 benchmarks/src/bin/compare_benchmarks.rs diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index fd4d68213..cbaf81be8 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -44,106 +44,107 @@ jobs: run: | make bench - # Step 1: Check if current benchmarks differ significantly from previous ones in current branch - - name: Check Benchmark Diff vs Current Branch - if: github.event_name == 'pull_request' - run: | - # Get previous bench_data.json from current branch (before the latest commit) - git fetch origin ${{ github.head_ref }} - if git show HEAD~1:benchmarks/bench_data.json > bench_data_previous.json 2>/dev/null; then - echo "Found previous benchmark data in current branch" - cd benchmarks - cargo run --bin check-benchmark-diff -- bench_data.json ../bench_data_previous.json 1.0 - else - echo "No previous benchmark data found in current branch - treating as first run" - fi - env: - CARGO_TERM_COLOR: always +# # Step 1: Check if current benchmarks differ significantly from previous ones in current branch +# # todo [sab] the approach is buggy, because if current commit updates benches, it will still fail. +# - name: Check Benchmark Diff vs Current Branch +# if: github.event_name == 'pull_request' +# run: | +# # Get previous bench_data.json from current branch (before the latest commit) +# git fetch origin ${{ github.head_ref }} +# if git show HEAD~1:benchmarks/bench_data.json > bench_data_previous.json 2>/dev/null; then +# echo "Found previous benchmark data in current branch" +# cd benchmarks +# cargo run --bin bench-analyzer -- check-diff --current bench_data.json --previous ../bench_data_previous.json --threshold 1.0 +# else +# echo "No previous benchmark data found in current branch - treating as first run" +# fi +# env: +# CARGO_TERM_COLOR: always - # Step 2: If diff check passes, compare with master baseline - - name: Checkout master branch for baseline - if: github.event_name == 'pull_request' - uses: actions/checkout@v4 - with: - ref: master - path: master-branch +# # Step 2: If diff check passes, compare with master baseline +# - name: Checkout master branch for baseline +# if: github.event_name == 'pull_request' +# uses: actions/checkout@v4 +# with: +# ref: master +# path: master-branch - # Copy baseline JSON from master - - name: Copy Baseline JSON from Master - if: github.event_name == 'pull_request' - run: | - cp master-branch/benchmarks/bench_data.json baseline.json +# # Copy baseline JSON from master +# - name: Copy Baseline JSON from Master +# if: github.event_name == 'pull_request' +# run: | +# cp master-branch/benchmarks/bench_data.json baseline.json -# todo [sab] remove - - name: Display Bench Data - if: github.event_name == 'pull_request' - run: | - echo "=== Current PR Bench Data ===" - cat benchmarks/bench_data.json - echo "" - echo "=== Master Baseline Bench Data ===" - cat baseline.json +# # todo [sab] remove +# - name: Display Bench Data +# if: github.event_name == 'pull_request' +# run: | +# echo "=== Current PR Bench Data ===" +# cat benchmarks/bench_data.json +# echo "" +# echo "=== Master Baseline Bench Data ===" +# cat baseline.json - # Compare benchmarks and generate markdown table - - name: Compare Benchmarks vs Master - if: github.event_name == 'pull_request' - run: | - cd benchmarks - cargo run --bin compare-benchmarks -- bench_data.json ../baseline.json comparison.md - env: - CARGO_TERM_COLOR: always +# # Compare benchmarks and generate markdown table +# - name: Compare Benchmarks vs Master +# if: github.event_name == 'pull_request' +# run: | +# cd benchmarks +# cargo run --bin bench-analyzer -- compare --current bench_data.json --baseline ../baseline.json --output comparison.md +# env: +# CARGO_TERM_COLOR: always - # Read the comparison markdown for the comment - - name: Read Comparison Result - if: github.event_name == 'pull_request' - id: comparison - run: | - cd benchmarks - echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT - cat comparison.md >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT +# # Read the comparison markdown for the comment +# - name: Read Comparison Result +# if: github.event_name == 'pull_request' +# id: comparison +# run: | +# cd benchmarks +# echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT +# cat comparison.md >> $GITHUB_OUTPUT +# echo 'EOF' >> $GITHUB_OUTPUT -# todo [sab] write a new comment, not updated (edited) - # Comment the comparison table on the PR - - name: Comment PR with Benchmark Comparison - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; +# # todo [sab] write a new comment, not updated (edited) +# # Comment the comparison table on the PR +# - name: Comment PR with Benchmark Comparison +# if: github.event_name == 'pull_request' +# uses: actions/github-script@v7 +# with: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# script: | +# const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; - // Find existing benchmark comment - const comments = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); +# // Find existing benchmark comment +# const comments = await github.rest.issues.listComments({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# issue_number: context.issue.number, +# }); - const botComment = comments.data.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('šŸ”¬ Benchmark Comparison') - ); +# const botComment = comments.data.find(comment => +# comment.user.type === 'Bot' && +# comment.body.includes('šŸ”¬ Benchmark Comparison') +# ); - const commentBody = `${comparisonTable} +# const commentBody = `${comparisonTable} - --- - šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; +# --- +# šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; - if (botComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: commentBody - }); - } else { - // Create new comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: commentBody - }); - } +# if (botComment) { +# // Update existing comment +# await github.rest.issues.updateComment({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# comment_id: botComment.id, +# body: commentBody +# }); +# } else { +# // Create new comment +# await github.rest.issues.createComment({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# issue_number: context.issue.number, +# body: commentBody +# }); +# } diff --git a/Cargo.lock b/Cargo.lock index d981c18e8..e67ff9780 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,6 +830,7 @@ version = "0.9.0" dependencies = [ "alloc-stress", "anyhow", + "clap", "compute-stress", "convert_case 0.7.1", "counter-bench", diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 2c07c654e..f5c3f320d 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -7,19 +7,17 @@ license.workspace = true repository.workspace = true [[bin]] -name = "compare-benchmarks" -path = "src/bin/compare_benchmarks.rs" - -[[bin]] -name = "check-benchmark-diff" -path = "src/bin/check_benchmark_diff.rs" +name = "bench-analyzer" +path = "src/bin/bench_analyzer.rs" [dependencies] anyhow.workspace = true +clap = { version = "4.0", features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde-json.workspace = true fs2.workspace = true sails-rs.workspace = true +itertools.workspace = true [build-dependencies] sails-rs = { workspace = true, features = ["build"] } @@ -41,4 +39,3 @@ redirect-proxy = { path = "../examples/redirect/proxy" } redirect-proxy-client = { path = "../examples/redirect/proxy-client" } tokio = { workspace = true, features = ["rt", "macros"] } tempfile.workspace = true -itertools.workspace = true diff --git a/benchmarks/bench_data_previous.json b/benchmarks/bench_data_previous.json new file mode 100644 index 000000000..1c214476d --- /dev/null +++ b/benchmarks/bench_data_previous.json @@ -0,0 +1,19 @@ +{ + "compute": 450513691329, + "alloc": { + "0": 563795739, + "12": 567302331, + "143": 726670926, + "986": 827656693, + "10945": 2018421219, + "46367": 6403682702, + "121392": 16858374460, + "317810": 43146693797 + }, + "counter": { + "async_call": 850482107, + "sync_call": 677860234 + }, + "cross_program": 2511573058, + "redirect": 3612969613 +} \ No newline at end of file diff --git a/benchmarks/comparison.md b/benchmarks/comparison.md index e0fc5928f..a952615cf 100644 --- a/benchmarks/comparison.md +++ b/benchmarks/comparison.md @@ -11,8 +11,8 @@ | alloc - 46367 | 6_403_682_702 | 6_403_682_702 | +0 | +0.00% | āœ… | | alloc - 121392 | 16_858_374_460 | 16_858_374_460 | +0 | +0.00% | āœ… | | alloc - 317810 | 43_146_693_797 | 43_146_693_797 | +0 | +0.00% | āœ… | -| counter - sync_call | 677_860_234 | 677_860_234 | +0 | +0.00% | āœ… | -| counter - async_call | 850_482_107 | 850_482_107 | +0 | +0.00% | āœ… | +| counter - async_call | 699_056_814 | 850_482_107 | 151_425_293 | -17.80% | šŸš€ | +| counter - sync_call | 43_471_993_060 | 677_860_234 | +42_794_132_826 | +6313.12% | āŒ | | cross_program | 2_511_573_058 | 2_511_573_058 | +0 | +0.00% | āœ… | | redirect | 3_612_969_613 | 3_612_969_613 | +0 | +0.00% | āœ… | diff --git a/benchmarks/src/benchmarks.rs b/benchmarks/src/benchmarks.rs index 9c2fb7d41..69fea2b09 100644 --- a/benchmarks/src/benchmarks.rs +++ b/benchmarks/src/benchmarks.rs @@ -169,7 +169,7 @@ async fn alloc_stress_bench() { for (len, gas_benches) in benches { crate::store_bench_data(|bench_data| { - bench_data.alloc.insert(len, median(gas_benches)); + bench_data.update_alloc_bench(len, median(gas_benches)); }) .unwrap(); } @@ -195,7 +195,7 @@ async fn compute_stress_bench() { gas_benches.sort_unstable(); crate::store_bench_data(|bench_data| { - bench_data.compute = median(gas_benches); + bench_data.update_compute_bench(median(gas_benches)); }) .unwrap(); } @@ -240,8 +240,8 @@ async fn counter_bench() { gas_benches_async.sort_unstable(); crate::store_bench_data(|bench_data| { - bench_data.counter.sync_call = median(gas_benches_sync); - bench_data.counter.async_call = median(gas_benches_async); + bench_data.update_counter_bench(false, median(gas_benches_sync)); + bench_data.update_counter_bench(true, median(gas_benches_async)); }) .unwrap(); } @@ -269,7 +269,7 @@ async fn cross_program_bench() { gas_benches.sort_unstable(); crate::store_bench_data(|bench_data| { - bench_data.cross_program = median(gas_benches); + bench_data.update_cross_program_bench(median(gas_benches)); }) .unwrap(); } @@ -321,7 +321,7 @@ async fn redirect_bench() { .collect::>(); crate::store_bench_data(|bench_data| { - bench_data.redirect = median(gas_benches); + bench_data.update_redirect_bench(median(gas_benches)); }) .unwrap(); } diff --git a/benchmarks/src/bin/bench_analyzer.rs b/benchmarks/src/bin/bench_analyzer.rs new file mode 100644 index 000000000..7abe80833 --- /dev/null +++ b/benchmarks/src/bin/bench_analyzer.rs @@ -0,0 +1,257 @@ +use anyhow::{Context, Result, anyhow}; +use benchmarks::{BenchCategory, BenchDataFile, BenchDataOuter}; +use clap::Parser; +use itertools::Either; +use std::fs; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(version, about, long_about = None)] +#[command(name = "bench-analyzer")] +#[command(about = "A tool for analyzing benchmark data differences and comparisons")] +struct Cli { + /// Current benchmark data file + #[arg(long)] + current: PathBuf, + + /// Other benchmark data file + #[arg(long)] + other: PathBuf, + + /// Threshold percentage for failure + #[arg(long)] + threshold: Option, + + /// Report markdown file + #[arg(long)] + output: Option, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + compare_bench(cli.current, cli.other, cli.output, cli.threshold) +} + +fn compare_bench( + current: PathBuf, + other: PathBuf, + report_output: Option, + threshold: Option, +) -> Result<()> { + let (current_data, other_data) = get_bench_data(current, other)?; + + let mut report = String::new(); + report.push_str("## šŸ”¬ Benchmark Comparison\n\n"); + report.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n"); + report.push_str("|-----------|---------|----------|---------|----------|--------|\n"); + + let mut threshold_failed = threshold.map(|_| false); + let mut report = current_data + .into_iter() + .zip(other_data) + .map( + |((current_category, current_value), (other_category, other_value))| { + assert_eq!(current_category, other_category, "Categories do not match"); + + let comparison = BenchCategoryComparison::new( + current_category, + current_value, + other_value, + threshold, + ); + + if matches!(threshold_failed, Some(false)) && comparison.has_failed_threshold() { + let _ = threshold_failed.insert(true); + } + + comparison + }, + ) + .fold(report, |mut report, comparison| { + add_comparison_to_report(&mut report, comparison); + report + }); + + add_report_conclusion(&mut report, threshold, threshold_failed); + + // Printing the final report + println!("{report}"); + + if matches!(threshold_failed, Some(true)) { + return Err(anyhow!("Benchmark contains tests failing the threshold.")); + } + + if let Some(report_output) = report_output { + fs::write(&report_output, &report).context("Failed to write report output")?; + + println!( + "\nComparison table written to '{}'", + report_output.display() + ); + } + + Ok(()) +} + +fn get_bench_data(current: PathBuf, previous: PathBuf) -> Result<(BenchDataOuter, BenchDataOuter)> { + let mut current_file = + BenchDataFile::open(current).context("Failed to open current benchmark data file")?; + let mut previous_file = + BenchDataFile::open(previous).context("Failed to open previous benchmark data file")?; + + let current_data = current_file.read_bench_data()?; + let previous_data = previous_file.read_bench_data()?; + + Ok((current_data, previous_data)) +} + +#[derive(Debug)] +struct BenchCategoryComparison { + category: BenchCategory, + current: u64, + other: u64, + diff: i64, + diff_percent: f64, + status: Either, +} + +impl BenchCategoryComparison { + fn new( + category: BenchCategory, + current: u64, + other: u64, + maybe_threshold: Option, + ) -> Self { + let diff = current as i64 - other as i64; + let diff_percent = (diff as f64 / other as f64) * 100.0; + let status = match maybe_threshold { + Some(threshold) => { + let exceeds = diff_percent.abs() > threshold; + if exceeds { + Either::Left(ThresholdPassStatus::Fail) + } else { + Either::Left(ThresholdPassStatus::Pass) + } + } + None => { + if diff_percent.abs() < 1.0 { + // [0,..1.0) + Either::Right(PerformanceStatus::NoChange) + } else if diff_percent < -5.0 { + // [-inf, -5.0) + Either::Right(PerformanceStatus::SignificantImprovement) + } else if diff_percent < 0.0 { + // [-5.0, 0.0) + Either::Right(PerformanceStatus::MinorImprovement) + } else if diff_percent < 5.0 { + // [0.0, 5.0) + Either::Right(PerformanceStatus::MinorRegression) + } else { + // [5.0, inf) + Either::Right(PerformanceStatus::SignificantRegression) + } + } + }; + + Self { + category, + current, + other, + diff, + diff_percent, + status, + } + } + + fn has_failed_threshold(&self) -> bool { + self.status + .as_ref() + .left() + .map(|status| matches!(status, ThresholdPassStatus::Fail)) + .unwrap_or(false) + } +} + +#[derive(Debug, Clone, Copy)] +enum ThresholdPassStatus { + Pass, + Fail, +} + +#[derive(Debug, Clone, Copy)] +enum PerformanceStatus { + SignificantImprovement, + MinorImprovement, + NoChange, + SignificantRegression, + MinorRegression, +} + +fn status_to_str(status: &Either) -> &'static str { + match status { + Either::Left(ThresholdPassStatus::Pass) => "āœ… PASS", + Either::Left(ThresholdPassStatus::Fail) => "āŒ FAIL", + Either::Right(PerformanceStatus::SignificantImprovement) => "šŸš€", + Either::Right(PerformanceStatus::MinorImprovement) => "šŸ‘", + Either::Right(PerformanceStatus::NoChange) => "āœ…", + Either::Right(PerformanceStatus::SignificantRegression) => "āŒ", + Either::Right(PerformanceStatus::MinorRegression) => "āš ļø", + } +} + +fn format_number(num: u64) -> String { + let num_str = num.to_string(); + let mut result = String::new(); + + for (i, ch) in num_str.chars().rev().enumerate() { + if i > 0 && i % 3 == 0 { + result.push('_'); + } + result.push(ch); + } + + result.chars().rev().collect() +} + +fn add_comparison_to_report(report: &mut String, comparison: BenchCategoryComparison) { + let BenchCategoryComparison { + category, + current, + other, + diff, + diff_percent, + status, + } = comparison; + + let current = format_number(current); + let other = format_number(other); + let diff_sign = if diff >= 0 { "+" } else { "-" }; + let diff_percent_sign = if diff_percent >= 0.0 { "+" } else { "" }; + let diff = format_number(diff.unsigned_abs()); + let status_str = status_to_str(&status); + + report.push_str(&format!( + "| {category} | {current} | {other} | {diff_sign}{diff} | {diff_percent_sign}{diff_percent:.2}% | {status_str} |\n", + )); +} + +fn add_report_conclusion( + report: &mut String, + threshold: Option, + threshold_failed: Option, +) { + match threshold_failed { + Some(true) => { + let threshold = threshold.expect("threshold is required when threshold_failed is true"); + let err_str = format!("\nāŒ Benchmark threshold {threshold:.1}% check failed!\n"); + report.push_str(&err_str); + } + Some(false) => { + report.push_str("\nāœ… All benchmark differences are within acceptable thresholds."); + } + None => { + report.push_str("\n### Legend\n- šŸš€ Significant improvement (>5% reduction)\n- āœ… No significant change or minor improvement\n- āš ļø Minor regression (<5% increase)\n- āŒ Significant regression (>5% increase)\n"); + } + } +} diff --git a/benchmarks/src/bin/check_benchmark_diff.rs b/benchmarks/src/bin/check_benchmark_diff.rs deleted file mode 100644 index 59205f19a..000000000 --- a/benchmarks/src/bin/check_benchmark_diff.rs +++ /dev/null @@ -1,235 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::fs; -use std::path::Path; -use std::process; - -// todo [sab] check diff on negative values - -#[derive(Deserialize, Clone)] -struct BenchData { - compute: u64, - alloc: HashMap, - counter: HashMap, - cross_program: u64, - redirect: u64, -} - -#[derive(Serialize)] -struct DiffResult { - benchmark: String, - current: u64, - previous: u64, - diff_percent: f64, - exceeds_threshold: bool, -} - -fn calculate_diff_percent(current: u64, previous: u64) -> f64 { - if previous == 0 { - if current == 0 { - 0.0 - } else { - 100.0 // Consider any non-zero value as 100% increase from zero - } - } else { - ((current as f64 - previous as f64) / previous as f64) * 100.0 - } -} - -fn check_benchmark_value(name: String, current: u64, previous: u64, threshold: f64) -> DiffResult { - let diff_percent = calculate_diff_percent(current, previous); - let exceeds_threshold = diff_percent.abs() > threshold; - - DiffResult { - benchmark: name, - current, - previous, - diff_percent, - exceeds_threshold, - } -} - -fn main() -> Result<()> { - let args: Vec = env::args().collect(); - - if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { - println!("Benchmark Difference Checker"); - println!(); - println!("USAGE:"); - println!(" {} [CURRENT_FILE] [PREVIOUS_FILE] [THRESHOLD]", args[0]); - println!(); - println!("ARGUMENTS:"); - println!(" CURRENT_FILE Current benchmark data (default: bench_data.json)"); - println!(" PREVIOUS_FILE Previous benchmark data (default: bench_data_previous.json)"); - println!(" THRESHOLD Threshold percentage for failure (default: 1.0)"); - println!(); - println!("DESCRIPTION:"); - println!(" Compares current vs previous benchmark data and fails if any benchmark"); - println!(" differs by more than the threshold percentage. Exit code 0 = pass, 1 = fail."); - return Ok(()); - } - - let current_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); - let previous_file = args.get(2).unwrap_or(&"bench_data_previous.json".to_string()).clone(); - let threshold: f64 = args.get(3) - .and_then(|s| s.parse().ok()) - .unwrap_or(1.0); - - // Check if files exist - if !Path::new(¤t_file).exists() { - eprintln!("āŒ Current file '{}' does not exist", current_file); - process::exit(1); - } - - if !Path::new(&previous_file).exists() { - eprintln!("āš ļø Previous file '{}' does not exist - treating as first run", previous_file); - println!("āœ… No previous benchmarks to compare against. Skipping diff check."); - process::exit(0); - } - - // Read the files - let current_content = fs::read_to_string(¤t_file)?; - let previous_content = fs::read_to_string(&previous_file)?; - - let current_data: BenchData = serde_json::from_str(¤t_content)?; - let previous_data: BenchData = serde_json::from_str(&previous_content)?; - - let mut results = Vec::new(); - let mut has_failures = false; - - // Check compute - let diff = check_benchmark_value( - "compute".to_string(), - current_data.compute, - previous_data.compute, - threshold, - ); - if diff.exceeds_threshold { - has_failures = true; - } - results.push(diff); - - // Check alloc benchmarks - let mut alloc_keys: std::collections::HashSet = current_data.alloc.keys().cloned().collect(); - alloc_keys.extend(previous_data.alloc.keys().cloned()); - let mut alloc_keys: Vec<_> = alloc_keys.into_iter().collect(); - alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); - - for key in alloc_keys { - let current_val = current_data.alloc.get(&key).unwrap_or(&0); - let previous_val = previous_data.alloc.get(&key).unwrap_or(&0); - let diff = check_benchmark_value( - format!("alloc-{}", key), - *current_val, - *previous_val, - threshold, - ); - if diff.exceeds_threshold { - has_failures = true; - } - results.push(diff); - } - - // Check counter benchmarks - let mut counter_keys: std::collections::HashSet = current_data.counter.keys().cloned().collect(); - counter_keys.extend(previous_data.counter.keys().cloned()); - let counter_keys: Vec<_> = counter_keys.into_iter().collect(); - - for key in counter_keys { - let current_val = current_data.counter.get(&key).unwrap_or(&0); - let previous_val = previous_data.counter.get(&key).unwrap_or(&0); - let diff = check_benchmark_value( - format!("counter-{}", key), - *current_val, - *previous_val, - threshold, - ); - if diff.exceeds_threshold { - has_failures = true; - } - results.push(diff); - } - - // Check cross_program - let diff = check_benchmark_value( - "cross_program".to_string(), - current_data.cross_program, - previous_data.cross_program, - threshold, - ); - if diff.exceeds_threshold { - has_failures = true; - } - results.push(diff); - - // Check redirect - let diff = check_benchmark_value( - "redirect".to_string(), - current_data.redirect, - previous_data.redirect, - threshold, - ); - if diff.exceeds_threshold { - has_failures = true; - } - results.push(diff); - - // Print results - println!("šŸ” Benchmark Difference Analysis (threshold: {:.1}%)", threshold); - println!("ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); - - let mut passed = 0; - let mut failed = 0; - - for result in &results { - let status = if result.exceeds_threshold { - failed += 1; - "āŒ FAIL" - } else { - passed += 1; - "āœ… PASS" - }; - - let sign = if result.diff_percent >= 0.0 { "+" } else { "" }; - println!( - "{} | {:20} | {:>15} ā†’ {:>15} | {}{:>6.2}%", - status, - result.benchmark, - format_number(result.previous), - format_number(result.current), - sign, - result.diff_percent - ); - } - - println!("ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); - println!("šŸ“Š Summary: {} passed, {} failed", passed, failed); - - if has_failures { - println!(); - println!("āŒ BENCHMARK DIFF CHECK FAILED!"); - println!(" Some benchmarks differ by more than {:.1}% from the previous run.", threshold); - println!(" This indicates significant performance changes that need investigation."); - process::exit(1); - } else { - println!(); - println!("āœ… All benchmark differences are within acceptable threshold ({:.1}%)", threshold); - process::exit(0); - } -} - -fn format_number(num: u64) -> String { - let num_str = num.to_string(); - let mut result = String::new(); - - for (i, ch) in num_str.chars().rev().enumerate() { - if i > 0 && i % 3 == 0 { - result.push('_'); - } - result.push(ch); - } - - result.chars().rev().collect() -} diff --git a/benchmarks/src/bin/compare_benchmarks.rs b/benchmarks/src/bin/compare_benchmarks.rs deleted file mode 100644 index 7308de40e..000000000 --- a/benchmarks/src/bin/compare_benchmarks.rs +++ /dev/null @@ -1,221 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::fs; -use std::path::Path; - -#[derive(Deserialize, Clone)] -struct BenchData { - compute: u64, - alloc: HashMap, - counter: HashMap, - cross_program: u64, - redirect: u64, -} - -#[derive(Serialize)] -struct ComparisonResult { - name: String, - current: u64, - baseline: u64, - change: i64, - change_percent: f64, - status: String, -} - -fn format_gas(gas: u64) -> String { - let gas_str = gas.to_string(); - let mut result = String::new(); - - for (i, ch) in gas_str.chars().rev().enumerate() { - if i > 0 && i % 3 == 0 { - result.push('_'); - } - result.push(ch); - } - - result.chars().rev().collect() -} - -fn calculate_change_status(change_percent: f64) -> String { - if change_percent.abs() < 1.0 { - "āœ…".to_string() // No significant change - } else if change_percent < -5.0 { - "šŸš€".to_string() // Significant improvement - } else if change_percent < 0.0 { - "āœ…".to_string() // Minor improvement - } else if change_percent < 5.0 { - "āš ļø".to_string() // Minor regression - } else { - "āŒ".to_string() // Significant regression - } -} - -fn compare_values(name: String, current: u64, baseline: u64) -> ComparisonResult { - let change = current as i64 - baseline as i64; - let change_percent = if baseline > 0 { - (change as f64 / baseline as f64) * 100.0 - } else { - 0.0 - }; - let status = calculate_change_status(change_percent); - - ComparisonResult { - name, - current, - baseline, - change, - change_percent, - status, - } -} - -fn generate_markdown_table(comparisons: &[ComparisonResult]) -> String { - let mut markdown = String::new(); - - markdown.push_str("## šŸ”¬ Benchmark Comparison\n\n"); - markdown.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n"); - markdown.push_str("|-----------|---------|----------|---------|----------|--------|\n"); - - for comp in comparisons { - let change_sign = if comp.change >= 0 { "+" } else { "" }; - markdown.push_str(&format!( - "| {} | {} | {} | {}{} | {}{:.2}% | {} |\n", - comp.name, - format_gas(comp.current), - format_gas(comp.baseline), - change_sign, - format_gas(comp.change.abs() as u64), - if comp.change_percent >= 0.0 { "+" } else { "" }, - comp.change_percent, - comp.status - )); - } - - markdown.push_str("\n### Legend\n"); - markdown.push_str("- šŸš€ Significant improvement (>5% reduction)\n"); - markdown.push_str("- āœ… No significant change or minor improvement\n"); - markdown.push_str("- āš ļø Minor regression (<5% increase)\n"); - markdown.push_str("- āŒ Significant regression (>5% increase)\n"); - - markdown -} - -fn main() -> Result<()> { - let args: Vec = env::args().collect(); - - if args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) { - println!("Benchmark Comparison Tool"); - println!(); - println!("USAGE:"); - println!(" {} [CURRENT_FILE] [BASELINE_FILE] [OUTPUT_FILE]", args[0]); - println!(); - println!("ARGUMENTS:"); - println!(" CURRENT_FILE Current benchmark data (default: bench_data.json)"); - println!(" BASELINE_FILE Baseline benchmark data (default: baseline.json)"); - println!(" OUTPUT_FILE Output markdown file (default: comparison.md)"); - println!(); - println!("DESCRIPTION:"); - println!(" Compares two benchmark JSON files and generates a markdown table"); - println!(" showing the differences with status indicators."); - return Ok(()); - } - - let current_file = args.get(1).unwrap_or(&"bench_data.json".to_string()).clone(); - let baseline_file = args.get(2).unwrap_or(&"baseline.json".to_string()).clone(); - let output_file = args.get(3).unwrap_or(&"comparison.md".to_string()).clone(); - - // Read the files - if !Path::new(¤t_file).exists() { - eprintln!("Current file '{}' does not exist", current_file); - std::process::exit(1); - } - - if !Path::new(&baseline_file).exists() { - eprintln!("Baseline file '{}' does not exist", baseline_file); - std::process::exit(1); - } - - let current_content = fs::read_to_string(¤t_file)?; - let baseline_content = fs::read_to_string(&baseline_file)?; - - let current_data: BenchData = serde_json::from_str(¤t_content)?; - let baseline_data: BenchData = serde_json::from_str(&baseline_content)?; - - let mut comparisons = Vec::new(); - - // Compare compute - comparisons.push(compare_values( - "Compute".to_string(), - current_data.compute, - baseline_data.compute, - )); - - // Compare alloc benchmarks (get all keys from both datasets) - let mut alloc_keys: std::collections::HashSet = current_data.alloc.keys().cloned().collect(); - alloc_keys.extend(baseline_data.alloc.keys().cloned()); - let mut alloc_keys: Vec<_> = alloc_keys.into_iter().collect(); - alloc_keys.sort_by_key(|k| k.parse::().unwrap_or(0)); - - for key in alloc_keys { - let current_val = current_data.alloc.get(&key).unwrap_or(&0); - let baseline_val = baseline_data.alloc.get(&key).unwrap_or(&0); - comparisons.push(compare_values( - format!("alloc - {}", key), - *current_val, - *baseline_val, - )); - } - - // Compare counter benchmarks - let mut counter_keys: std::collections::HashSet = current_data.counter.keys().cloned().collect(); - counter_keys.extend(baseline_data.counter.keys().cloned()); - let counter_keys: Vec<_> = counter_keys.into_iter().collect(); - - for key in counter_keys { - let current_val = current_data.counter.get(&key).unwrap_or(&0); - let baseline_val = baseline_data.counter.get(&key).unwrap_or(&0); - comparisons.push(compare_values( - format!("counter - {}", key), - *current_val, - *baseline_val, - )); - } - - // Compare cross_program - comparisons.push(compare_values( - "cross_program".to_string(), - current_data.cross_program, - baseline_data.cross_program, - )); - - // Compare redirect - comparisons.push(compare_values( - "redirect".to_string(), - current_data.redirect, - baseline_data.redirect, - )); - - // Generate markdown table - let markdown = generate_markdown_table(&comparisons); - - // Write to file - fs::write(&output_file, &markdown)?; - - // Also output to stdout for GitHub Actions - println!("{}", markdown); - - // Summary - let total_benchmarks = comparisons.len(); - let improvements = comparisons.iter().filter(|c| c.change_percent < -1.0).count(); - let regressions = comparisons.iter().filter(|c| c.change_percent > 1.0).count(); - let no_change = total_benchmarks - improvements - regressions; - - println!("šŸ“Š **Summary**: {} total benchmarks - {} improvements, {} no significant change, {} regressions", - total_benchmarks, improvements, no_change, regressions); - - println!("\nComparison table written to '{}'", output_file); - - Ok(()) -} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs index 4fbb7745b..382ec28f5 100644 --- a/benchmarks/src/lib.rs +++ b/benchmarks/src/lib.rs @@ -14,14 +14,112 @@ use anyhow::{Context, Result}; use fs2::FileExt; use serde::{Deserialize, Serialize}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, btree_map::IntoIter as BTreeMapIntoIter}, env, + fmt::Display, fs::{File, OpenOptions}, io::{Read, Seek, SeekFrom, Write}, path::{Path, PathBuf}, }; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct BenchDataOuter(BTreeMap); + +impl BenchDataOuter { + pub fn from_json_str(str: &str) -> Result { + let data: BenchData = serde_json::from_str(str) + .context("Failed to deserialize `BenchData` from JSON string")?; + + let mut map = BTreeMap::new(); + map.insert(BenchCategory::Compute, data.compute); + for (key, value) in data.alloc { + map.insert(BenchCategory::Alloc(key), value); + } + map.insert(BenchCategory::CounterSync, data.counter.sync_call); + map.insert(BenchCategory::CounterAsync, data.counter.async_call); + map.insert(BenchCategory::CrossProgram, data.cross_program); + map.insert(BenchCategory::Redirect, data.redirect); + + Ok(Self(map)) + } + + pub fn update_compute_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::Compute, value); + } + + pub fn update_alloc_bench(&mut self, size: usize, value: u64) { + self.0.insert(BenchCategory::Alloc(size), value); + } + + pub fn update_counter_bench(&mut self, is_async: bool, value: u64) { + if is_async { + self.0.insert(BenchCategory::CounterAsync, value); + } else { + self.0.insert(BenchCategory::CounterSync, value); + } + } + + pub fn update_cross_program_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::CrossProgram, value); + } + + pub fn update_redirect_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::Redirect, value); + } + + pub fn into_json_string(self) -> Result { + let mut bench_data = BenchData::default(); + for (key, value) in self.0 { + // match statement is crucial for not missing any new added category + match key { + BenchCategory::Compute => bench_data.compute = value, + BenchCategory::Alloc(size) => { + bench_data.alloc.insert(size, value); + } + BenchCategory::CounterSync => bench_data.counter.sync_call = value, + BenchCategory::CounterAsync => bench_data.counter.async_call = value, + BenchCategory::CrossProgram => bench_data.cross_program = value, + BenchCategory::Redirect => bench_data.redirect = value, + } + } + + serde_json::to_string_pretty(&bench_data) + .context("Failed to serialize `BenchData` to JSON string") + } +} + +impl IntoIterator for BenchDataOuter { + type Item = (BenchCategory, u64); + type IntoIter = BTreeMapIntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum BenchCategory { + Compute, + Alloc(usize), + CounterSync, + CounterAsync, + CrossProgram, + Redirect, +} + +impl Display for BenchCategory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BenchCategory::Compute => write!(f, "compute"), + BenchCategory::Alloc(size) => write!(f, "alloc-{size}"), + BenchCategory::CounterSync => write!(f, "counter_sync"), + BenchCategory::CounterAsync => write!(f, "counter_async"), + BenchCategory::CrossProgram => write!(f, "cross_program"), + BenchCategory::Redirect => write!(f, "redirect"), + } + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct BenchData { pub compute: u64, pub alloc: BTreeMap, @@ -30,54 +128,97 @@ pub struct BenchData { pub redirect: u64, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct BenchDataFile(File); + +impl BenchDataFile { + pub fn open(path: impl AsRef) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .context("Failed to open or create bench data file")?; + + Ok(Self(file)) + } + + pub fn lock_exclusive(&mut self) -> Result<()> { + self.0 + .lock_exclusive() + .context("Failed to lock bench data file for writing") + } + + pub fn unlock(&self) -> Result<()> { + ::unlock(&self.0).context("Failed to unlock bench data file") + } + + pub fn read_bench_data(&mut self) -> Result { + let mut content = String::new(); + self.0 + .read_to_string(&mut content) + .context("Failed reading bench data bytes to string")?; + let bench_data = + BenchDataOuter::from_json_str(&content).context("Failed to deserialize bench data")?; + + Ok(bench_data) + } + + pub fn update_bench_data(&mut self, updated: BenchDataOuter) -> Result<()> { + // Serialize back + let bench_data_string = updated + .into_json_string() + .context("Failed to serialize updated bench data")?; + + // Write updated bench data + self.0.set_len(0).context("Failed to erase file content")?; + self.0 + .seek(SeekFrom::Start(0)) + .context("Failed to seek to the start of the file")?; + self.0 + .write_all(bench_data_string.as_bytes()) + .context("Failed to write serialized bench data to file")?; + self.0 + .flush() + .context("Failed to flush bench data to file")?; + + Ok(()) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct CounterBenchData { pub async_call: u64, pub sync_call: u64, } -pub fn store_bench_data(f: impl FnOnce(&mut BenchData)) -> Result<()> { +pub fn store_bench_data(f: impl FnOnce(&mut BenchDataOuter)) -> Result<()> { let path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("bench_data.json"); store_bench_data_to_file(path, f) } -fn store_bench_data_to_file(path: impl AsRef, f: impl FnOnce(&mut BenchData)) -> Result<()> { - // Open file - let mut file = OpenOptions::new() - .read(true) - .write(true) - .open(path) - .context("Failed to open or create bench data file")?; +fn store_bench_data_to_file( + path: impl AsRef, + f: impl FnOnce(&mut BenchDataOuter), +) -> Result<()> { + let mut file = BenchDataFile::open(path).context("Failed to create `BenchDataFile`")?; - // Lock file file.lock_exclusive().unwrap_or_else(|e| { panic!("Failed to lock bench data file for writing: {e}"); }); - // Read bench data - let mut content = String::new(); - file.read_to_string(&mut content) - .context("Failed reading bench data bytes to string")?; - let mut bench_data = - serde_json::from_str(&content).context("Failed to deserialize bench data")?; + let mut bench_data = file + .read_bench_data() + .context("Failed to read existing bench data")?; // Handle bench data f(&mut bench_data); - // Serialize back - let bench_data_string = serde_json::to_string_pretty(&bench_data)?; - - // Write updated bench data - file.set_len(0).context("Failed to erase file content")?; - file.seek(SeekFrom::Start(0)) - .context("Failed to seek to the start of the file")?; - file.write_all(bench_data_string.as_bytes()) - .context("Failed to write serialized bench data to file")?; - file.flush().context("Failed to flush bench data to file")?; + file.update_bench_data(bench_data) + .context("Failed to update bench data")?; - // Unlock file - ::unlock(&file).context("Failed to unlock bench data file") + // Unlock the file + file.unlock() + .context("Failed to unlock bench data file after writing") } #[cfg(test)] From 40060eec32989025dc76b63ca8fb938ccb537a29 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 10:06:30 +0300 Subject: [PATCH 32/50] clean-up refactoring for bench-analyzer --- benchmarks/src/bin/bench_analyzer.rs | 163 +++------------ benchmarks/src/entities.rs | 288 +++++++++++++++++++++++++++ benchmarks/src/file.rs | 72 +++++++ benchmarks/src/lib.rs | 221 +++----------------- 4 files changed, 420 insertions(+), 324 deletions(-) create mode 100644 benchmarks/src/entities.rs create mode 100644 benchmarks/src/file.rs diff --git a/benchmarks/src/bin/bench_analyzer.rs b/benchmarks/src/bin/bench_analyzer.rs index 7abe80833..35c3b9e56 100644 --- a/benchmarks/src/bin/bench_analyzer.rs +++ b/benchmarks/src/bin/bench_analyzer.rs @@ -1,14 +1,16 @@ use anyhow::{Context, Result, anyhow}; -use benchmarks::{BenchCategory, BenchDataFile, BenchDataOuter}; +use benchmarks::{ + BenchCategoryComparison, BenchCategoryComparisonReport, BenchData, BenchDataFile, +}; use clap::Parser; -use itertools::Either; -use std::fs; -use std::path::PathBuf; +use std::{fs, path::PathBuf}; #[derive(Parser)] #[command(version, about, long_about = None)] #[command(name = "bench-analyzer")] -#[command(about = "A tool for analyzing benchmark data differences and comparisons")] +#[command( + about = "A tool for analyzing benchmark data by comparing current and previous benchmark results." +)] struct Cli { /// Current benchmark data file #[arg(long)] @@ -30,27 +32,27 @@ struct Cli { fn main() -> Result<()> { let cli = Cli::parse(); - compare_bench(cli.current, cli.other, cli.output, cli.threshold) + analyze_benches(cli.current, cli.other, cli.output, cli.threshold) } -fn compare_bench( +fn analyze_benches( current: PathBuf, other: PathBuf, report_output: Option, threshold: Option, ) -> Result<()> { + // Get benches data from the provided files. let (current_data, other_data) = get_bench_data(current, other)?; - let mut report = String::new(); - report.push_str("## šŸ”¬ Benchmark Comparison\n\n"); - report.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n"); - report.push_str("|-----------|---------|----------|---------|----------|--------|\n"); - + // Flag to track if any benchmarks fail the threshold check. let mut threshold_failed = threshold.map(|_| false); + + // Create unfinished report. let mut report = current_data .into_iter() .zip(other_data) .map( + // Create a comparison entity for each benchmark category. |((current_category, current_value), (other_category, other_value))| { assert_eq!(current_category, other_category, "Categories do not match"); @@ -68,20 +70,24 @@ fn compare_bench( comparison }, ) - .fold(report, |mut report, comparison| { + .fold(initialize_report(), |mut report, comparison| { + // Add each comparison to the report. add_comparison_to_report(&mut report, comparison); report }); + // Finish the report. add_report_conclusion(&mut report, threshold, threshold_failed); - // Printing the final report + // Printing the finalized report. println!("{report}"); + // If any benchmarks failed the threshold check, return an error. if matches!(threshold_failed, Some(true)) { return Err(anyhow!("Benchmark contains tests failing the threshold.")); } + // If an output path is provided, write the report to that file. if let Some(report_output) = report_output { fs::write(&report_output, &report).context("Failed to write report output")?; @@ -94,7 +100,7 @@ fn compare_bench( Ok(()) } -fn get_bench_data(current: PathBuf, previous: PathBuf) -> Result<(BenchDataOuter, BenchDataOuter)> { +fn get_bench_data(current: PathBuf, previous: PathBuf) -> Result<(BenchData, BenchData)> { let mut current_file = BenchDataFile::open(current).context("Failed to open current benchmark data file")?; let mut previous_file = @@ -106,133 +112,28 @@ fn get_bench_data(current: PathBuf, previous: PathBuf) -> Result<(BenchDataOuter Ok((current_data, previous_data)) } -#[derive(Debug)] -struct BenchCategoryComparison { - category: BenchCategory, - current: u64, - other: u64, - diff: i64, - diff_percent: f64, - status: Either, -} - -impl BenchCategoryComparison { - fn new( - category: BenchCategory, - current: u64, - other: u64, - maybe_threshold: Option, - ) -> Self { - let diff = current as i64 - other as i64; - let diff_percent = (diff as f64 / other as f64) * 100.0; - let status = match maybe_threshold { - Some(threshold) => { - let exceeds = diff_percent.abs() > threshold; - if exceeds { - Either::Left(ThresholdPassStatus::Fail) - } else { - Either::Left(ThresholdPassStatus::Pass) - } - } - None => { - if diff_percent.abs() < 1.0 { - // [0,..1.0) - Either::Right(PerformanceStatus::NoChange) - } else if diff_percent < -5.0 { - // [-inf, -5.0) - Either::Right(PerformanceStatus::SignificantImprovement) - } else if diff_percent < 0.0 { - // [-5.0, 0.0) - Either::Right(PerformanceStatus::MinorImprovement) - } else if diff_percent < 5.0 { - // [0.0, 5.0) - Either::Right(PerformanceStatus::MinorRegression) - } else { - // [5.0, inf) - Either::Right(PerformanceStatus::SignificantRegression) - } - } - }; - - Self { - category, - current, - other, - diff, - diff_percent, - status, - } - } - - fn has_failed_threshold(&self) -> bool { - self.status - .as_ref() - .left() - .map(|status| matches!(status, ThresholdPassStatus::Fail)) - .unwrap_or(false) - } -} - -#[derive(Debug, Clone, Copy)] -enum ThresholdPassStatus { - Pass, - Fail, -} - -#[derive(Debug, Clone, Copy)] -enum PerformanceStatus { - SignificantImprovement, - MinorImprovement, - NoChange, - SignificantRegression, - MinorRegression, -} - -fn status_to_str(status: &Either) -> &'static str { - match status { - Either::Left(ThresholdPassStatus::Pass) => "āœ… PASS", - Either::Left(ThresholdPassStatus::Fail) => "āŒ FAIL", - Either::Right(PerformanceStatus::SignificantImprovement) => "šŸš€", - Either::Right(PerformanceStatus::MinorImprovement) => "šŸ‘", - Either::Right(PerformanceStatus::NoChange) => "āœ…", - Either::Right(PerformanceStatus::SignificantRegression) => "āŒ", - Either::Right(PerformanceStatus::MinorRegression) => "āš ļø", - } -} - -fn format_number(num: u64) -> String { - let num_str = num.to_string(); - let mut result = String::new(); - - for (i, ch) in num_str.chars().rev().enumerate() { - if i > 0 && i % 3 == 0 { - result.push('_'); - } - result.push(ch); - } +fn initialize_report() -> String { + let mut report = String::new(); + report.push_str("## šŸ”¬ Benchmark Comparison\n\n"); + report.push_str("| Benchmark | Current | Baseline | Change | Change % | Status |\n"); + report.push_str("|-----------|---------|----------|---------|----------|--------|\n"); - result.chars().rev().collect() + report } fn add_comparison_to_report(report: &mut String, comparison: BenchCategoryComparison) { - let BenchCategoryComparison { + let BenchCategoryComparisonReport { category, current, other, + diff_sign, diff, + diff_percent_sign, diff_percent, status, - } = comparison; - - let current = format_number(current); - let other = format_number(other); - let diff_sign = if diff >= 0 { "+" } else { "-" }; - let diff_percent_sign = if diff_percent >= 0.0 { "+" } else { "" }; - let diff = format_number(diff.unsigned_abs()); - let status_str = status_to_str(&status); - + } = comparison.into(); report.push_str(&format!( - "| {category} | {current} | {other} | {diff_sign}{diff} | {diff_percent_sign}{diff_percent:.2}% | {status_str} |\n", + "| {category} | {current} | {other} | {diff_sign}{diff} | {diff_percent_sign}{diff_percent:.2}% | {status} |\n", )); } diff --git a/benchmarks/src/entities.rs b/benchmarks/src/entities.rs new file mode 100644 index 000000000..722d86f30 --- /dev/null +++ b/benchmarks/src/entities.rs @@ -0,0 +1,288 @@ +use anyhow::{Context, Result}; +use itertools::Either; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, btree_map::IntoIter as BTreeMapIntoIter}, + fmt::Display, +}; + +/// A collection holding benchmark data categorized by [`BenchCategory`]. +pub struct BenchData(BTreeMap); + +impl BenchData { + /// Creates a new `BenchData` instance from a JSON string. + pub fn from_json_str(str: &str) -> Result { + let data: BenchDataSerde = serde_json::from_str(str) + .context("Failed to deserialize `BenchData` from JSON string")?; + + let mut map = BTreeMap::new(); + map.insert(BenchCategory::Compute, data.compute); + for (key, value) in data.alloc { + map.insert(BenchCategory::Alloc(key), value); + } + map.insert(BenchCategory::CounterSync, data.counter.sync_call); + map.insert(BenchCategory::CounterAsync, data.counter.async_call); + map.insert(BenchCategory::CrossProgram, data.cross_program); + map.insert(BenchCategory::Redirect, data.redirect); + + Ok(Self(map)) + } + + /// Update compute benchmark category value. + pub fn update_compute_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::Compute, value); + } + + /// Update allocation benchmark category value. + pub fn update_alloc_bench(&mut self, size: usize, value: u64) { + self.0.insert(BenchCategory::Alloc(size), value); + } + + /// Update counter benchmark category value. + pub fn update_counter_bench(&mut self, is_async: bool, value: u64) { + if is_async { + self.0.insert(BenchCategory::CounterAsync, value); + } else { + self.0.insert(BenchCategory::CounterSync, value); + } + } + + /// Update cross-program benchmark category value. + pub fn update_cross_program_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::CrossProgram, value); + } + + /// Update redirect benchmark category value. + pub fn update_redirect_bench(&mut self, value: u64) { + self.0.insert(BenchCategory::Redirect, value); + } + + /// Convert the benchmark data into a JSON string. + pub fn into_json_string(self) -> Result { + let mut bench_data = BenchDataSerde::default(); + for (key, value) in self.0 { + // match statement is crucial for not missing any new added category + match key { + BenchCategory::Compute => bench_data.compute = value, + BenchCategory::Alloc(size) => { + bench_data.alloc.insert(size, value); + } + BenchCategory::CounterSync => bench_data.counter.sync_call = value, + BenchCategory::CounterAsync => bench_data.counter.async_call = value, + BenchCategory::CrossProgram => bench_data.cross_program = value, + BenchCategory::Redirect => bench_data.redirect = value, + } + } + + serde_json::to_string_pretty(&bench_data) + .context("Failed to serialize `BenchData` to JSON string") + } +} + +impl IntoIterator for BenchData { + type Item = (BenchCategory, u64); + type IntoIter = BTreeMapIntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// Benchmark data stored in the benchmarks file. +/// +/// This struct is used to serialize and deserialize benchmark data +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct BenchDataSerde { + pub compute: u64, + pub alloc: BTreeMap, + pub counter: CounterBenchDataSerde, + pub cross_program: u64, + pub redirect: u64, +} + +/// Counter test benchmark data stored in the benchmarks file. +/// +/// This struct is used to serialize and deserialize benchmark data +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct CounterBenchDataSerde { + pub async_call: u64, + pub sync_call: u64, +} + +/// Benchmark category that can be read (written) from (to) the benchmarks file. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum BenchCategory { + Compute, + Alloc(usize), + CounterSync, + CounterAsync, + CrossProgram, + Redirect, +} + +impl Display for BenchCategory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BenchCategory::Compute => write!(f, "compute"), + BenchCategory::Alloc(size) => write!(f, "alloc-{size}"), + BenchCategory::CounterSync => write!(f, "counter_sync"), + BenchCategory::CounterAsync => write!(f, "counter_async"), + BenchCategory::CrossProgram => write!(f, "cross_program"), + BenchCategory::Redirect => write!(f, "redirect"), + } + } +} + +/// Comparison entity for benchmark categories. +#[derive(Debug)] +pub struct BenchCategoryComparison { + category: BenchCategory, + current: u64, + other: u64, + diff: i64, + diff_percent: f64, + status: Either, +} + +impl BenchCategoryComparison { + pub fn new( + category: BenchCategory, + current: u64, + other: u64, + maybe_threshold: Option, + ) -> Self { + let diff = current as i64 - other as i64; + let diff_percent = (diff as f64 / other as f64) * 100.0; + let status = match maybe_threshold { + Some(threshold) => { + let exceeds = diff_percent.abs() > threshold; + if exceeds { + Either::Left(ThresholdPassStatus::Fail) + } else { + Either::Left(ThresholdPassStatus::Pass) + } + } + None => { + if diff_percent.abs() < 1.0 { + // [0,..1.0) + Either::Right(PerformanceStatus::NoChange) + } else if diff_percent < -5.0 { + // [-inf, -5.0) + Either::Right(PerformanceStatus::SignificantImprovement) + } else if diff_percent < 0.0 { + // [-5.0, 0.0) + Either::Right(PerformanceStatus::MinorImprovement) + } else if diff_percent < 5.0 { + // [0.0, 5.0) + Either::Right(PerformanceStatus::MinorRegression) + } else { + // [5.0, inf) + Either::Right(PerformanceStatus::SignificantRegression) + } + } + }; + + Self { + category, + current, + other, + diff, + diff_percent, + status, + } + } + + pub fn has_failed_threshold(&self) -> bool { + self.status + .as_ref() + .left() + .map(|status| matches!(status, ThresholdPassStatus::Fail)) + .unwrap_or(false) + } +} + +#[derive(Debug, Clone, Copy)] +enum ThresholdPassStatus { + Pass, + Fail, +} + +#[derive(Debug, Clone, Copy)] +enum PerformanceStatus { + SignificantImprovement, + MinorImprovement, + NoChange, + SignificantRegression, + MinorRegression, +} + +/// Report structure for benchmark category comparison. +/// +/// This struct is a placeholder to formatted benchmark comparison data. +/// The formatted data is later decided on a client side how to be displayed. +pub struct BenchCategoryComparisonReport { + pub category: String, + pub current: String, + pub other: String, + pub diff_sign: &'static str, + pub diff: String, + pub diff_percent_sign: &'static str, + pub diff_percent: f64, + pub status: &'static str, +} + +impl From for BenchCategoryComparisonReport { + fn from(comparison: BenchCategoryComparison) -> Self { + let category = comparison.category.to_string(); + let current = Self::format_number(comparison.current); + let other = Self::format_number(comparison.other); + let diff_sign = if comparison.diff >= 0 { "+" } else { "-" }; + let diff = Self::format_number(comparison.diff.unsigned_abs()); + let diff_percent_sign = if comparison.diff_percent >= 0.0 { + "+" + } else { + "" + }; + let diff_percent = comparison.diff_percent; + let status = Self::status_to_str(&comparison.status); + + BenchCategoryComparisonReport { + category, + current, + other, + diff_sign, + diff, + diff_percent_sign, + diff_percent, + status, + } + } +} + +impl BenchCategoryComparisonReport { + fn format_number(num: u64) -> String { + let num_str = num.to_string(); + let mut result = String::new(); + + for (i, ch) in num_str.chars().rev().enumerate() { + if i > 0 && i % 3 == 0 { + result.push('_'); + } + result.push(ch); + } + + result.chars().rev().collect() + } + + fn status_to_str(status: &Either) -> &'static str { + match status { + Either::Left(ThresholdPassStatus::Pass) => "āœ… PASS", + Either::Left(ThresholdPassStatus::Fail) => "āŒ FAIL", + Either::Right(PerformanceStatus::SignificantImprovement) => "šŸš€", + Either::Right(PerformanceStatus::MinorImprovement) => "šŸ‘", + Either::Right(PerformanceStatus::NoChange) => "āœ…", + Either::Right(PerformanceStatus::SignificantRegression) => "āŒ", + Either::Right(PerformanceStatus::MinorRegression) => "āš ļø", + } + } +} diff --git a/benchmarks/src/file.rs b/benchmarks/src/file.rs new file mode 100644 index 000000000..d379ef8ac --- /dev/null +++ b/benchmarks/src/file.rs @@ -0,0 +1,72 @@ +use crate::BenchData; +use anyhow::{Context, Result}; +use fs2::FileExt; +use std::{ + fs::{File, OpenOptions}, + io::{Read, Seek, SeekFrom, Write}, + path::Path, +}; + +/// A file that holds benchmark data. +pub struct BenchDataFile(File); + +impl BenchDataFile { + /// Opens a benchmark data file. + /// + /// If the file does not exist, the function fails. + pub fn open(path: impl AsRef) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .context("Failed to open or create bench data file")?; + + Ok(Self(file)) + } + + /// Locks the file for exclusive access. + pub fn lock_exclusive(&mut self) -> Result<()> { + self.0 + .lock_exclusive() + .context("Failed to lock bench data file for writing") + } + + /// Unlocks the file after exclusive access. + pub fn unlock(&self) -> Result<()> { + ::unlock(&self.0).context("Failed to unlock bench data file") + } + + /// Reads the benchmark data from the file. + pub fn read_bench_data(&mut self) -> Result { + let mut content = String::new(); + self.0 + .read_to_string(&mut content) + .context("Failed reading bench data bytes to string")?; + let bench_data = + BenchData::from_json_str(&content).context("Failed to deserialize bench data")?; + + Ok(bench_data) + } + + /// Converts the benchmark data into a JSON string and writes it to the file. + pub fn write_bench_data(&mut self, data: BenchData) -> Result<()> { + // Serialize back + let bench_data_string = data + .into_json_string() + .context("Failed to serialize updated bench data")?; + + // Write updated bench data + self.0.set_len(0).context("Failed to erase file content")?; + self.0 + .seek(SeekFrom::Start(0)) + .context("Failed to seek to the start of the file")?; + self.0 + .write_all(bench_data_string.as_bytes()) + .context("Failed to write serialized bench data to file")?; + self.0 + .flush() + .context("Failed to flush bench data to file")?; + + Ok(()) + } +} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs index 382ec28f5..973d8cbce 100644 --- a/benchmarks/src/lib.rs +++ b/benchmarks/src/lib.rs @@ -10,196 +10,26 @@ mod benchmarks; #[cfg(all(test, not(debug_assertions)))] mod clients; +mod entities; +mod file; + use anyhow::{Context, Result}; -use fs2::FileExt; -use serde::{Deserialize, Serialize}; +pub use entities::{ + BenchCategory, BenchCategoryComparison, BenchCategoryComparisonReport, BenchData, +}; +pub use file::BenchDataFile; use std::{ - collections::{BTreeMap, btree_map::IntoIter as BTreeMapIntoIter}, env, - fmt::Display, - fs::{File, OpenOptions}, - io::{Read, Seek, SeekFrom, Write}, path::{Path, PathBuf}, }; -pub struct BenchDataOuter(BTreeMap); - -impl BenchDataOuter { - pub fn from_json_str(str: &str) -> Result { - let data: BenchData = serde_json::from_str(str) - .context("Failed to deserialize `BenchData` from JSON string")?; - - let mut map = BTreeMap::new(); - map.insert(BenchCategory::Compute, data.compute); - for (key, value) in data.alloc { - map.insert(BenchCategory::Alloc(key), value); - } - map.insert(BenchCategory::CounterSync, data.counter.sync_call); - map.insert(BenchCategory::CounterAsync, data.counter.async_call); - map.insert(BenchCategory::CrossProgram, data.cross_program); - map.insert(BenchCategory::Redirect, data.redirect); - - Ok(Self(map)) - } - - pub fn update_compute_bench(&mut self, value: u64) { - self.0.insert(BenchCategory::Compute, value); - } - - pub fn update_alloc_bench(&mut self, size: usize, value: u64) { - self.0.insert(BenchCategory::Alloc(size), value); - } - - pub fn update_counter_bench(&mut self, is_async: bool, value: u64) { - if is_async { - self.0.insert(BenchCategory::CounterAsync, value); - } else { - self.0.insert(BenchCategory::CounterSync, value); - } - } - - pub fn update_cross_program_bench(&mut self, value: u64) { - self.0.insert(BenchCategory::CrossProgram, value); - } - - pub fn update_redirect_bench(&mut self, value: u64) { - self.0.insert(BenchCategory::Redirect, value); - } - - pub fn into_json_string(self) -> Result { - let mut bench_data = BenchData::default(); - for (key, value) in self.0 { - // match statement is crucial for not missing any new added category - match key { - BenchCategory::Compute => bench_data.compute = value, - BenchCategory::Alloc(size) => { - bench_data.alloc.insert(size, value); - } - BenchCategory::CounterSync => bench_data.counter.sync_call = value, - BenchCategory::CounterAsync => bench_data.counter.async_call = value, - BenchCategory::CrossProgram => bench_data.cross_program = value, - BenchCategory::Redirect => bench_data.redirect = value, - } - } - - serde_json::to_string_pretty(&bench_data) - .context("Failed to serialize `BenchData` to JSON string") - } -} - -impl IntoIterator for BenchDataOuter { - type Item = (BenchCategory, u64); - type IntoIter = BTreeMapIntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub enum BenchCategory { - Compute, - Alloc(usize), - CounterSync, - CounterAsync, - CrossProgram, - Redirect, -} - -impl Display for BenchCategory { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BenchCategory::Compute => write!(f, "compute"), - BenchCategory::Alloc(size) => write!(f, "alloc-{size}"), - BenchCategory::CounterSync => write!(f, "counter_sync"), - BenchCategory::CounterAsync => write!(f, "counter_async"), - BenchCategory::CrossProgram => write!(f, "cross_program"), - BenchCategory::Redirect => write!(f, "redirect"), - } - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct BenchData { - pub compute: u64, - pub alloc: BTreeMap, - pub counter: CounterBenchData, - pub cross_program: u64, - pub redirect: u64, -} - -pub struct BenchDataFile(File); - -impl BenchDataFile { - pub fn open(path: impl AsRef) -> Result { - let file = OpenOptions::new() - .read(true) - .write(true) - .open(path) - .context("Failed to open or create bench data file")?; - - Ok(Self(file)) - } - - pub fn lock_exclusive(&mut self) -> Result<()> { - self.0 - .lock_exclusive() - .context("Failed to lock bench data file for writing") - } - - pub fn unlock(&self) -> Result<()> { - ::unlock(&self.0).context("Failed to unlock bench data file") - } - - pub fn read_bench_data(&mut self) -> Result { - let mut content = String::new(); - self.0 - .read_to_string(&mut content) - .context("Failed reading bench data bytes to string")?; - let bench_data = - BenchDataOuter::from_json_str(&content).context("Failed to deserialize bench data")?; - - Ok(bench_data) - } - - pub fn update_bench_data(&mut self, updated: BenchDataOuter) -> Result<()> { - // Serialize back - let bench_data_string = updated - .into_json_string() - .context("Failed to serialize updated bench data")?; - - // Write updated bench data - self.0.set_len(0).context("Failed to erase file content")?; - self.0 - .seek(SeekFrom::Start(0)) - .context("Failed to seek to the start of the file")?; - self.0 - .write_all(bench_data_string.as_bytes()) - .context("Failed to write serialized bench data to file")?; - self.0 - .flush() - .context("Failed to flush bench data to file")?; - - Ok(()) - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct CounterBenchData { - pub async_call: u64, - pub sync_call: u64, -} - -pub fn store_bench_data(f: impl FnOnce(&mut BenchDataOuter)) -> Result<()> { +pub fn store_bench_data(f: impl FnOnce(&mut BenchData)) -> Result<()> { let path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("bench_data.json"); store_bench_data_to_file(path, f) } -fn store_bench_data_to_file( - path: impl AsRef, - f: impl FnOnce(&mut BenchDataOuter), -) -> Result<()> { +fn store_bench_data_to_file(path: impl AsRef, f: impl FnOnce(&mut BenchData)) -> Result<()> { let mut file = BenchDataFile::open(path).context("Failed to create `BenchDataFile`")?; file.lock_exclusive().unwrap_or_else(|e| { @@ -213,7 +43,8 @@ fn store_bench_data_to_file( // Handle bench data f(&mut bench_data); - file.update_bench_data(bench_data) + // Write updated bench data. + file.write_bench_data(bench_data) .context("Failed to update bench data")?; // Unlock the file @@ -224,16 +55,20 @@ fn store_bench_data_to_file( #[cfg(test)] mod tests { use super::*; - use std::thread; + use crate::entities::{BenchDataSerde, CounterBenchDataSerde}; + use std::{ + io::{Read, Seek, SeekFrom, Write}, + thread, + }; use tempfile::NamedTempFile; #[test] fn test_data_not_overwritten() { // Create initial bench data. - let initial_bench_data = BenchData { + let initial_bench_data = BenchDataSerde { compute: 123, - alloc: BTreeMap::new(), - counter: CounterBenchData { + alloc: Default::default(), + counter: CounterBenchDataSerde { async_call: 53, sync_call: 35, }, @@ -260,17 +95,17 @@ mod tests { // Spawn two threads to modify the bench data concurrently. let h1 = thread::spawn(move || { store_bench_data_to_file(path_h1, |bench_data| { - bench_data.compute = 42; - bench_data.cross_program = 0; + bench_data.update_compute_bench(42); + bench_data.update_cross_program_bench(0); }) .unwrap(); }); let h2 = thread::spawn(move || { store_bench_data_to_file(path_h2, |bench_data| { - bench_data.counter.async_call = 84; - bench_data.counter.sync_call = 126; - bench_data.redirect = 4343; + bench_data.update_counter_bench(true, 84); + bench_data.update_counter_bench(false, 126); + bench_data.update_redirect_bench(4343); }) .unwrap(); }); @@ -285,16 +120,16 @@ mod tests { .as_file_mut() .read_to_string(&mut content) .expect("Failed reading bench data bytes to string"); - let bench_data: BenchData = + let bench_data: BenchDataSerde = serde_json::from_str(&content).expect("Failed to deserialize bench data"); // Check that the bench data was modified correctly. assert_eq!( bench_data, - BenchData { + BenchDataSerde { compute: 42, - alloc: BTreeMap::new(), - counter: CounterBenchData { + alloc: Default::default(), + counter: CounterBenchDataSerde { async_call: 84, sync_call: 126, }, From f6e434cdc59a68fbeba1901d810f783a0a2058e9 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:17:51 +0300 Subject: [PATCH 33/50] clean-up the job --- .github/workflows/rs-bench.yml | 186 ++++++++++++--------------- Makefile | 3 + benchmarks/baseline.json | 19 --- benchmarks/bench_data.json | 4 +- benchmarks/bench_data_previous.json | 19 --- benchmarks/comparison.md | 23 ---- benchmarks/counter-bench/src/lib.rs | 13 +- benchmarks/src/benchmarks.rs | 10 +- benchmarks/src/bin/bench_analyzer.rs | 2 +- 9 files changed, 101 insertions(+), 178 deletions(-) delete mode 100644 benchmarks/baseline.json delete mode 100644 benchmarks/bench_data_previous.json delete mode 100644 benchmarks/comparison.md diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index cbaf81be8..396ba80de 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -2,14 +2,16 @@ name: '[rs] Benchmarks' on: pull_request: - # types: [labeled] - # paths: - # - 'rs/**' - # - 'Cargo.lock' - # - 'Cargo.toml' + types: [labeled] + paths: + - 'benchmarks/**' + - 'rs/**' + - 'Cargo.lock' + - 'Cargo.toml' push: branches: [master] paths: + - 'benchmarks/**' - 'rs/**' - 'Cargo.lock' - 'Cargo.toml' @@ -27,11 +29,8 @@ jobs: pull-requests: write issues: write steps: - # Checkout the current branch (PR branch) - name: Checkout PR branch uses: actions/checkout@v4 - with: - fetch-depth: 2 - name: Free Disk Space uses: ./.github/actions/free-disk-space @@ -39,112 +38,91 @@ jobs: - name: Install wasm-opt uses: ./.github/actions/install-wasm-utils - # Run benchmarks on PR branch - - name: Run Benchmarks + - name: Build bench-analyzer run: | - make bench + make build-bench-analyzer -# # Step 1: Check if current benchmarks differ significantly from previous ones in current branch -# # todo [sab] the approach is buggy, because if current commit updates benches, it will still fail. -# - name: Check Benchmark Diff vs Current Branch -# if: github.event_name == 'pull_request' -# run: | -# # Get previous bench_data.json from current branch (before the latest commit) -# git fetch origin ${{ github.head_ref }} -# if git show HEAD~1:benchmarks/bench_data.json > bench_data_previous.json 2>/dev/null; then -# echo "Found previous benchmark data in current branch" -# cd benchmarks -# cargo run --bin bench-analyzer -- check-diff --current bench_data.json --previous ../bench_data_previous.json --threshold 1.0 -# else -# echo "No previous benchmark data found in current branch - treating as first run" -# fi -# env: -# CARGO_TERM_COLOR: always + - name: Copy current benchmarks for a diff test + run: | + cp benchmarks/bench_data.json benchmarks/bench_data_before_bench_run.json -# # Step 2: If diff check passes, compare with master baseline -# - name: Checkout master branch for baseline -# if: github.event_name == 'pull_request' -# uses: actions/checkout@v4 -# with: -# ref: master -# path: master-branch + - name: Run benchmarks + run: | + make bench -# # Copy baseline JSON from master -# - name: Copy Baseline JSON from Master -# if: github.event_name == 'pull_request' -# run: | -# cp master-branch/benchmarks/bench_data.json baseline.json + # The check is done with a threshold test + - name: Check current branch has actual benchmark data + if: github.event_name == 'pull_request' + run: | + ./target/debug/bench-analyzer --current=benchmarks/bench_data.json --other=benchmarks/bench_data_before_bench_run.json --threshold=1 -# # todo [sab] remove -# - name: Display Bench Data -# if: github.event_name == 'pull_request' -# run: | -# echo "=== Current PR Bench Data ===" -# cat benchmarks/bench_data.json -# echo "" -# echo "=== Master Baseline Bench Data ===" -# cat baseline.json + # If diff check passes, compare with master baseline + # First copy baseline JSON from master branch + - name: Copy baseline benchmarks from master branch + if: github.event_name == 'pull_request' + run: | + git fetch origin master:refs/remotes/origin/master + git show origin/master:benchmarks/bench_data.json > benchmarks/baseline.json -# # Compare benchmarks and generate markdown table -# - name: Compare Benchmarks vs Master -# if: github.event_name == 'pull_request' -# run: | -# cd benchmarks -# cargo run --bin bench-analyzer -- compare --current bench_data.json --baseline ../baseline.json --output comparison.md -# env: -# CARGO_TERM_COLOR: always + # Now compare benchmarks and generate markdown table + - name: Compare Benchmarks vs Master + if: github.event_name == 'pull_request' + run: | + ./target/debug/bench-analyzer --current=benchmarks/bench_data.json --other=benchmarks/baseline.json --output=benchmarks/comparison.md + env: + CARGO_TERM_COLOR: always -# # Read the comparison markdown for the comment -# - name: Read Comparison Result -# if: github.event_name == 'pull_request' -# id: comparison -# run: | -# cd benchmarks -# echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT -# cat comparison.md >> $GITHUB_OUTPUT -# echo 'EOF' >> $GITHUB_OUTPUT + # Read the comparison markdown for the comment + - name: Read Comparison Result + if: github.event_name == 'pull_request' + id: comparison + run: | + cd benchmarks + echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT + cat benchmarks/comparison.md >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT -# # todo [sab] write a new comment, not updated (edited) -# # Comment the comparison table on the PR -# - name: Comment PR with Benchmark Comparison -# if: github.event_name == 'pull_request' -# uses: actions/github-script@v7 -# with: -# github-token: ${{ secrets.GITHUB_TOKEN }} -# script: | -# const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; + # todo [sab] write a new comment, not updated (edited) + # Comment the comparison table on the PR + - name: Comment PR with Benchmark Comparison + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; -# // Find existing benchmark comment -# const comments = await github.rest.issues.listComments({ -# owner: context.repo.owner, -# repo: context.repo.repo, -# issue_number: context.issue.number, -# }); + // Find existing benchmark comment + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); -# const botComment = comments.data.find(comment => -# comment.user.type === 'Bot' && -# comment.body.includes('šŸ”¬ Benchmark Comparison') -# ); + const botComment = comments.data.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ”¬ Benchmark Comparison') + ); -# const commentBody = `${comparisonTable} + const commentBody = `${comparisonTable} -# --- -# šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; + --- + šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; -# if (botComment) { -# // Update existing comment -# await github.rest.issues.updateComment({ -# owner: context.repo.owner, -# repo: context.repo.repo, -# comment_id: botComment.id, -# body: commentBody -# }); -# } else { -# // Create new comment -# await github.rest.issues.createComment({ -# owner: context.repo.owner, -# repo: context.repo.repo, -# issue_number: context.issue.number, -# body: commentBody -# }); -# } + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody + }); + } diff --git a/Makefile b/Makefile index 98f32a86c..b92ded98c 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ clippy: bench: @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo test --release --manifest-path=benchmarks/Cargo.toml +build-bench-analyzer: + @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo run --bin bench-analyzer + build-parser: @echo "Building idlparser" @cargo build -p sails-idl-parser --target=wasm32-unknown-unknown --release diff --git a/benchmarks/baseline.json b/benchmarks/baseline.json deleted file mode 100644 index 1c214476d..000000000 --- a/benchmarks/baseline.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "compute": 450513691329, - "alloc": { - "0": 563795739, - "12": 567302331, - "143": 726670926, - "986": 827656693, - "10945": 2018421219, - "46367": 6403682702, - "121392": 16858374460, - "317810": 43146693797 - }, - "counter": { - "async_call": 850482107, - "sync_call": 677860234 - }, - "cross_program": 2511573058, - "redirect": 3612969613 -} \ No newline at end of file diff --git a/benchmarks/bench_data.json b/benchmarks/bench_data.json index 56ab4c433..1c214476d 100644 --- a/benchmarks/bench_data.json +++ b/benchmarks/bench_data.json @@ -11,8 +11,8 @@ "317810": 43146693797 }, "counter": { - "async_call": 699056814, - "sync_call": 43471993060 + "async_call": 850482107, + "sync_call": 677860234 }, "cross_program": 2511573058, "redirect": 3612969613 diff --git a/benchmarks/bench_data_previous.json b/benchmarks/bench_data_previous.json deleted file mode 100644 index 1c214476d..000000000 --- a/benchmarks/bench_data_previous.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "compute": 450513691329, - "alloc": { - "0": 563795739, - "12": 567302331, - "143": 726670926, - "986": 827656693, - "10945": 2018421219, - "46367": 6403682702, - "121392": 16858374460, - "317810": 43146693797 - }, - "counter": { - "async_call": 850482107, - "sync_call": 677860234 - }, - "cross_program": 2511573058, - "redirect": 3612969613 -} \ No newline at end of file diff --git a/benchmarks/comparison.md b/benchmarks/comparison.md deleted file mode 100644 index a952615cf..000000000 --- a/benchmarks/comparison.md +++ /dev/null @@ -1,23 +0,0 @@ -## šŸ”¬ Benchmark Comparison - -| Benchmark | Current | Baseline | Change | Change % | Status | -|-----------|---------|----------|---------|----------|--------| -| Compute | 450_513_691_329 | 450_513_691_329 | +0 | +0.00% | āœ… | -| alloc - 0 | 563_795_739 | 563_795_739 | +0 | +0.00% | āœ… | -| alloc - 12 | 567_302_331 | 567_302_331 | +0 | +0.00% | āœ… | -| alloc - 143 | 726_670_926 | 726_670_926 | +0 | +0.00% | āœ… | -| alloc - 986 | 827_656_693 | 827_656_693 | +0 | +0.00% | āœ… | -| alloc - 10945 | 2_018_421_219 | 2_018_421_219 | +0 | +0.00% | āœ… | -| alloc - 46367 | 6_403_682_702 | 6_403_682_702 | +0 | +0.00% | āœ… | -| alloc - 121392 | 16_858_374_460 | 16_858_374_460 | +0 | +0.00% | āœ… | -| alloc - 317810 | 43_146_693_797 | 43_146_693_797 | +0 | +0.00% | āœ… | -| counter - async_call | 699_056_814 | 850_482_107 | 151_425_293 | -17.80% | šŸš€ | -| counter - sync_call | 43_471_993_060 | 677_860_234 | +42_794_132_826 | +6313.12% | āŒ | -| cross_program | 2_511_573_058 | 2_511_573_058 | +0 | +0.00% | āœ… | -| redirect | 3_612_969_613 | 3_612_969_613 | +0 | +0.00% | āœ… | - -### Legend -- šŸš€ Significant improvement (>5% reduction) -- āœ… No significant change or minor improvement -- āš ļø Minor regression (<5% increase) -- āŒ Significant regression (>5% increase) diff --git a/benchmarks/counter-bench/src/lib.rs b/benchmarks/counter-bench/src/lib.rs index a989795f0..42e3bf1d0 100644 --- a/benchmarks/counter-bench/src/lib.rs +++ b/benchmarks/counter-bench/src/lib.rs @@ -10,15 +10,18 @@ pub struct CounterBenchService; impl CounterBenchService { #[export] pub fn inc(&mut self) -> u64 { - let mut data: Vec = vec![]; - for _ in 0..317810 { - data.push(0); - } + // todo [sab] + // let mut data: Vec = vec![]; + // for _ in 0..317810 { + // data.push(0); + // } unsafe { let prev = COUNTER; COUNTER += 1; + + prev } - data.len() as u64 + // data.len() as u64 } #[export] diff --git a/benchmarks/src/benchmarks.rs b/benchmarks/src/benchmarks.rs index 69fea2b09..03028e712 100644 --- a/benchmarks/src/benchmarks.rs +++ b/benchmarks/src/benchmarks.rs @@ -207,21 +207,21 @@ async fn counter_bench() { let (remoting, pid) = create_program_async!((CounterBenchProgramFactory::, wasm_path)); - // let mut expected_value = 0; + let mut expected_value = 0; let (mut gas_benches_sync, mut gas_benches_async): (Vec<_>, Vec<_>) = (0..100) .enumerate() .map(|(i, _)| { let is_sync = i % 2 == 0; let gas = if is_sync { let (stress_resp, gas_sync_inc) = call_action!(remoting, pid, Inc); - // assert_eq!(stress_resp, expected_value); - // expected_value += 1; + assert_eq!(stress_resp, expected_value); + expected_value += 1; gas_sync_inc } else { let (stress_resp, gas_async_inc) = call_action!(remoting, pid, IncAsync); - // assert_eq!(stress_resp, expected_value); - // expected_value += 1; + assert_eq!(stress_resp, expected_value); + expected_value += 1; gas_async_inc }; diff --git a/benchmarks/src/bin/bench_analyzer.rs b/benchmarks/src/bin/bench_analyzer.rs index 35c3b9e56..504e80c1a 100644 --- a/benchmarks/src/bin/bench_analyzer.rs +++ b/benchmarks/src/bin/bench_analyzer.rs @@ -152,7 +152,7 @@ fn add_report_conclusion( report.push_str("\nāœ… All benchmark differences are within acceptable thresholds."); } None => { - report.push_str("\n### Legend\n- šŸš€ Significant improvement (>5% reduction)\n- āœ… No significant change or minor improvement\n- āš ļø Minor regression (<5% increase)\n- āŒ Significant regression (>5% increase)\n"); + report.push_str("\n### Legend\n- šŸš€ Significant improvement (>5% reduction)\n- šŸ‘ Minor improvement (<5% reduction)\n- āœ… No significant change\n- āš ļø Minor regression (<5% increase)\n- āŒ Significant regression (>5% increase)\n"); } } } From 793f53d183992e5e25c6ad87d56ec5112aa4eea4 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:31:56 +0300 Subject: [PATCH 34/50] fix bench-analyzer build, add label check to the job --- .github/workflows/rs-bench.yml | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 396ba80de..cc44c9e47 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -2,7 +2,7 @@ name: '[rs] Benchmarks' on: pull_request: - types: [labeled] + types: [opened, synchronized, reopened, labeled] paths: - 'benchmarks/**' - 'rs/**' @@ -22,7 +22,7 @@ env: jobs: benchmark: - # if: contains(github.event.label.name, 'run-benchmarks') + if: contains(github.event.label.name, 'run-benchmarks') runs-on: ubuntu-latest permissions: contents: write diff --git a/Makefile b/Makefile index b92ded98c..d5844f9cf 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ bench: @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo test --release --manifest-path=benchmarks/Cargo.toml build-bench-analyzer: - @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo run --bin bench-analyzer + @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo build --bin bench-analyzer build-parser: @echo "Building idlparser" From f983c354f1ef8dd37e32a30a19ccfbf67bb77d9b Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:32:38 +0300 Subject: [PATCH 35/50] fmt --- benchmarks/src/entities.rs | 2 +- benchmarks/src/file.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/src/entities.rs b/benchmarks/src/entities.rs index 722d86f30..110a5d814 100644 --- a/benchmarks/src/entities.rs +++ b/benchmarks/src/entities.rs @@ -217,7 +217,7 @@ enum PerformanceStatus { } /// Report structure for benchmark category comparison. -/// +/// /// This struct is a placeholder to formatted benchmark comparison data. /// The formatted data is later decided on a client side how to be displayed. pub struct BenchCategoryComparisonReport { diff --git a/benchmarks/src/file.rs b/benchmarks/src/file.rs index d379ef8ac..fcbd24748 100644 --- a/benchmarks/src/file.rs +++ b/benchmarks/src/file.rs @@ -12,7 +12,7 @@ pub struct BenchDataFile(File); impl BenchDataFile { /// Opens a benchmark data file. - /// + /// /// If the file does not exist, the function fails. pub fn open(path: impl AsRef) -> Result { let file = OpenOptions::new() From 0cc787e35fca9580f48668da5a429d7cc461f2c2 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:35:18 +0300 Subject: [PATCH 36/50] fix job --- .github/workflows/rs-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index cc44c9e47..07af9dcd8 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -22,7 +22,7 @@ env: jobs: benchmark: - if: contains(github.event.label.name, 'run-benchmarks') + if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest permissions: contents: write From 2e8a7e83328c914c6da6638e290ffc093d56ae25 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:38:50 +0300 Subject: [PATCH 37/50] check with no paths --- .github/workflows/rs-bench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 07af9dcd8..89acf51ec 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -3,11 +3,11 @@ name: '[rs] Benchmarks' on: pull_request: types: [opened, synchronized, reopened, labeled] - paths: - - 'benchmarks/**' - - 'rs/**' - - 'Cargo.lock' - - 'Cargo.toml' + # paths: + # - 'benchmarks/**' + # - 'rs/**' + # - 'Cargo.lock' + # - 'Cargo.toml' push: branches: [master] paths: From 42a3996c2138b6a8aefde47a30bae936e438fb3e Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:44:22 +0300 Subject: [PATCH 38/50] add paths --- .github/workflows/rs-bench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 89acf51ec..07af9dcd8 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -3,11 +3,11 @@ name: '[rs] Benchmarks' on: pull_request: types: [opened, synchronized, reopened, labeled] - # paths: - # - 'benchmarks/**' - # - 'rs/**' - # - 'Cargo.lock' - # - 'Cargo.toml' + paths: + - 'benchmarks/**' + - 'rs/**' + - 'Cargo.lock' + - 'Cargo.toml' push: branches: [master] paths: From 6b30e3afef7ce3b1605e31e16e1442e88d001456 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 12:45:37 +0300 Subject: [PATCH 39/50] trigger benches on changing the benchmarks crate --- benchmarks/counter-bench/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/counter-bench/src/lib.rs b/benchmarks/counter-bench/src/lib.rs index 42e3bf1d0..f4466cc09 100644 --- a/benchmarks/counter-bench/src/lib.rs +++ b/benchmarks/counter-bench/src/lib.rs @@ -10,7 +10,7 @@ pub struct CounterBenchService; impl CounterBenchService { #[export] pub fn inc(&mut self) -> u64 { - // todo [sab] + // todo [sab] remove comments // let mut data: Vec = vec![]; // for _ in 0..317810 { // data.push(0); From 1a864700ff7d837035aa483688ab90b5cddb797c Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 13:25:20 +0300 Subject: [PATCH 40/50] fix event type --- .github/workflows/rs-bench.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 07af9dcd8..749907e66 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -2,12 +2,13 @@ name: '[rs] Benchmarks' on: pull_request: - types: [opened, synchronized, reopened, labeled] + types: [opened, synchronize, reopened, labeled] paths: - 'benchmarks/**' - 'rs/**' - 'Cargo.lock' - 'Cargo.toml' + - '.github/workflows/rs-bench.yml' push: branches: [master] paths: @@ -15,6 +16,7 @@ on: - 'rs/**' - 'Cargo.lock' - 'Cargo.toml' + - '.github/workflows/rs-bench.yml' env: CARGO_TERM_COLOR: always From 8d26933c738bddc9c42d402b165df389c0ba300a Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 13:38:28 +0300 Subject: [PATCH 41/50] empty commit must not trigger CI, because didn't change paths From 388b2dad93b95c3d3b1623fa8cacdd3137e3d01e Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 13:41:23 +0300 Subject: [PATCH 42/50] test not triggered --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d5844f9cf..fb0bbe7be 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ bench: build-bench-analyzer: @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo build --bin bench-analyzer - +# test not triggered build-parser: @echo "Building idlparser" @cargo build -p sails-idl-parser --target=wasm32-unknown-unknown --release From 594e3eac54c6b25e3dfa2879c565086822164185 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 13:45:08 +0300 Subject: [PATCH 43/50] fix job --- .github/workflows/rs-bench.yml | 6 ++++-- Makefile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 749907e66..f43557463 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -9,6 +9,7 @@ on: - 'Cargo.lock' - 'Cargo.toml' - '.github/workflows/rs-bench.yml' + - 'examples/**' push: branches: [master] paths: @@ -17,6 +18,7 @@ on: - 'Cargo.lock' - 'Cargo.toml' - '.github/workflows/rs-bench.yml' + - 'examples/**' env: CARGO_TERM_COLOR: always @@ -24,7 +26,8 @@ env: jobs: benchmark: - if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') + if: (github.event_name == 'push') || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) runs-on: ubuntu-latest permissions: contents: write @@ -79,7 +82,6 @@ jobs: if: github.event_name == 'pull_request' id: comparison run: | - cd benchmarks echo 'COMPARISON_TABLE<> $GITHUB_OUTPUT cat benchmarks/comparison.md >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT diff --git a/Makefile b/Makefile index fb0bbe7be..d5844f9cf 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ bench: build-bench-analyzer: @__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING=1 cargo build --bin bench-analyzer -# test not triggered + build-parser: @echo "Building idlparser" @cargo build -p sails-idl-parser --target=wasm32-unknown-unknown --release From 229cd08c823668641f6af09a306d79bb2a53220c Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 14:12:41 +0300 Subject: [PATCH 44/50] no label commit From 1521fde6a126682fa2f7c3d6c6aa48755d94c568 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 14:13:18 +0300 Subject: [PATCH 45/50] no label commit2 From 4d06ef5d3c15039ea0ebce69e8abb5906fa6ea56 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 14:15:14 +0300 Subject: [PATCH 46/50] add not benched changes --- benchmarks/counter-bench/src/lib.rs | 13 ++++++------- benchmarks/src/benchmarks.rs | 10 +++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/benchmarks/counter-bench/src/lib.rs b/benchmarks/counter-bench/src/lib.rs index f4466cc09..e3f8519c2 100644 --- a/benchmarks/counter-bench/src/lib.rs +++ b/benchmarks/counter-bench/src/lib.rs @@ -10,18 +10,17 @@ pub struct CounterBenchService; impl CounterBenchService { #[export] pub fn inc(&mut self) -> u64 { - // todo [sab] remove comments - // let mut data: Vec = vec![]; - // for _ in 0..317810 { - // data.push(0); - // } + let mut data: Vec = vec![]; + for _ in 0..317810 { + data.push(0); + } unsafe { let prev = COUNTER; COUNTER += 1; - prev + // prev } - // data.len() as u64 + data.len() as u64 } #[export] diff --git a/benchmarks/src/benchmarks.rs b/benchmarks/src/benchmarks.rs index 03028e712..69fea2b09 100644 --- a/benchmarks/src/benchmarks.rs +++ b/benchmarks/src/benchmarks.rs @@ -207,21 +207,21 @@ async fn counter_bench() { let (remoting, pid) = create_program_async!((CounterBenchProgramFactory::, wasm_path)); - let mut expected_value = 0; + // let mut expected_value = 0; let (mut gas_benches_sync, mut gas_benches_async): (Vec<_>, Vec<_>) = (0..100) .enumerate() .map(|(i, _)| { let is_sync = i % 2 == 0; let gas = if is_sync { let (stress_resp, gas_sync_inc) = call_action!(remoting, pid, Inc); - assert_eq!(stress_resp, expected_value); - expected_value += 1; + // assert_eq!(stress_resp, expected_value); + // expected_value += 1; gas_sync_inc } else { let (stress_resp, gas_async_inc) = call_action!(remoting, pid, IncAsync); - assert_eq!(stress_resp, expected_value); - expected_value += 1; + // assert_eq!(stress_resp, expected_value); + // expected_value += 1; gas_async_inc }; From 369788e9f5190d7577c2dfed8046130247338566 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 14:28:05 +0300 Subject: [PATCH 47/50] re-bench --- benchmarks/bench_data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/bench_data.json b/benchmarks/bench_data.json index 1c214476d..56ab4c433 100644 --- a/benchmarks/bench_data.json +++ b/benchmarks/bench_data.json @@ -11,8 +11,8 @@ "317810": 43146693797 }, "counter": { - "async_call": 850482107, - "sync_call": 677860234 + "async_call": 699056814, + "sync_call": 43471993060 }, "cross_program": 2511573058, "redirect": 3612969613 From 994dfe57690e60f80187cf9d2e904a4d5d9d14d5 Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 15:05:37 +0300 Subject: [PATCH 48/50] always post benches, adjust counter-bench and re-bench --- .github/workflows/rs-bench.yml | 39 ++++++----------------------- benchmarks/bench_data.json | 4 +-- benchmarks/counter-bench/src/lib.rs | 7 +----- benchmarks/src/benchmarks.rs | 10 ++++---- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index f43557463..7b8a9254d 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -86,7 +86,6 @@ jobs: cat benchmarks/comparison.md >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT - # todo [sab] write a new comment, not updated (edited) # Comment the comparison table on the PR - name: Comment PR with Benchmark Comparison if: github.event_name == 'pull_request' @@ -96,37 +95,15 @@ jobs: script: | const comparisonTable = `${{ steps.comparison.outputs.COMPARISON_TABLE }}`; - // Find existing benchmark comment - const comments = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const botComment = comments.data.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('šŸ”¬ Benchmark Comparison') - ); - const commentBody = `${comparisonTable} --- - šŸ¤– This comment was automatically generated by the benchmark comparison workflow.`; + šŸ¤– This comment was automatically generated by the benchmark comparison workflow at ${{ github.sha }}.`; - if (botComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: commentBody - }); - } else { - // Create new comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: commentBody - }); - } + // Always create a new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody + }); diff --git a/benchmarks/bench_data.json b/benchmarks/bench_data.json index 56ab4c433..1c214476d 100644 --- a/benchmarks/bench_data.json +++ b/benchmarks/bench_data.json @@ -11,8 +11,8 @@ "317810": 43146693797 }, "counter": { - "async_call": 699056814, - "sync_call": 43471993060 + "async_call": 850482107, + "sync_call": 677860234 }, "cross_program": 2511573058, "redirect": 3612969613 diff --git a/benchmarks/counter-bench/src/lib.rs b/benchmarks/counter-bench/src/lib.rs index e3f8519c2..8744a82a3 100644 --- a/benchmarks/counter-bench/src/lib.rs +++ b/benchmarks/counter-bench/src/lib.rs @@ -10,17 +10,12 @@ pub struct CounterBenchService; impl CounterBenchService { #[export] pub fn inc(&mut self) -> u64 { - let mut data: Vec = vec![]; - for _ in 0..317810 { - data.push(0); - } unsafe { let prev = COUNTER; COUNTER += 1; - // prev + prev } - data.len() as u64 } #[export] diff --git a/benchmarks/src/benchmarks.rs b/benchmarks/src/benchmarks.rs index 69fea2b09..03028e712 100644 --- a/benchmarks/src/benchmarks.rs +++ b/benchmarks/src/benchmarks.rs @@ -207,21 +207,21 @@ async fn counter_bench() { let (remoting, pid) = create_program_async!((CounterBenchProgramFactory::, wasm_path)); - // let mut expected_value = 0; + let mut expected_value = 0; let (mut gas_benches_sync, mut gas_benches_async): (Vec<_>, Vec<_>) = (0..100) .enumerate() .map(|(i, _)| { let is_sync = i % 2 == 0; let gas = if is_sync { let (stress_resp, gas_sync_inc) = call_action!(remoting, pid, Inc); - // assert_eq!(stress_resp, expected_value); - // expected_value += 1; + assert_eq!(stress_resp, expected_value); + expected_value += 1; gas_sync_inc } else { let (stress_resp, gas_async_inc) = call_action!(remoting, pid, IncAsync); - // assert_eq!(stress_resp, expected_value); - // expected_value += 1; + assert_eq!(stress_resp, expected_value); + expected_value += 1; gas_async_inc }; From a6de3c7b3dfeb9cce65b736f0d82d989ed65e83b Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Thu, 14 Aug 2025 15:26:55 +0300 Subject: [PATCH 49/50] trigger CI From 19e7d82ce2a05662d524e7ebfe479df08b49ff1a Mon Sep 17 00:00:00 2001 From: Sabaun Taraki Date: Mon, 18 Aug 2025 20:13:38 +0300 Subject: [PATCH 50/50] remove redundant cargo term color setting --- .github/workflows/rs-bench.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/rs-bench.yml b/.github/workflows/rs-bench.yml index 7b8a9254d..51e10e145 100644 --- a/.github/workflows/rs-bench.yml +++ b/.github/workflows/rs-bench.yml @@ -74,8 +74,6 @@ jobs: if: github.event_name == 'pull_request' run: | ./target/debug/bench-analyzer --current=benchmarks/bench_data.json --other=benchmarks/baseline.json --output=benchmarks/comparison.md - env: - CARGO_TERM_COLOR: always # Read the comparison markdown for the comment - name: Read Comparison Result