From 88b3470efbf383e0d6f0ac9f13e17513ac9d1a94 Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 15:39:03 +0000 Subject: [PATCH 1/6] fix: bencher --- .github/workflows/snapshot_benchmark_main.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/snapshot_benchmark_main.yml b/.github/workflows/snapshot_benchmark_main.yml index 283ffbdb2c..eae6ea2d61 100644 --- a/.github/workflows/snapshot_benchmark_main.yml +++ b/.github/workflows/snapshot_benchmark_main.yml @@ -1,6 +1,6 @@ on: push: - branches: main + branches: ** jobs: benchmark_base_branch: @@ -19,7 +19,7 @@ jobs: bencher run \ --project bms \ --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch main \ + --branch test \ --testbed ubuntu-latest \ --threshold-measure latency \ --threshold-test t_test \ @@ -28,5 +28,7 @@ jobs: --thresholds-reset \ --err \ --adapter json \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - bencher run --adapter rust_criterion "cargo bench --features lua54" \ No newline at end of file + --github-actions '${{ secrets.GITHUB_TOKEN }}' + env: + BENCHER_CMD: | + bencher run --adapter rust_criterion "cargo bench --features lua54" \ No newline at end of file From 9dbfa9735b74a9d1b3764afb1ce2d7b083f8473e Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 15:40:15 +0000 Subject: [PATCH 2/6] try something --- .github/workflows/snapshot_benchmark_main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapshot_benchmark_main.yml b/.github/workflows/snapshot_benchmark_main.yml index eae6ea2d61..caf7380ecb 100644 --- a/.github/workflows/snapshot_benchmark_main.yml +++ b/.github/workflows/snapshot_benchmark_main.yml @@ -1,6 +1,6 @@ on: push: - branches: ** + branches: "fix/bencher" jobs: benchmark_base_branch: @@ -19,7 +19,7 @@ jobs: bencher run \ --project bms \ --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch test \ + --branch "fix/bencher" \ --testbed ubuntu-latest \ --threshold-measure latency \ --threshold-test t_test \ From 92ac98683c73248cd38c68f553cb98e21958d19b Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 15:47:54 +0000 Subject: [PATCH 3/6] properly fix --- .github/workflows/snapshot_benchmark_main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapshot_benchmark_main.yml b/.github/workflows/snapshot_benchmark_main.yml index caf7380ecb..23b3681f40 100644 --- a/.github/workflows/snapshot_benchmark_main.yml +++ b/.github/workflows/snapshot_benchmark_main.yml @@ -27,8 +27,8 @@ jobs: --threshold-upper-boundary 0.99 \ --thresholds-reset \ --err \ - --adapter json \ + --adapter rust_criterion \ --github-actions '${{ secrets.GITHUB_TOKEN }}' env: BENCHER_CMD: | - bencher run --adapter rust_criterion "cargo bench --features lua54" \ No newline at end of file + cargo bench --features lua54 \ No newline at end of file From 5e8981abac94b267669903439cc74c5c277f316e Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 17:30:42 +0000 Subject: [PATCH 4/6] move bencher check into matrix --- .github/workflows/snapshot_benchmark_main.yml | 34 -------- crates/xtask/src/main.rs | 85 +++++++++++++++++++ 2 files changed, 85 insertions(+), 34 deletions(-) delete mode 100644 .github/workflows/snapshot_benchmark_main.yml diff --git a/.github/workflows/snapshot_benchmark_main.yml b/.github/workflows/snapshot_benchmark_main.yml deleted file mode 100644 index 23b3681f40..0000000000 --- a/.github/workflows/snapshot_benchmark_main.yml +++ /dev/null @@ -1,34 +0,0 @@ -on: - push: - branches: "fix/bencher" - -jobs: - benchmark_base_branch: - name: Continuous Benchmarking with Bencher - permissions: - checks: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run Xtask initializer - run: | - cargo xtask init - - uses: bencherdev/bencher@main - - name: Track base branch benchmarks with Bencher - run: | - bencher run \ - --project bms \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch "fix/bencher" \ - --testbed ubuntu-latest \ - --threshold-measure latency \ - --threshold-test t_test \ - --threshold-max-sample-size 64 \ - --threshold-upper-boundary 0.99 \ - --thresholds-reset \ - --err \ - --adapter rust_criterion \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' - env: - BENCHER_CMD: | - cargo bench --features lua54 \ No newline at end of file diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 77713de1c9..91674d5aa0 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -8,6 +8,7 @@ use std::{ collections::{HashMap, HashSet}, ffi::{OsStr, OsString}, io::Write, + os::unix::process::CommandExt, path::{Path, PathBuf}, process::Command, str::FromStr, @@ -349,6 +350,9 @@ impl App { Xtasks::Install { binary } => { cmd.arg("install").arg(binary.as_ref()); } + Xtasks::Bench {} => { + cmd.arg("bench"); + } } cmd @@ -634,6 +638,8 @@ enum Xtasks { /// ] /// CiMatrix, + /// Runs bencher in dry mode by default if not on the main branch + Bench {}, } #[derive(Serialize, Clone)] @@ -709,6 +715,7 @@ impl Xtasks { bevy_features, } => Self::codegen(app_settings, output_dir, bevy_features), Xtasks::Install { binary } => Self::install(app_settings, binary), + Xtasks::Bench {} => Self::bench(app_settings), }?; Ok("".into()) @@ -1208,6 +1215,77 @@ impl Xtasks { Ok(()) } + fn bench(app_settings: GlobalArgs) -> Result<()> { + // first of all figure out which branch we're on + // run // git rev-parse --abbrev-ref HEAD + + let command = Command::new("git") + .args(["rev-parse", "--abbrev-ref", "HEAD"]) + .current_dir(Self::workspace_dir(&app_settings).unwrap()) + .output() + .with_context(|| "Trying to figure out which branch we're on in benchmarking")?; + let branch = String::from_utf8(command.stdout)?; + + let is_main = branch.trim() == "main"; + + // figure out if we're running in github actions + let github_token = std::env::var("GITHUB_TOKEN").ok(); + + // get testbed + // we want this to be a combination of + // is_github_ci? + // OS + // machine id + + let os = std::env::consts::OS; + + let testbed = format!( + "{os}{}", + github_token.is_some().then_some("-gha").unwrap_or_default() + ); + + // also figure out if we're on a fork + + let token = std::env::var("BENCHER_API_TOKEN").ok(); + + let mut bencher_cmd = Command::new("bencher"); + bencher_cmd + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .current_dir(Self::workspace_dir(&app_settings).unwrap()) + .arg("run") + .args(["--project", "bms"]) + .args(["--branch", &format!("\"{branch}\"")]) + .args(["--token", &token.unwrap_or_default()]) + .args(["--testbed", &testbed]) + .args(["--build-time"]) + .args(["--threshold-measure", "latency"]) + .args(["--threshold-test", "t_test"]) + .args(["--threshold-max-sample-size", "64"]) + .args(["--threshold-upper-boundary", "0.99"]) + .args(["--thresholds-reset"]) + .args(["--err"]); + + if let Some(token) = github_token { + bencher_cmd.args(["--github-actions", &token]); + } + + if !is_main { + bencher_cmd.args(["--dry-run"]); + } + + bencher_cmd + .args(["--adapter", "rust_criterion"]) + .arg("cargo bench --features=lua54"); + + let out = bencher_cmd.output()?; + if !out.status.success() { + bail!("Failed to run bencher: {:?}", out); + } + + Ok(()) + } + fn set_cargo_coverage_settings() { // This makes local dev hell // std::env::set_var("CARGO_INCREMENTAL", "0"); @@ -1369,6 +1447,13 @@ impl Xtasks { }, }); + // also run a benchmark + // on non-main branches this will just dry run + output.push(App { + global_args: default_args.clone(), + subcmd: Xtasks::Bench {}, + }); + // and finally run tests with coverage output.push(App { global_args: default_args From 7723bdba96ada24ce99e16c99401a1eef24e534c Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 17:36:57 +0000 Subject: [PATCH 5/6] remove unused import --- .github/workflows/bevy_mod_scripting.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/bevy_mod_scripting.yml b/.github/workflows/bevy_mod_scripting.yml index 4b4e3490c0..e5afebca0c 100644 --- a/.github/workflows/bevy_mod_scripting.yml +++ b/.github/workflows/bevy_mod_scripting.yml @@ -72,6 +72,9 @@ jobs: echo "Convert to single line JSON" jq -c . matrix.json > matrix-one-line.json echo "matrix=$(cat matrix-one-line.json)" >> $GITHUB_OUTPUT +env: + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} check: needs: [check-needs-run, generate-job-matrix] From 2aa9dd02733f252c400ab7ec187fc61e5420bec6 Mon Sep 17 00:00:00 2001 From: makspll Date: Sat, 22 Mar 2025 17:37:06 +0000 Subject: [PATCH 6/6] remove unused import --- crates/xtask/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 91674d5aa0..72bfcba346 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -8,7 +8,6 @@ use std::{ collections::{HashMap, HashSet}, ffi::{OsStr, OsString}, io::Write, - os::unix::process::CommandExt, path::{Path, PathBuf}, process::Command, str::FromStr,