diff --git a/assets/benchmarks/function/call.lua b/assets/benchmarks/function/call.lua new file mode 100644 index 0000000000..21b351ac09 --- /dev/null +++ b/assets/benchmarks/function/call.lua @@ -0,0 +1,3 @@ +function bench() + noop() +end \ No newline at end of file diff --git a/assets/benchmarks/function/call.rhai b/assets/benchmarks/function/call.rhai new file mode 100644 index 0000000000..5adf764732 --- /dev/null +++ b/assets/benchmarks/function/call.rhai @@ -0,0 +1,3 @@ +fn bench(){ + noop.call(); +} \ No newline at end of file diff --git a/assets/benchmarks/function/call_4_args.lua b/assets/benchmarks/function/call_4_args.lua new file mode 100644 index 0000000000..e595a6d465 --- /dev/null +++ b/assets/benchmarks/function/call_4_args.lua @@ -0,0 +1,3 @@ +function bench() + noop_4_args(1,"asd",{1,2,3}, {asd = 1}) +end \ No newline at end of file diff --git a/assets/benchmarks/function/call_4_args.rhai b/assets/benchmarks/function/call_4_args.rhai new file mode 100644 index 0000000000..299b35871f --- /dev/null +++ b/assets/benchmarks/function/call_4_args.rhai @@ -0,0 +1,3 @@ +fn bench(){ + noop_4_args.call(1,"asd",[1,2,3],#{ asd: 1}); +} \ No newline at end of file diff --git a/assets/benchmarks/math/vec_mat_ops.lua b/assets/benchmarks/math/vec_mat_ops.lua new file mode 100644 index 0000000000..218a9ef060 --- /dev/null +++ b/assets/benchmarks/math/vec_mat_ops.lua @@ -0,0 +1,20 @@ + +reseed() + +local matrix = nil +local vector = nil +function pre_bench() + -- generate random 3x3 matrix and 3x1 vec + vector = Vec3.new(random(1,999), random(1,999), random(1,999)) + matrix = Mat3.from_cols( + Vec3.new(random(1,999), random(1,999), random(1,999)), + Vec3.new(random(1,999), random(1,999), random(1,999)), + Vec3.new(random(1,999), random(1,999), random(1,999)) + ) +end + +function bench() + local mul = matrix * vector + local add = matrix + vector + local div = vector / matrix +end \ No newline at end of file diff --git a/assets/benchmarks/math/vec_mat_ops.rhai b/assets/benchmarks/math/vec_mat_ops.rhai new file mode 100644 index 0000000000..b204d1c5f2 --- /dev/null +++ b/assets/benchmarks/math/vec_mat_ops.rhai @@ -0,0 +1,19 @@ + +reseed.call(); + +let matrix = (); +let vector = (); +fn pre_bench(){ + vector = Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)); + matrix = Mat3.from_cols.call( + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)), + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)), + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)) + ); +} + +fn bench() { + let mul = matrix * vector; + let add = matrix + vector; + let div = vector / matrix; +} \ No newline at end of file diff --git a/assets/benchmarks/query/1000_entities.lua b/assets/benchmarks/query/1000_entities.lua new file mode 100644 index 0000000000..9eb806dfcf --- /dev/null +++ b/assets/benchmarks/query/1000_entities.lua @@ -0,0 +1,35 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 1000 do + local entity = world.spawn() + -- spawn 1000 entities with random components + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/1000_entities.rhai b/assets/benchmarks/query/1000_entities.rhai new file mode 100644 index 0000000000..80d3a0f88c --- /dev/null +++ b/assets/benchmarks/query/1000_entities.rhai @@ -0,0 +1,34 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=1000 { + let entity = world.spawn_.call(); + // spawn 1000 entities with random components + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/query/100_entities.lua b/assets/benchmarks/query/100_entities.lua new file mode 100644 index 0000000000..6cad058a8e --- /dev/null +++ b/assets/benchmarks/query/100_entities.lua @@ -0,0 +1,34 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 100 do + local entity = world.spawn() + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/100_entities.rhai b/assets/benchmarks/query/100_entities.rhai new file mode 100644 index 0000000000..2b6b5021a2 --- /dev/null +++ b/assets/benchmarks/query/100_entities.rhai @@ -0,0 +1,33 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=100 { + let entity = world.spawn_.call(); + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/query/10_entities.lua b/assets/benchmarks/query/10_entities.lua new file mode 100644 index 0000000000..77fc24b76c --- /dev/null +++ b/assets/benchmarks/query/10_entities.lua @@ -0,0 +1,34 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 10 do + local entity = world.spawn() + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/10_entities.rhai b/assets/benchmarks/query/10_entities.rhai new file mode 100644 index 0000000000..14343b82f9 --- /dev/null +++ b/assets/benchmarks/query/10_entities.rhai @@ -0,0 +1,33 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=10 { + let entity = world.spawn_.call(); + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/reflection/10.lua b/assets/benchmarks/reflection/10.lua new file mode 100644 index 0000000000..45b57f265c --- /dev/null +++ b/assets/benchmarks/reflection/10.lua @@ -0,0 +1,52 @@ +-- create a dynamic component with n depth of fields +local Component = world.register_new_component("DeepComponent"); +local new_entity = world.spawn(); + +reseed() + +local keys = { + "_0", + "my_key", + "longish_key" +} + +-- Recursively build a single nested table using the pre-selected step_keys. +local function make_path(step_keys, index) + if index > #step_keys then + return {} + end + local key = step_keys[index] + return { [key] = make_path(step_keys, index + 1) } +end + + +local current_reference = nil +local steps = 10 +local step_keys = {} +function pre_bench() + step_keys = {} + current_reference = nil + -- Choose keys for every step. + for i = 1, steps do + local key = keys[random_int(1, #keys)] + step_keys[i] = key + end + + -- Build the nested table. + local instance = make_path(step_keys, 1) + world.remove_component(new_entity, Component) + world.insert_component(new_entity, Component, construct(types.DynamicComponent, { + data = instance + })) + current_reference = world.get_component(new_entity, Component) +end + +function bench() + -- reference into random key into current_reference steps times + local reference = current_reference.data + local current_step = 1 + while current_step <= steps do + reference = reference[step_keys[current_step]] + current_step = current_step + 1 + end +end \ No newline at end of file diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai new file mode 100644 index 0000000000..0f483a096d --- /dev/null +++ b/assets/benchmarks/reflection/10.rhai @@ -0,0 +1,52 @@ +let Component = world.register_new_component.call("DeepComponent"); +let new_entity = world.spawn_.call(); + +reseed.call(); + +let keys = [ + "_0", + "my_key", + "longish_key" +]; + +fn make_path(step_keys, index) { + if index >= step_keys.len() { + return #{}; + }; + let key = step_keys[index]; + let nested = make_path(step_keys, index + 1); + let obj = #{}; + obj.set(key,nested); + return obj; +}; + +let current_reference = (); +let steps = 10; +let step_keys = []; + +fn pre_bench() { + step_keys = []; + current_reference = (); + + // Choose keys for every step. + for i in 0 .. steps { + let key = keys[random_int.call(0, keys.len() - 1)]; + step_keys.push(key); + }; + + // Build the nested table. + let instance = make_path(step_keys, 0); + world.remove_component.call(new_entity, Component); + world.insert_component.call(new_entity, Component, construct.call(types.DynamicComponent, #{ "data": instance })); + current_reference = world.get_component.call(new_entity, Component); +}; + +fn bench() { + // reference into random key into current_reference steps times + let reference = current_reference.data; + let current_step = 0; + while current_step < steps { + reference = reference[step_keys[current_step]]; + current_step += 1; + }; +}; \ No newline at end of file diff --git a/assets/benchmarks/reflection/100.lua b/assets/benchmarks/reflection/100.lua new file mode 100644 index 0000000000..a61578ed0d --- /dev/null +++ b/assets/benchmarks/reflection/100.lua @@ -0,0 +1,53 @@ +-- create a dynamic component with n depth of fields +local Component = world.register_new_component("DeepComponent"); +local new_entity = world.spawn(); + +reseed() + +local keys = { + "_0", + "my_key", + "longish_key" +} + +-- Recursively build a single nested table using the pre-selected step_keys. +local function make_path(step_keys, index) + if index > #step_keys then + return {} + end + local key = step_keys[index] + return { [key] = make_path(step_keys, index + 1) } +end + + +local current_reference = nil +local steps = 100 +local step_keys = {} +function pre_bench() + step_keys = {} + current_reference = nil + + -- Choose keys for every step. + for i = 1, steps do + local key = keys[random_int(1, #keys)] + step_keys[i] = key + end + + -- Build the nested table. + local instance = make_path(step_keys, 1) + world.remove_component(new_entity, Component) + world.insert_component(new_entity, Component, construct(types.DynamicComponent, { + data = instance + })) + current_reference = world.get_component(new_entity, Component) +end + +function bench() + -- reference into random key into current_reference steps times + local reference = current_reference.data + local current_step = 1 + while current_step <= steps do + reference = reference[step_keys[current_step]] + current_step = current_step + 1 + end +end \ No newline at end of file diff --git a/assets/benchmarks/reflection/100.rhai b/assets/benchmarks/reflection/100.rhai new file mode 100644 index 0000000000..32bef1925e --- /dev/null +++ b/assets/benchmarks/reflection/100.rhai @@ -0,0 +1,52 @@ +let Component = world.register_new_component.call("DeepComponent"); +let new_entity = world.spawn_.call(); + +reseed.call(); + +let keys = [ + "_0", + "my_key", + "longish_key" +]; + +fn make_path(step_keys, index) { + if index >= step_keys.len() { + return #{}; + }; + let key = step_keys[index]; + let nested = make_path(step_keys, index + 1); + let obj = #{}; + obj.set(key,nested); + return obj; +}; + +let current_reference = (); +let steps = 100; +let step_keys = []; + +fn pre_bench() { + step_keys = []; + current_reference = (); + + // Choose keys for every step. + for i in 0 .. steps { + let key = keys[random_int.call(0, keys.len() - 1)]; + step_keys.push(key); + }; + + // Build the nested table. + let instance = make_path(step_keys, 0); + world.remove_component.call(new_entity, Component); + world.insert_component.call(new_entity, Component, construct.call(types.DynamicComponent, #{ "data": instance })); + current_reference = world.get_component.call(new_entity, Component); +}; + +fn bench() { + // reference into random key into current_reference steps times + let reference = current_reference.data; + let current_step = 0; + while current_step < steps { + reference = reference[step_keys[current_step]]; + current_step += 1; + }; +}; \ No newline at end of file diff --git a/crates/testing_crates/script_integration_test_harness/Cargo.toml b/crates/testing_crates/script_integration_test_harness/Cargo.toml index 3e7111d35b..45c85b81bc 100644 --- a/crates/testing_crates/script_integration_test_harness/Cargo.toml +++ b/crates/testing_crates/script_integration_test_harness/Cargo.toml @@ -22,3 +22,5 @@ pretty_assertions = "1.*" bevy_mod_scripting_lua = { path = "../../languages/bevy_mod_scripting_lua", optional = true } bevy_mod_scripting_rhai = { path = "../../languages/bevy_mod_scripting_rhai", optional = true } criterion = "0.5" +rand = "0.9" +rand_chacha = "0.9" diff --git a/crates/testing_crates/script_integration_test_harness/src/lib.rs b/crates/testing_crates/script_integration_test_harness/src/lib.rs index 2078cf2ef4..0518ec05d8 100644 --- a/crates/testing_crates/script_integration_test_harness/src/lib.rs +++ b/crates/testing_crates/script_integration_test_harness/src/lib.rs @@ -142,7 +142,7 @@ pub fn make_test_rhai_plugin() -> bevy_mod_scripting_rhai::RhaiScriptingPlugin { RhaiScriptingPlugin::default().add_runtime_initializer(|runtime| { let mut runtime = runtime.write(); - + runtime.set_max_call_levels(1000); runtime.register_fn("assert", |a: Dynamic, b: &str| { if !a.is::() { panic!("Expected a boolean value, but got {:?}", a); @@ -355,7 +355,11 @@ pub fn run_lua_benchmark( criterion, |ctxt, _runtime, label, criterion| { let bencher: Function = ctxt.globals().get("bench").map_err(|e| e.to_string())?; + let pre_bencher: Option = ctxt.globals().get("pre_bench").ok(); criterion.bench_function(label, |c| { + if let Some(pre_bencher) = &pre_bencher { + pre_bencher.call::<()>(()).unwrap(); + } c.iter(|| { bencher.call::<()>(()).unwrap(); }) @@ -382,7 +386,14 @@ pub fn run_rhai_benchmark( |ctxt, runtime, label, criterion| { let runtime = runtime.read(); const ARGS: [usize; 0] = []; + let has_pre_bench = ctxt.ast.iter_functions().any(|f| f.name == "pre_bench"); criterion.bench_function(label, |c| { + // call "pre_bench" if any + if has_pre_bench { + let _ = runtime + .call_fn::(&mut ctxt.scope, &ctxt.ast, "pre_bench", ARGS) + .unwrap(); + } c.iter(|| { let _ = runtime .call_fn::(&mut ctxt.scope, &ctxt.ast, "bench", ARGS) diff --git a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs index 71a5268816..910ee6c412 100644 --- a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs +++ b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs @@ -1,4 +1,7 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use bevy::{ app::App, @@ -15,12 +18,20 @@ use bevy_mod_scripting_core::{ }, pretty_print::DisplayWithWorld, ReflectReference, ScriptComponentRegistration, ScriptResourceRegistration, - ScriptTypeRegistration, + ScriptTypeRegistration, ScriptValue, }, error::InteropError, }; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha12Rng; use test_utils::test_data::EnumerateTestComponents; +// lazy lock rng state +static RNG: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + let seed = [42u8; 32]; + Mutex::new(ChaCha12Rng::from_seed(seed)) +}); + pub fn register_test_functions(world: &mut App) { let world = world.world_mut(); NamespaceBuilder::::new_unregistered(world) @@ -108,7 +119,29 @@ pub fn register_test_functions(world: &mut App) { NamespaceBuilder::::new_unregistered(world) .register("global_hello_world", || Ok("hi!")) + .register("random", |start: Option, end: Option| { + let start = start.unwrap_or(0); + let end = end.unwrap_or(1); + let mut rng = RNG.lock().unwrap(); + rng.random_range::(start..=end) + }) + .register("random_int", |start: Option, end: Option| { + let start = start.unwrap_or(0); + let end = end.unwrap_or(1); + let mut rng = RNG.lock().unwrap(); + rng.random_range::(start..=end) + }) + .register("reseed", || { + let seed = [42u8; 32]; + let mut rng = RNG.lock().unwrap(); + *rng = ChaCha12Rng::from_seed(seed); + }) .register("make_hashmap", |map: HashMap| map) + .register("noop", || {}) + .register( + "noop_4_args", + |_a: ScriptValue, _b: ScriptValue, _c: ScriptValue, _d: ScriptValue| {}, + ) .register( "assert_str_eq", |s1: String, s2: String, reason: Option| { diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index e24e372ffb..3f43e9b86a 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -349,8 +349,12 @@ impl App { Xtasks::Install { binary } => { cmd.arg("install").arg(binary.as_ref()); } - Xtasks::Bench {} => { + Xtasks::Bench { publish } => { cmd.arg("bench"); + + if publish { + cmd.arg("--publish"); + } } } @@ -638,7 +642,12 @@ enum Xtasks { /// CiMatrix, /// Runs bencher in dry mode by default if not on the main branch - Bench {}, + /// To publish main branch defaults set publish mode to true + Bench { + /// Publish the benchmarks when on main + #[clap(long, default_value = "false", help = "Publish the benchmarks")] + publish: bool, + }, } #[derive(Serialize, Clone)] @@ -714,7 +723,7 @@ impl Xtasks { bevy_features, } => Self::codegen(app_settings, output_dir, bevy_features), Xtasks::Install { binary } => Self::install(app_settings, binary), - Xtasks::Bench {} => Self::bench(app_settings), + Xtasks::Bench { publish: execute } => Self::bench(app_settings, execute), }?; Ok("".into()) @@ -1214,7 +1223,7 @@ impl Xtasks { Ok(()) } - fn bench(app_settings: GlobalArgs) -> Result<()> { + fn bench(app_settings: GlobalArgs, execute: bool) -> Result<()> { // first of all figure out which branch we're on // run // git rev-parse --abbrev-ref HEAD let workspace_dir = Self::workspace_dir(&app_settings).unwrap(); @@ -1268,7 +1277,7 @@ impl Xtasks { bencher_cmd.args(["--github-actions", &token]); } - if !is_main { + if !is_main || !execute { bencher_cmd.args(["--dry-run"]); } @@ -1285,6 +1294,143 @@ impl Xtasks { bail!("Failed to run bencher: {:?}", out); } + // if we're on linux and publishing and on main synch graphs + if os == "linux" && is_main && execute { + Self::synch_bencher_graphs()?; + } + + Ok(()) + } + + fn synch_bencher_graphs() -> Result<()> { + // first run `bencher benchmark list bms + // this produces list of objects each containing a `uuid` and `name` + + let parse_list_of_dicts = |bytes: Vec| { + serde_json::from_slice::>>(&bytes) + .with_context(|| "Could not parse bencher output") + }; + + let token = std::env::var("BENCHER_API_TOKEN").ok(); + let mut bencher_cmd = Command::new("bencher"); + let benchmarks = bencher_cmd + .arg("benchmark") + .args(["list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list benchmarks")?; + if !benchmarks.status.success() { + bail!("Failed to list benchmarks: {:?}", benchmarks); + } + + // parse teh name and uuid pairs + let benchmarks = parse_list_of_dicts(benchmarks.stdout)? + .into_iter() + .map(|p| { + let name = p.get("name").expect("no name in project"); + let uuid = p.get("uuid").expect("no uuid in project"); + (name.clone(), uuid.clone()) + }) + .collect::>(); + + // delete all plots using bencher plot list bms to get "uuid's" + // then bencher plot delete bms + + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list plots")?; + + if !bencher_cmd.status.success() { + bail!("Failed to list plots: {:?}", bencher_cmd); + } + + let plots = parse_list_of_dicts(bencher_cmd.stdout)? + .into_iter() + .map(|p| { + let uuid = p.get("uuid").expect("no uuid in plot"); + uuid.clone() + }) + .collect::>(); + + for uuid in plots { + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "delete", "bms", &uuid]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not delete plot")?; + + if !bencher_cmd.status.success() { + bail!("Failed to delete plot: {:?}", bencher_cmd); + } + } + let testbeds = Command::new("bencher") + .arg("testbed") + .args(["list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list testbeds")?; + + if !testbeds.status.success() { + bail!("Failed to list testbeds: {:?}", testbeds); + } + + let testbeds = parse_list_of_dicts(testbeds.stdout)? + .into_iter() + .map(|p| { + let name = p.get("name").expect("no name in testbed"); + let uuid = p.get("uuid").expect("no uuid in testbed"); + (name.clone(), uuid.clone()) + }) + .filter(|(name, _)| name.contains("gha")) + .collect::>(); + + let group_to_benchmark_map: HashMap<_, Vec<_>> = + benchmarks + .iter() + .fold(HashMap::new(), |mut acc, (name, uuid)| { + let group = name.split('/').next().unwrap_or_default(); + acc.entry(group.to_owned()).or_default().push(uuid.clone()); + acc + }); + + // create plot using + // bencher plot create --x-axis date_time --branches main --testbeds --benchmarks --measures latency + + for (group, uuids) in group_to_benchmark_map { + for (testbed_name, testbed_uuid) in testbeds.iter() { + let without_gha = testbed_name.replace("-gha", ""); + let plot_name = format!("{without_gha} {group}"); + + let window_months = 12; + let window_seconds = window_months * 30 * 24 * 60 * 60; + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "create", "bms"]) + .args(["--title", &plot_name]) + .arg("--upper-boundary") + .args(["--x-axis", "date_time"]) + .args(["--window", &window_seconds.to_string()]) + .args(["--branches", "main"]) + .args(["--testbeds", testbed_uuid]) + .args(["--benchmarks", &uuids.join(",")]) + .args(["--measures", "latency"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not create plot")?; + + if !bencher_cmd.status.success() { + bail!("Failed to create plot: {:?}", bencher_cmd); + } + } + } + Ok(()) } @@ -1453,7 +1599,7 @@ impl Xtasks { // on non-main branches this will just dry run output.push(App { global_args: default_args.clone(), - subcmd: Xtasks::Bench {}, + subcmd: Xtasks::Bench { publish: true }, }); // and finally run tests with coverage