From da3e3492f6ac1034b8371216f94d33103ac0cec6 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:15:26 +0000 Subject: [PATCH 01/11] chore/more-benchmarks --- assets/benchmarks/query/1000_entities.lua | 35 +++++++++++++ assets/benchmarks/query/1000_entities.rhai | 34 +++++++++++++ assets/benchmarks/query/100_entities.lua | 34 +++++++++++++ assets/benchmarks/query/100_entities.rhai | 33 ++++++++++++ assets/benchmarks/query/10_entities.lua | 34 +++++++++++++ assets/benchmarks/query/10_entities.rhai | 33 ++++++++++++ assets/benchmarks/reflection/10.lua | 50 +++++++++++++++++++ assets/benchmarks/reflection/10.rhai | 49 ++++++++++++++++++ .../Cargo.toml | 2 + .../src/lib.rs | 11 ++++ .../src/test_functions.rs | 30 ++++++++++- crates/xtask/src/main.rs | 21 +++++--- 12 files changed, 359 insertions(+), 7 deletions(-) create mode 100644 assets/benchmarks/query/1000_entities.lua create mode 100644 assets/benchmarks/query/1000_entities.rhai create mode 100644 assets/benchmarks/query/100_entities.lua create mode 100644 assets/benchmarks/query/100_entities.rhai create mode 100644 assets/benchmarks/query/10_entities.lua create mode 100644 assets/benchmarks/query/10_entities.rhai create mode 100644 assets/benchmarks/reflection/10.lua create mode 100644 assets/benchmarks/reflection/10.rhai diff --git a/assets/benchmarks/query/1000_entities.lua b/assets/benchmarks/query/1000_entities.lua new file mode 100644 index 0000000000..9eb806dfcf --- /dev/null +++ b/assets/benchmarks/query/1000_entities.lua @@ -0,0 +1,35 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 1000 do + local entity = world.spawn() + -- spawn 1000 entities with random components + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/1000_entities.rhai b/assets/benchmarks/query/1000_entities.rhai new file mode 100644 index 0000000000..80d3a0f88c --- /dev/null +++ b/assets/benchmarks/query/1000_entities.rhai @@ -0,0 +1,34 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=1000 { + let entity = world.spawn_.call(); + // spawn 1000 entities with random components + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/query/100_entities.lua b/assets/benchmarks/query/100_entities.lua new file mode 100644 index 0000000000..6cad058a8e --- /dev/null +++ b/assets/benchmarks/query/100_entities.lua @@ -0,0 +1,34 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 100 do + local entity = world.spawn() + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/100_entities.rhai b/assets/benchmarks/query/100_entities.rhai new file mode 100644 index 0000000000..2b6b5021a2 --- /dev/null +++ b/assets/benchmarks/query/100_entities.rhai @@ -0,0 +1,33 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=100 { + let entity = world.spawn_.call(); + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/query/10_entities.lua b/assets/benchmarks/query/10_entities.lua new file mode 100644 index 0000000000..77fc24b76c --- /dev/null +++ b/assets/benchmarks/query/10_entities.lua @@ -0,0 +1,34 @@ +local entity_a = world.spawn() +local entity_b = world.spawn() +local entity_c = world.spawn() + +local components = { + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +} + +reseed() + +for i = 1, 10 do + local entity = world.spawn() + local left_to_pick = {1,2,3,4} + for j = 1, 3 do + local index = random_int(1, #left_to_pick) + local component = components[left_to_pick[index]] + table.remove(left_to_pick, index) + world.add_default_component(entity, component) + end +end + +function bench() + for i,result in pairs(world.query() + :component(types.CompWithFromWorldAndComponentData) + :component(types.SimpleTupleStruct) + :with(types.SimpleEnum) + :without(types.CompWithDefaultAndComponentData) + :build()) do + + end +end \ No newline at end of file diff --git a/assets/benchmarks/query/10_entities.rhai b/assets/benchmarks/query/10_entities.rhai new file mode 100644 index 0000000000..14343b82f9 --- /dev/null +++ b/assets/benchmarks/query/10_entities.rhai @@ -0,0 +1,33 @@ +let entity_a = world.spawn_.call(); +let entity_b = world.spawn_.call(); +let entity_c = world.spawn_.call(); + +let components = [ + types.CompWithDefaultAndComponentData, + types.CompWithFromWorldAndComponentData, + types.SimpleTupleStruct, + types.SimpleEnum, +]; + +reseed.call(); + +for i in 1..=10 { + let entity = world.spawn_.call(); + let left_to_pick = [0, 1, 2, 3]; + for j in 1..=3 { + let index = random_int.call(1, left_to_pick.len()) - 1; + let component = components[left_to_pick[index]]; + left_to_pick.remove(index); + world.add_default_component.call(entity, component); + }; +}; + +fn bench() { + for (result, i) in world.query.call() + .component.call(types.CompWithFromWorldAndComponentData) + .component.call(types.SimpleTupleStruct) + .with_.call(types.SimpleEnum) + .without.call(types.CompWithDefaultAndComponentData) + .build.call() { + }; +} \ No newline at end of file diff --git a/assets/benchmarks/reflection/10.lua b/assets/benchmarks/reflection/10.lua new file mode 100644 index 0000000000..fb957c4d8a --- /dev/null +++ b/assets/benchmarks/reflection/10.lua @@ -0,0 +1,50 @@ +-- create a dynamic component with n depth of fields +local Component = world.register_new_component("DeepComponent"); +local new_entity = world.spawn(); + +reseed() + +local keys = { + "_0", + "my_key", + "longish_key" +} + +-- Recursively build a single nested table using the pre-selected step_keys. +local function make_path(step_keys, index) + if index > #step_keys then + return {} + end + local key = step_keys[index] + return { [key] = make_path(step_keys, index + 1) } +end + + +local current_reference = nil +local steps = 10 +local step_keys = {} +function pre_bench() + -- Choose keys for every step. + for i = 1, steps do + local key = keys[random_int(1, #keys)] + step_keys[i] = key + end + + -- Build the nested table. + local instance = make_path(step_keys, 1) + world.remove_component(new_entity, Component) + world.insert_component(new_entity, Component, construct(types.DynamicComponent, { + data = instance + })) + current_reference = world.get_component(new_entity, Component) +end + +function bench() + -- reference into random key into current_reference steps times + local reference = current_reference.data + local current_step = 1 + while current_step <= steps do + reference = reference[step_keys[current_step]] + current_step = current_step + 1 + end +end \ No newline at end of file diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai new file mode 100644 index 0000000000..d1f395ec63 --- /dev/null +++ b/assets/benchmarks/reflection/10.rhai @@ -0,0 +1,49 @@ +let Component = world.register_new_component_.call("DeepComponent"); +let new_entity = world.spawn_.call(); + +reseed.call(); + +let keys = [ + "_0", + "my_key", + "longish_key" +]; + +fn make_path(step_keys, index) { + if index >= step_keys.len() { + return #{}; + }; + let key = step_keys[index]; + let nested = make_path(step_keys, index + 1); + let obj = #{}; + obj.insert(key, nested); + return obj; +}; + +let current_reference = (); +let steps = 10; +let step_keys = []; + +fn pre_bench() { + // Choose keys for every step. + for i in 0 .. steps { + let key = keys[random_int.call(0, keys.len() - 1)]; + step_keys.push(key); + }; + + // Build the nested table. + let instance = make_path(step_keys, 0); + world.remove_component.call(new_entity, Component); + world.insert_component.call(new_entity, Component, construct.call(types.DynamicComponent, #{ "data": instance })); + current_reference = world.get_component.call(new_entity, Component); +}; + +fn bench() { + // reference into random key into current_reference steps times + let reference = current_reference.data; + let current_step = 0; + while current_step < steps { + reference = reference[step_keys[current_step]]; + current_step += 1; + }; +}; \ No newline at end of file diff --git a/crates/testing_crates/script_integration_test_harness/Cargo.toml b/crates/testing_crates/script_integration_test_harness/Cargo.toml index 3e7111d35b..45c85b81bc 100644 --- a/crates/testing_crates/script_integration_test_harness/Cargo.toml +++ b/crates/testing_crates/script_integration_test_harness/Cargo.toml @@ -22,3 +22,5 @@ pretty_assertions = "1.*" bevy_mod_scripting_lua = { path = "../../languages/bevy_mod_scripting_lua", optional = true } bevy_mod_scripting_rhai = { path = "../../languages/bevy_mod_scripting_rhai", optional = true } criterion = "0.5" +rand = "0.9" +rand_chacha = "0.9" diff --git a/crates/testing_crates/script_integration_test_harness/src/lib.rs b/crates/testing_crates/script_integration_test_harness/src/lib.rs index 2078cf2ef4..4bc9d1e44c 100644 --- a/crates/testing_crates/script_integration_test_harness/src/lib.rs +++ b/crates/testing_crates/script_integration_test_harness/src/lib.rs @@ -355,7 +355,11 @@ pub fn run_lua_benchmark( criterion, |ctxt, _runtime, label, criterion| { let bencher: Function = ctxt.globals().get("bench").map_err(|e| e.to_string())?; + let pre_bencher: Option = ctxt.globals().get("pre_bench").ok(); criterion.bench_function(label, |c| { + if let Some(pre_bencher) = &pre_bencher { + pre_bencher.call::<()>(()).unwrap(); + } c.iter(|| { bencher.call::<()>(()).unwrap(); }) @@ -382,7 +386,14 @@ pub fn run_rhai_benchmark( |ctxt, runtime, label, criterion| { let runtime = runtime.read(); const ARGS: [usize; 0] = []; + let has_pre_bench = ctxt.ast.iter_functions().any(|f| f.name == "pre_bench"); criterion.bench_function(label, |c| { + // call "pre_bench" if any + if has_pre_bench { + let _ = runtime + .call_fn::(&mut ctxt.scope, &ctxt.ast, "pre_bench", ARGS) + .unwrap(); + } c.iter(|| { let _ = runtime .call_fn::(&mut ctxt.scope, &ctxt.ast, "bench", ARGS) diff --git a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs index 71a5268816..1287691970 100644 --- a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs +++ b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs @@ -1,4 +1,7 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use bevy::{ app::App, @@ -19,8 +22,16 @@ use bevy_mod_scripting_core::{ }, error::InteropError, }; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha12Rng; use test_utils::test_data::EnumerateTestComponents; +// lazy lock rng state +static RNG: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + let seed = [42u8; 32]; + Mutex::new(ChaCha12Rng::from_seed(seed)) +}); + pub fn register_test_functions(world: &mut App) { let world = world.world_mut(); NamespaceBuilder::::new_unregistered(world) @@ -108,6 +119,23 @@ pub fn register_test_functions(world: &mut App) { NamespaceBuilder::::new_unregistered(world) .register("global_hello_world", || Ok("hi!")) + .register("random", |start: Option, end: Option| { + let start = start.unwrap_or(0); + let end = end.unwrap_or(1); + let mut rng = RNG.lock().unwrap(); + rng.random_range::(start..=end) + }) + .register("random_int", |start: Option, end: Option| { + let start = start.unwrap_or(0); + let end = end.unwrap_or(1); + let mut rng = RNG.lock().unwrap(); + rng.random_range::(start..=end) + }) + .register("reseed", || { + let seed = [42u8; 32]; + let mut rng = RNG.lock().unwrap(); + *rng = ChaCha12Rng::from_seed(seed); + }) .register("make_hashmap", |map: HashMap| map) .register( "assert_str_eq", diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index e24e372ffb..d32bd90d07 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -349,8 +349,12 @@ impl App { Xtasks::Install { binary } => { cmd.arg("install").arg(binary.as_ref()); } - Xtasks::Bench {} => { + Xtasks::Bench { publish: execute } => { cmd.arg("bench"); + + if execute { + cmd.arg("--execute"); + } } } @@ -638,7 +642,12 @@ enum Xtasks { /// CiMatrix, /// Runs bencher in dry mode by default if not on the main branch - Bench {}, + /// To publish main branch defaults set publish mode to true + Bench { + /// Publish the benchmarks when on main + #[clap(long, default_value = "false", help = "Publish the benchmarks")] + publish: bool, + }, } #[derive(Serialize, Clone)] @@ -714,7 +723,7 @@ impl Xtasks { bevy_features, } => Self::codegen(app_settings, output_dir, bevy_features), Xtasks::Install { binary } => Self::install(app_settings, binary), - Xtasks::Bench {} => Self::bench(app_settings), + Xtasks::Bench { publish: execute } => Self::bench(app_settings, execute), }?; Ok("".into()) @@ -1214,7 +1223,7 @@ impl Xtasks { Ok(()) } - fn bench(app_settings: GlobalArgs) -> Result<()> { + fn bench(app_settings: GlobalArgs, execute: bool) -> Result<()> { // first of all figure out which branch we're on // run // git rev-parse --abbrev-ref HEAD let workspace_dir = Self::workspace_dir(&app_settings).unwrap(); @@ -1268,7 +1277,7 @@ impl Xtasks { bencher_cmd.args(["--github-actions", &token]); } - if !is_main { + if !is_main || !execute { bencher_cmd.args(["--dry-run"]); } @@ -1453,7 +1462,7 @@ impl Xtasks { // on non-main branches this will just dry run output.push(App { global_args: default_args.clone(), - subcmd: Xtasks::Bench {}, + subcmd: Xtasks::Bench { publish: true }, }); // and finally run tests with coverage From a0e3ca1a5bb901aca40f25568899508a7b93ac3c Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:20:14 +0000 Subject: [PATCH 02/11] typo --- assets/benchmarks/reflection/10.rhai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai index d1f395ec63..229cc22f17 100644 --- a/assets/benchmarks/reflection/10.rhai +++ b/assets/benchmarks/reflection/10.rhai @@ -1,4 +1,4 @@ -let Component = world.register_new_component_.call("DeepComponent"); +let Component = world.register_new_component.call("DeepComponent"); let new_entity = world.spawn_.call(); reseed.call(); From 2383978aff81ac05c192d5b61fbdb369909fdba7 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:21:11 +0000 Subject: [PATCH 03/11] add bigger benchmarks for reflection --- assets/benchmarks/reflection/10.lua | 2 +- assets/benchmarks/reflection/100.lua | 50 +++++++++++++++++++++++++++ assets/benchmarks/reflection/100.rhai | 49 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 assets/benchmarks/reflection/100.lua create mode 100644 assets/benchmarks/reflection/100.rhai diff --git a/assets/benchmarks/reflection/10.lua b/assets/benchmarks/reflection/10.lua index fb957c4d8a..84b2f0bbc6 100644 --- a/assets/benchmarks/reflection/10.lua +++ b/assets/benchmarks/reflection/10.lua @@ -21,7 +21,7 @@ end local current_reference = nil -local steps = 10 +local steps = 100 local step_keys = {} function pre_bench() -- Choose keys for every step. diff --git a/assets/benchmarks/reflection/100.lua b/assets/benchmarks/reflection/100.lua new file mode 100644 index 0000000000..fb957c4d8a --- /dev/null +++ b/assets/benchmarks/reflection/100.lua @@ -0,0 +1,50 @@ +-- create a dynamic component with n depth of fields +local Component = world.register_new_component("DeepComponent"); +local new_entity = world.spawn(); + +reseed() + +local keys = { + "_0", + "my_key", + "longish_key" +} + +-- Recursively build a single nested table using the pre-selected step_keys. +local function make_path(step_keys, index) + if index > #step_keys then + return {} + end + local key = step_keys[index] + return { [key] = make_path(step_keys, index + 1) } +end + + +local current_reference = nil +local steps = 10 +local step_keys = {} +function pre_bench() + -- Choose keys for every step. + for i = 1, steps do + local key = keys[random_int(1, #keys)] + step_keys[i] = key + end + + -- Build the nested table. + local instance = make_path(step_keys, 1) + world.remove_component(new_entity, Component) + world.insert_component(new_entity, Component, construct(types.DynamicComponent, { + data = instance + })) + current_reference = world.get_component(new_entity, Component) +end + +function bench() + -- reference into random key into current_reference steps times + local reference = current_reference.data + local current_step = 1 + while current_step <= steps do + reference = reference[step_keys[current_step]] + current_step = current_step + 1 + end +end \ No newline at end of file diff --git a/assets/benchmarks/reflection/100.rhai b/assets/benchmarks/reflection/100.rhai new file mode 100644 index 0000000000..a7ebd371b8 --- /dev/null +++ b/assets/benchmarks/reflection/100.rhai @@ -0,0 +1,49 @@ +let Component = world.register_new_component.call("DeepComponent"); +let new_entity = world.spawn_.call(); + +reseed.call(); + +let keys = [ + "_0", + "my_key", + "longish_key" +]; + +fn make_path(step_keys, index) { + if index >= step_keys.len() { + return #{}; + }; + let key = step_keys[index]; + let nested = make_path(step_keys, index + 1); + let obj = #{}; + obj.insert(key, nested); + return obj; +}; + +let current_reference = (); +let steps = 100; +let step_keys = []; + +fn pre_bench() { + // Choose keys for every step. + for i in 0 .. steps { + let key = keys[random_int.call(0, keys.len() - 1)]; + step_keys.push(key); + }; + + // Build the nested table. + let instance = make_path(step_keys, 0); + world.remove_component.call(new_entity, Component); + world.insert_component.call(new_entity, Component, construct.call(types.DynamicComponent, #{ "data": instance })); + current_reference = world.get_component.call(new_entity, Component); +}; + +fn bench() { + // reference into random key into current_reference steps times + let reference = current_reference.data; + let current_step = 0; + while current_step < steps { + reference = reference[step_keys[current_step]]; + current_step += 1; + }; +}; \ No newline at end of file From d03bd48fdc3ed934b7f9102a102bee89e8f3ac58 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:23:21 +0000 Subject: [PATCH 04/11] fix typo --- assets/benchmarks/reflection/10.rhai | 2 +- assets/benchmarks/reflection/100.rhai | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai index 229cc22f17..6345f8326a 100644 --- a/assets/benchmarks/reflection/10.rhai +++ b/assets/benchmarks/reflection/10.rhai @@ -16,7 +16,7 @@ fn make_path(step_keys, index) { let key = step_keys[index]; let nested = make_path(step_keys, index + 1); let obj = #{}; - obj.insert(key, nested); + obj[key] = nested; return obj; }; diff --git a/assets/benchmarks/reflection/100.rhai b/assets/benchmarks/reflection/100.rhai index a7ebd371b8..45e654b770 100644 --- a/assets/benchmarks/reflection/100.rhai +++ b/assets/benchmarks/reflection/100.rhai @@ -16,7 +16,7 @@ fn make_path(step_keys, index) { let key = step_keys[index]; let nested = make_path(step_keys, index + 1); let obj = #{}; - obj.insert(key, nested); + obj[key] = nested; return obj; }; From 09fb48b5ba46ecb4b74c302a3a5b487801486686 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:24:21 +0000 Subject: [PATCH 05/11] typo --- assets/benchmarks/reflection/10.rhai | 2 +- assets/benchmarks/reflection/100.rhai | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai index 6345f8326a..fe7eda8939 100644 --- a/assets/benchmarks/reflection/10.rhai +++ b/assets/benchmarks/reflection/10.rhai @@ -16,7 +16,7 @@ fn make_path(step_keys, index) { let key = step_keys[index]; let nested = make_path(step_keys, index + 1); let obj = #{}; - obj[key] = nested; + obj.set(key,nested); return obj; }; diff --git a/assets/benchmarks/reflection/100.rhai b/assets/benchmarks/reflection/100.rhai index 45e654b770..43d833eece 100644 --- a/assets/benchmarks/reflection/100.rhai +++ b/assets/benchmarks/reflection/100.rhai @@ -16,7 +16,7 @@ fn make_path(step_keys, index) { let key = step_keys[index]; let nested = make_path(step_keys, index + 1); let obj = #{}; - obj[key] = nested; + obj.set(key,nested); return obj; }; From 21f9c587cf5f18f6d7369ee0a9ba2553b6dca861 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 12:25:57 +0000 Subject: [PATCH 06/11] increase stack depth for rhai --- .../testing_crates/script_integration_test_harness/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/testing_crates/script_integration_test_harness/src/lib.rs b/crates/testing_crates/script_integration_test_harness/src/lib.rs index 4bc9d1e44c..0518ec05d8 100644 --- a/crates/testing_crates/script_integration_test_harness/src/lib.rs +++ b/crates/testing_crates/script_integration_test_harness/src/lib.rs @@ -142,7 +142,7 @@ pub fn make_test_rhai_plugin() -> bevy_mod_scripting_rhai::RhaiScriptingPlugin { RhaiScriptingPlugin::default().add_runtime_initializer(|runtime| { let mut runtime = runtime.write(); - + runtime.set_max_call_levels(1000); runtime.register_fn("assert", |a: Dynamic, b: &str| { if !a.is::() { panic!("Expected a boolean value, but got {:?}", a); From a37223a4ba61895145c0d4ccbfccb3afc26855e5 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 16:57:55 +0000 Subject: [PATCH 07/11] fix benchmarks --- assets/benchmarks/reflection/10.lua | 2 ++ assets/benchmarks/reflection/10.rhai | 3 +++ assets/benchmarks/reflection/100.lua | 3 +++ assets/benchmarks/reflection/100.rhai | 3 +++ 4 files changed, 11 insertions(+) diff --git a/assets/benchmarks/reflection/10.lua b/assets/benchmarks/reflection/10.lua index 84b2f0bbc6..68ab7a213e 100644 --- a/assets/benchmarks/reflection/10.lua +++ b/assets/benchmarks/reflection/10.lua @@ -24,6 +24,8 @@ local current_reference = nil local steps = 100 local step_keys = {} function pre_bench() + step_keys = {} + current_reference = nil -- Choose keys for every step. for i = 1, steps do local key = keys[random_int(1, #keys)] diff --git a/assets/benchmarks/reflection/10.rhai b/assets/benchmarks/reflection/10.rhai index fe7eda8939..0f483a096d 100644 --- a/assets/benchmarks/reflection/10.rhai +++ b/assets/benchmarks/reflection/10.rhai @@ -25,6 +25,9 @@ let steps = 10; let step_keys = []; fn pre_bench() { + step_keys = []; + current_reference = (); + // Choose keys for every step. for i in 0 .. steps { let key = keys[random_int.call(0, keys.len() - 1)]; diff --git a/assets/benchmarks/reflection/100.lua b/assets/benchmarks/reflection/100.lua index fb957c4d8a..4c137d1d52 100644 --- a/assets/benchmarks/reflection/100.lua +++ b/assets/benchmarks/reflection/100.lua @@ -24,6 +24,9 @@ local current_reference = nil local steps = 10 local step_keys = {} function pre_bench() + step_keys = {} + current_reference = nil + -- Choose keys for every step. for i = 1, steps do local key = keys[random_int(1, #keys)] diff --git a/assets/benchmarks/reflection/100.rhai b/assets/benchmarks/reflection/100.rhai index 43d833eece..32bef1925e 100644 --- a/assets/benchmarks/reflection/100.rhai +++ b/assets/benchmarks/reflection/100.rhai @@ -25,6 +25,9 @@ let steps = 100; let step_keys = []; fn pre_bench() { + step_keys = []; + current_reference = (); + // Choose keys for every step. for i in 0 .. steps { let key = keys[random_int.call(0, keys.len() - 1)]; From 2cfd7adfe157d5ee93a95bf1d282592a89d47290 Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 17:07:17 +0000 Subject: [PATCH 08/11] fix xtask bench --- crates/xtask/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index d32bd90d07..80fa56631a 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -349,11 +349,11 @@ impl App { Xtasks::Install { binary } => { cmd.arg("install").arg(binary.as_ref()); } - Xtasks::Bench { publish: execute } => { + Xtasks::Bench { publish } => { cmd.arg("bench"); - if execute { - cmd.arg("--execute"); + if publish { + cmd.arg("--publish"); } } } From e7058a8ee36a3a71c1cd4b71744e072e6332ed7c Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 17:22:59 +0000 Subject: [PATCH 09/11] add more benchmarks --- assets/benchmarks/function/call.lua | 3 +++ assets/benchmarks/function/call.rhai | 3 +++ assets/benchmarks/function/call_4_args.lua | 3 +++ assets/benchmarks/function/call_4_args.rhai | 3 +++ assets/benchmarks/math/vec_mat_ops.lua | 20 +++++++++++++++++++ assets/benchmarks/math/vec_mat_ops.rhai | 19 ++++++++++++++++++ .../src/test_functions.rs | 7 ++++++- 7 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 assets/benchmarks/function/call.lua create mode 100644 assets/benchmarks/function/call.rhai create mode 100644 assets/benchmarks/function/call_4_args.lua create mode 100644 assets/benchmarks/function/call_4_args.rhai create mode 100644 assets/benchmarks/math/vec_mat_ops.lua create mode 100644 assets/benchmarks/math/vec_mat_ops.rhai diff --git a/assets/benchmarks/function/call.lua b/assets/benchmarks/function/call.lua new file mode 100644 index 0000000000..21b351ac09 --- /dev/null +++ b/assets/benchmarks/function/call.lua @@ -0,0 +1,3 @@ +function bench() + noop() +end \ No newline at end of file diff --git a/assets/benchmarks/function/call.rhai b/assets/benchmarks/function/call.rhai new file mode 100644 index 0000000000..5adf764732 --- /dev/null +++ b/assets/benchmarks/function/call.rhai @@ -0,0 +1,3 @@ +fn bench(){ + noop.call(); +} \ No newline at end of file diff --git a/assets/benchmarks/function/call_4_args.lua b/assets/benchmarks/function/call_4_args.lua new file mode 100644 index 0000000000..e595a6d465 --- /dev/null +++ b/assets/benchmarks/function/call_4_args.lua @@ -0,0 +1,3 @@ +function bench() + noop_4_args(1,"asd",{1,2,3}, {asd = 1}) +end \ No newline at end of file diff --git a/assets/benchmarks/function/call_4_args.rhai b/assets/benchmarks/function/call_4_args.rhai new file mode 100644 index 0000000000..299b35871f --- /dev/null +++ b/assets/benchmarks/function/call_4_args.rhai @@ -0,0 +1,3 @@ +fn bench(){ + noop_4_args.call(1,"asd",[1,2,3],#{ asd: 1}); +} \ No newline at end of file diff --git a/assets/benchmarks/math/vec_mat_ops.lua b/assets/benchmarks/math/vec_mat_ops.lua new file mode 100644 index 0000000000..218a9ef060 --- /dev/null +++ b/assets/benchmarks/math/vec_mat_ops.lua @@ -0,0 +1,20 @@ + +reseed() + +local matrix = nil +local vector = nil +function pre_bench() + -- generate random 3x3 matrix and 3x1 vec + vector = Vec3.new(random(1,999), random(1,999), random(1,999)) + matrix = Mat3.from_cols( + Vec3.new(random(1,999), random(1,999), random(1,999)), + Vec3.new(random(1,999), random(1,999), random(1,999)), + Vec3.new(random(1,999), random(1,999), random(1,999)) + ) +end + +function bench() + local mul = matrix * vector + local add = matrix + vector + local div = vector / matrix +end \ No newline at end of file diff --git a/assets/benchmarks/math/vec_mat_ops.rhai b/assets/benchmarks/math/vec_mat_ops.rhai new file mode 100644 index 0000000000..b204d1c5f2 --- /dev/null +++ b/assets/benchmarks/math/vec_mat_ops.rhai @@ -0,0 +1,19 @@ + +reseed.call(); + +let matrix = (); +let vector = (); +fn pre_bench(){ + vector = Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)); + matrix = Mat3.from_cols.call( + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)), + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)), + Vec3.new_.call(random.call(1,999), random.call(1,999), random.call(1,999)) + ); +} + +fn bench() { + let mul = matrix * vector; + let add = matrix + vector; + let div = vector / matrix; +} \ No newline at end of file diff --git a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs index 1287691970..910ee6c412 100644 --- a/crates/testing_crates/script_integration_test_harness/src/test_functions.rs +++ b/crates/testing_crates/script_integration_test_harness/src/test_functions.rs @@ -18,7 +18,7 @@ use bevy_mod_scripting_core::{ }, pretty_print::DisplayWithWorld, ReflectReference, ScriptComponentRegistration, ScriptResourceRegistration, - ScriptTypeRegistration, + ScriptTypeRegistration, ScriptValue, }, error::InteropError, }; @@ -137,6 +137,11 @@ pub fn register_test_functions(world: &mut App) { *rng = ChaCha12Rng::from_seed(seed); }) .register("make_hashmap", |map: HashMap| map) + .register("noop", || {}) + .register( + "noop_4_args", + |_a: ScriptValue, _b: ScriptValue, _c: ScriptValue, _d: ScriptValue| {}, + ) .register( "assert_str_eq", |s1: String, s2: String, reason: Option| { From b465d469f3ead45729912158964a64c6bea5e62f Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 17:30:31 +0000 Subject: [PATCH 10/11] fix bug --- assets/benchmarks/reflection/10.lua | 2 +- assets/benchmarks/reflection/100.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/benchmarks/reflection/10.lua b/assets/benchmarks/reflection/10.lua index 68ab7a213e..45b57f265c 100644 --- a/assets/benchmarks/reflection/10.lua +++ b/assets/benchmarks/reflection/10.lua @@ -21,7 +21,7 @@ end local current_reference = nil -local steps = 100 +local steps = 10 local step_keys = {} function pre_bench() step_keys = {} diff --git a/assets/benchmarks/reflection/100.lua b/assets/benchmarks/reflection/100.lua index 4c137d1d52..a61578ed0d 100644 --- a/assets/benchmarks/reflection/100.lua +++ b/assets/benchmarks/reflection/100.lua @@ -21,7 +21,7 @@ end local current_reference = nil -local steps = 10 +local steps = 100 local step_keys = {} function pre_bench() step_keys = {} From 25a1d6ef46bc31cf9974541baf98b35aa7355dab Mon Sep 17 00:00:00 2001 From: makspll Date: Sun, 23 Mar 2025 18:12:09 +0000 Subject: [PATCH 11/11] add graph synch code --- crates/xtask/src/main.rs | 137 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 80fa56631a..3f43e9b86a 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -1294,6 +1294,143 @@ impl Xtasks { bail!("Failed to run bencher: {:?}", out); } + // if we're on linux and publishing and on main synch graphs + if os == "linux" && is_main && execute { + Self::synch_bencher_graphs()?; + } + + Ok(()) + } + + fn synch_bencher_graphs() -> Result<()> { + // first run `bencher benchmark list bms + // this produces list of objects each containing a `uuid` and `name` + + let parse_list_of_dicts = |bytes: Vec| { + serde_json::from_slice::>>(&bytes) + .with_context(|| "Could not parse bencher output") + }; + + let token = std::env::var("BENCHER_API_TOKEN").ok(); + let mut bencher_cmd = Command::new("bencher"); + let benchmarks = bencher_cmd + .arg("benchmark") + .args(["list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list benchmarks")?; + if !benchmarks.status.success() { + bail!("Failed to list benchmarks: {:?}", benchmarks); + } + + // parse teh name and uuid pairs + let benchmarks = parse_list_of_dicts(benchmarks.stdout)? + .into_iter() + .map(|p| { + let name = p.get("name").expect("no name in project"); + let uuid = p.get("uuid").expect("no uuid in project"); + (name.clone(), uuid.clone()) + }) + .collect::>(); + + // delete all plots using bencher plot list bms to get "uuid's" + // then bencher plot delete bms + + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list plots")?; + + if !bencher_cmd.status.success() { + bail!("Failed to list plots: {:?}", bencher_cmd); + } + + let plots = parse_list_of_dicts(bencher_cmd.stdout)? + .into_iter() + .map(|p| { + let uuid = p.get("uuid").expect("no uuid in plot"); + uuid.clone() + }) + .collect::>(); + + for uuid in plots { + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "delete", "bms", &uuid]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not delete plot")?; + + if !bencher_cmd.status.success() { + bail!("Failed to delete plot: {:?}", bencher_cmd); + } + } + let testbeds = Command::new("bencher") + .arg("testbed") + .args(["list", "bms"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not list testbeds")?; + + if !testbeds.status.success() { + bail!("Failed to list testbeds: {:?}", testbeds); + } + + let testbeds = parse_list_of_dicts(testbeds.stdout)? + .into_iter() + .map(|p| { + let name = p.get("name").expect("no name in testbed"); + let uuid = p.get("uuid").expect("no uuid in testbed"); + (name.clone(), uuid.clone()) + }) + .filter(|(name, _)| name.contains("gha")) + .collect::>(); + + let group_to_benchmark_map: HashMap<_, Vec<_>> = + benchmarks + .iter() + .fold(HashMap::new(), |mut acc, (name, uuid)| { + let group = name.split('/').next().unwrap_or_default(); + acc.entry(group.to_owned()).or_default().push(uuid.clone()); + acc + }); + + // create plot using + // bencher plot create --x-axis date_time --branches main --testbeds --benchmarks --measures latency + + for (group, uuids) in group_to_benchmark_map { + for (testbed_name, testbed_uuid) in testbeds.iter() { + let without_gha = testbed_name.replace("-gha", ""); + let plot_name = format!("{without_gha} {group}"); + + let window_months = 12; + let window_seconds = window_months * 30 * 24 * 60 * 60; + let bencher_cmd = Command::new("bencher") + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .args(["plot", "create", "bms"]) + .args(["--title", &plot_name]) + .arg("--upper-boundary") + .args(["--x-axis", "date_time"]) + .args(["--window", &window_seconds.to_string()]) + .args(["--branches", "main"]) + .args(["--testbeds", testbed_uuid]) + .args(["--benchmarks", &uuids.join(",")]) + .args(["--measures", "latency"]) + .args(["--token", &token.clone().unwrap_or_default()]) + .output() + .with_context(|| "Could not create plot")?; + + if !bencher_cmd.status.success() { + bail!("Failed to create plot: {:?}", bencher_cmd); + } + } + } + Ok(()) }