From e373c51b9c1f49c86b2b3cfa8f30bce759f12a87 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 11:43:52 -0800 Subject: [PATCH 01/16] PointerId::Focus --- crates/bevy_picking/src/pointer.rs | 4 ++++ examples/ui/directional_navigation.rs | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/bevy_picking/src/pointer.rs b/crates/bevy_picking/src/pointer.rs index 42291f7a7daa6..df5fef2b8a26e 100644 --- a/crates/bevy_picking/src/pointer.rs +++ b/crates/bevy_picking/src/pointer.rs @@ -34,6 +34,10 @@ pub enum PointerId { Mouse, /// A touch input, usually numbered by window touch events from `winit`. Touch(u64), + /// An emulated pointer linked to the focused entity. + /// + /// Generally triggered by the `Enter` key or an `A` input on a gamepad. + Focus, /// A custom, uniquely identified pointer. Useful for mocking inputs or implementing a software /// controlled cursor. #[reflect(ignore)] diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index d80b24f3b7692..40f17dab2be03 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -387,8 +387,7 @@ fn interact_with_focused_button( commands.trigger_targets( Pointer:: { target: focused_entity, - // We're pretending that we're a mouse - pointer_id: PointerId::Mouse, + pointer_id: PointerId::Focus, // This field isn't used, so we're just setting it to a placeholder value pointer_location: Location { target: NormalizedRenderTarget::Image( From 424de779c59d9bb803732fe56142e35c500465c1 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 12:21:21 -0800 Subject: [PATCH 02/16] Fix typo spotted --- crates/bevy_picking/src/pointer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_picking/src/pointer.rs b/crates/bevy_picking/src/pointer.rs index df5fef2b8a26e..f1c30d019c8d6 100644 --- a/crates/bevy_picking/src/pointer.rs +++ b/crates/bevy_picking/src/pointer.rs @@ -267,7 +267,7 @@ pub enum PointerAction { pub struct PointerInput { /// The id of the pointer. pub pointer_id: PointerId, - /// The location of the pointer. For [[`PointerAction::Moved`]], this is the location after the movement. + /// The location of the pointer. For [`PointerAction::Moved`], this is the location after the movement. pub location: Location, /// The action that the event describes. pub action: PointerAction, From 133eb10209bbdeef820f685c42143ab2ecbd60a7 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 12:22:16 -0800 Subject: [PATCH 03/16] Docs note for UI picking backend --- crates/bevy_ui/src/picking_backend.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index b7d7db37a01e5..d74d7cba81a66 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -58,6 +58,9 @@ pub struct NodeQuery { /// /// Bevy's [`UiStack`] orders all nodes in the order they will be rendered, which is the same order /// we need for determining picking. +/// +/// Like all picking backends, this system reads the [`PointerId`] and [`PointerLocation`] components, +/// and produces [`PointerHits`] events. pub fn ui_picking( pointers: Query<(&PointerId, &PointerLocation)>, camera_query: Query<(Entity, &Camera, Has)>, From 20aa98cbdb20bc21eaf5424c0f61c297d21c801f Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 14:46:07 -0800 Subject: [PATCH 04/16] Spawn a default focus pointer --- crates/bevy_picking/src/input.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bevy_picking/src/input.rs b/crates/bevy_picking/src/input.rs index 94e195e4b3c78..1ca5054e76069 100644 --- a/crates/bevy_picking/src/input.rs +++ b/crates/bevy_picking/src/input.rs @@ -79,7 +79,7 @@ impl Default for PointerInputPlugin { impl Plugin for PointerInputPlugin { fn build(&self, app: &mut App) { app.insert_resource(*self) - .add_systems(Startup, spawn_mouse_pointer) + .add_systems(Startup, (spawn_mouse_pointer, spawn_focus_pointer)) .add_systems( First, ( @@ -103,6 +103,11 @@ pub fn spawn_mouse_pointer(mut commands: Commands) { commands.spawn(PointerId::Mouse); } +/// Spawns the default focus pointer. +pub fn spawn_focus_pointer(mut commands: Commands) { + commands.spawn(PointerId::Focus); +} + /// Sends mouse pointer events to be processed by the core plugin pub fn mouse_pick_events( // Input From ec9a56379b384f922b3f44b5ee2675037cf16820 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 14:56:10 -0800 Subject: [PATCH 05/16] More docs for PointerInput --- crates/bevy_picking/src/pointer.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/bevy_picking/src/pointer.rs b/crates/bevy_picking/src/pointer.rs index f1c30d019c8d6..b7c0beba86a95 100644 --- a/crates/bevy_picking/src/pointer.rs +++ b/crates/bevy_picking/src/pointer.rs @@ -244,6 +244,8 @@ impl Location { } /// Types of actions that can be taken by pointers. +/// +/// These are sent as the payload of [`PointerInput`] events. #[derive(Debug, Clone, Copy, Reflect)] pub enum PointerAction { /// A button has been pressed on the pointer. @@ -263,13 +265,25 @@ pub enum PointerAction { } /// An input event effecting a pointer. +/// +/// These events are generated from user input in the [`PointerInputPlugin`](crate::input::PointerInputPlugin) +/// and are read by the [`PointerInput::receive`] system +/// to modify the state of existing pointer entities. #[derive(Event, Debug, Clone, Reflect)] pub struct PointerInput { /// The id of the pointer. + /// + /// Used to identify which pointer entity to update. + /// If no match is found, the event is ignored: no new pointer entity is created. pub pointer_id: PointerId, /// The location of the pointer. For [`PointerAction::Moved`], this is the location after the movement. + /// + /// Defines exactly where, on the [`NormalizedRenderTarget`] that the pointer event came from, + /// the event occurred. pub location: Location, /// The action that the event describes. + /// + /// This is the action that the pointer took, such as pressing a button or moving. pub action: PointerAction, } @@ -306,6 +320,8 @@ impl PointerInput { } /// Updates pointer entities according to the input events. + /// + /// Entities are matched by their [`PointerId`]. If no match is found, the event is ignored. pub fn receive( mut events: EventReader, mut pointers: Query<(&PointerId, &mut PointerLocation, &mut PointerPress)>, From c8bcc82aa18fd251fbdbbe1748ac231c7b31c616 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 14:58:00 -0800 Subject: [PATCH 06/16] First attempt at a mocking API --- crates/bevy_ui/src/picking_backend.rs | 117 +++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index d74d7cba81a66..d7ed78f6c9c89 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -23,14 +23,17 @@ use crate::{focus::pick_rounded_rect, prelude::*, UiStack}; use bevy_app::prelude::*; -use bevy_ecs::{prelude::*, query::QueryData}; -use bevy_math::{Rect, Vec2}; +use bevy_ecs::{prelude::*, query::QueryData, system::SystemState}; +use bevy_math::{Rect, Vec2, Vec3Swizzles}; use bevy_render::prelude::*; use bevy_transform::prelude::*; use bevy_utils::HashMap; use bevy_window::PrimaryWindow; -use bevy_picking::backend::prelude::*; +use bevy_picking::{ + backend::prelude::*, + pointer::{Location, PointerAction, PointerInput}, +}; /// A plugin that adds picking support for UI nodes. #[derive(Clone)] @@ -221,3 +224,111 @@ pub fn ui_picking( output.send(PointerHits::new(*pointer, picks, order)); } } + +/// An extension trait for easily simulating pointer events on UI nodes. +/// +/// This is useful for driving UI interactions from keyboard or gamepad input, +/// but is also valuable for headless testing. +pub trait UiPointerMockingExt { + /// Simulate a [`Pointer`](bevy_picking::events::Pointer) event, + /// at the origin of the provided UI node. + /// + /// The entity that represents the [`PointerId`] provided should already exist, + /// as this method does not create it. + /// + /// Under the hood, this generates [`PointerInput`] events, + /// which then spawns a pointer entity with the [`PointerLocation`] and [`PointerId`] components, + /// which is read by the [`PointerInput::receive`] system to modify existing pointer entities, + /// and ultimately then processed into UI events by the [`ui_picking`] system. + /// + /// When using [`UiPlugin`](crate::UiPlugin), that system runs in the [`PreUpdate`] schedule, + /// under the [`PickSet::Backend`] set. + /// To ensure that these events are seen at the right time, + /// you should generally call this method in systems scheduled during [`First`], + /// as part of the [`PickSet::Input`] system set. + /// + /// # Warning + /// + /// If the node is not pickable, or is blocked by a higher node, + /// these events may not have any effect, even if sent correctly! + fn simulate_pointer_on_node( + &mut self, + pointer_id: PointerId, + pointer_action: PointerAction, + node: Entity, + ); +} + +impl UiPointerMockingExt for World { + // This method was constructed by copying the relevant parts of the `ui_picking` system, + // and should be kept in sync with it. + // TODO: make this method (and the command) fallible and define a proper error type + fn simulate_pointer_on_node( + &mut self, + pointer_id: PointerId, + pointer_action: PointerAction, + node_entity: Entity, + ) { + // Figure out which camera this node is associated with + let mut default_camera_system_state = SystemState::::new(self); + let default_ui_camera = default_camera_system_state.get(self); + let default_camera_entity = default_ui_camera.get(); + + let mut node_query = self.query::(); + let Ok(node_item) = node_query.get(self, node_entity) else { + return; + }; + + let Some(camera_entity) = node_item + .target_camera + .map(TargetCamera::entity) + .or(default_camera_entity) + else { + return; + }; + + // Find the primary window, needed to normalize the render target + let mut primary_window_query: QueryState> = + self.query_filtered::>(); + // If we find 0 or 2+ primary windows, treat it as if none were found + let maybe_primary_window_entity = primary_window_query.get_single(self).ok(); + + let Some(camera) = self.get::(camera_entity) else { + return; + }; + + // Generate the correct render target for the pointer + let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { + return; + }; + + // Calculate the pointer position in the render target + // For UI nodes, their final position is stored on their global transform, + // in pixels, with the origin at the top-left corner of the camera's viewport. + let Some(node_global_transform) = self.get::(node_entity) else { + return; + }; + let position = node_global_transform.translation().xy(); + + let pointer_location = Location { target, position }; + + self.send_event(PointerInput { + pointer_id, + location: pointer_location, + action: pointer_action, + }); + } +} + +impl UiPointerMockingExt for Commands<'_, '_> { + fn simulate_pointer_on_node( + &mut self, + pointer_id: PointerId, + pointer_action: PointerAction, + node: Entity, + ) { + self.queue(move |world: &mut World| { + world.simulate_pointer_on_node(pointer_id, pointer_action, node) + }); + } +} From 8c16a342dbeac21d3deaa4855a80eed79d468250 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 15:13:32 -0800 Subject: [PATCH 07/16] Make methods fallible and move to an EntityCommand --- crates/bevy_ui/src/picking_backend.rs | 166 ++++++++++++++------------ 1 file changed, 89 insertions(+), 77 deletions(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index d7ed78f6c9c89..b7e62e6a64b12 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -34,6 +34,7 @@ use bevy_picking::{ backend::prelude::*, pointer::{Location, PointerAction, PointerInput}, }; +use thiserror::Error; /// A plugin that adds picking support for UI nodes. #[derive(Clone)] @@ -225,6 +226,85 @@ pub fn ui_picking( } } +/// Simulates a pointer event on a UI node. +/// +/// See [`UiPointerMockingExt::simulate_pointer_on_node`] for more information, +/// and a method that wraps this function for use with commands. +pub fn simulate_pointer_on_node( + world: &mut World, + pointer_id: PointerId, + pointer_action: PointerAction, + node_entity: Entity, +) -> Result<(), SimulatedNodePointerError> { + // Figure out which camera this node is associated with + let mut default_camera_system_state = SystemState::::new(world); + let default_ui_camera = default_camera_system_state.get(world); + let default_camera_entity = default_ui_camera.get(); + + let mut node_query = world.query::(); + let Ok(node_item) = node_query.get(world, node_entity) else { + return Err(SimulatedNodePointerError::NodeNotFound(node_entity)); + }; + + let Some(camera_entity) = node_item + .target_camera + .map(TargetCamera::entity) + .or(default_camera_entity) + else { + return Err(SimulatedNodePointerError::NoCameraFound); + }; + + // Find the primary window, needed to normalize the render target + let mut primary_window_query: QueryState> = + world.query_filtered::>(); + // If we find 0 or 2+ primary windows, treat it as if none were found + let maybe_primary_window_entity = primary_window_query.get_single(world).ok(); + + let Some(camera) = world.get::(camera_entity) else { + return Err(SimulatedNodePointerError::NoCameraFound); + }; + + // Generate the correct render target for the pointer + let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { + return Err(SimulatedNodePointerError::CouldNotComputeRenderTarget); + }; + + // Calculate the pointer position in the render target + // For UI nodes, their final position is stored on their global transform, + // in pixels, with the origin at the top-left corner of the camera's viewport. + let Some(node_global_transform) = world.get::(node_entity) else { + return Err(SimulatedNodePointerError::NodeNotFound(node_entity)); + }; + + let position = node_global_transform.translation().xy(); + + let pointer_location = Location { target, position }; + + world.send_event(PointerInput { + pointer_id, + location: pointer_location, + action: pointer_action, + }); + + Ok(()) +} + +/// An error returned by [`simulate_pointer_on_node`]. +#[derive(Debug, PartialEq, Clone, Error)] +pub enum SimulatedNodePointerError { + /// The entity provided could not be found. + /// + /// It must have both a [`TargetCamera`] and a [`GlobalTransform`] component. + #[error("The entity {0:?} could not be found.")] + NodeNotFound(Entity), + /// The camera associated with the node could not be found. + #[error("No camera could be found for the node.")] + NoCameraFound, + /// The [`NormalizedRenderTarget`](bevy_render::camera::NormalizedRenderTarget) could not be computed. + #[error("Could not compute the normalized render target for the camera.")] + CouldNotComputeRenderTarget, +} + /// An extension trait for easily simulating pointer events on UI nodes. /// /// This is useful for driving UI interactions from keyboard or gamepad input, @@ -233,6 +313,8 @@ pub trait UiPointerMockingExt { /// Simulate a [`Pointer`](bevy_picking::events::Pointer) event, /// at the origin of the provided UI node. /// + /// Calls [`simulate_pointer_on_node`] when applied to the [`World`]. + /// /// The entity that represents the [`PointerId`] provided should already exist, /// as this method does not create it. /// @@ -251,84 +333,14 @@ pub trait UiPointerMockingExt { /// /// If the node is not pickable, or is blocked by a higher node, /// these events may not have any effect, even if sent correctly! - fn simulate_pointer_on_node( - &mut self, - pointer_id: PointerId, - pointer_action: PointerAction, - node: Entity, - ); + fn simulate_pointer_on_node(&mut self, pointer_id: PointerId, pointer_action: PointerAction); } -impl UiPointerMockingExt for World { - // This method was constructed by copying the relevant parts of the `ui_picking` system, - // and should be kept in sync with it. - // TODO: make this method (and the command) fallible and define a proper error type - fn simulate_pointer_on_node( - &mut self, - pointer_id: PointerId, - pointer_action: PointerAction, - node_entity: Entity, - ) { - // Figure out which camera this node is associated with - let mut default_camera_system_state = SystemState::::new(self); - let default_ui_camera = default_camera_system_state.get(self); - let default_camera_entity = default_ui_camera.get(); - - let mut node_query = self.query::(); - let Ok(node_item) = node_query.get(self, node_entity) else { - return; - }; - - let Some(camera_entity) = node_item - .target_camera - .map(TargetCamera::entity) - .or(default_camera_entity) - else { - return; - }; - - // Find the primary window, needed to normalize the render target - let mut primary_window_query: QueryState> = - self.query_filtered::>(); - // If we find 0 or 2+ primary windows, treat it as if none were found - let maybe_primary_window_entity = primary_window_query.get_single(self).ok(); - - let Some(camera) = self.get::(camera_entity) else { - return; - }; - - // Generate the correct render target for the pointer - let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { - return; - }; - - // Calculate the pointer position in the render target - // For UI nodes, their final position is stored on their global transform, - // in pixels, with the origin at the top-left corner of the camera's viewport. - let Some(node_global_transform) = self.get::(node_entity) else { - return; - }; - let position = node_global_transform.translation().xy(); - - let pointer_location = Location { target, position }; - - self.send_event(PointerInput { - pointer_id, - location: pointer_location, - action: pointer_action, - }); - } -} - -impl UiPointerMockingExt for Commands<'_, '_> { - fn simulate_pointer_on_node( - &mut self, - pointer_id: PointerId, - pointer_action: PointerAction, - node: Entity, - ) { - self.queue(move |world: &mut World| { - world.simulate_pointer_on_node(pointer_id, pointer_action, node) - }); +impl UiPointerMockingExt for EntityCommands<'_> { + fn simulate_pointer_on_node(&mut self, pointer_id: PointerId, pointer_action: PointerAction) { + let node_entity = self.id(); + self.commands_mut().queue(move |world: &mut World| { + simulate_pointer_on_node(world, pointer_id, pointer_action, node_entity) + }) } } From edfaed2f2da9b43f2bf1e6f9e2f0914b6a8979d0 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 15:24:57 -0800 Subject: [PATCH 08/16] Clean up code --- crates/bevy_ui/src/picking_backend.rs | 42 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index b7e62e6a64b12..a75ccebabefa8 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -237,21 +237,23 @@ pub fn simulate_pointer_on_node( node_entity: Entity, ) -> Result<(), SimulatedNodePointerError> { // Figure out which camera this node is associated with - let mut default_camera_system_state = SystemState::::new(world); - let default_ui_camera = default_camera_system_state.get(world); - let default_camera_entity = default_ui_camera.get(); - - let mut node_query = world.query::(); - let Ok(node_item) = node_query.get(world, node_entity) else { - return Err(SimulatedNodePointerError::NodeNotFound(node_entity)); - }; - - let Some(camera_entity) = node_item - .target_camera - .map(TargetCamera::entity) - .or(default_camera_entity) - else { - return Err(SimulatedNodePointerError::NoCameraFound); + let camera_entity = match world.get::(node_entity) { + // Use the `TargetCamera` component if it exists + Some(explicit_target_camera) => explicit_target_camera.entity(), + // If none is set, fall back to the default UI camera + None => { + // We're reusing the `DefaultUiCamera` system state here for consistency / correctness, + // but there's a few hoops to jump through when working with exclusive world access. + let mut default_ui_camera_system_state = SystemState::::new(world); + // Borrow checker decrees that this can't be one line. + let default_ui_camera = default_ui_camera_system_state.get(world); + + // Now we can finally try and find a default camera to fall back to + match default_ui_camera.get() { + Some(default_camera_entity) => default_camera_entity, + None => return Err(SimulatedNodePointerError::NoCameraFound), + } + } }; // Find the primary window, needed to normalize the render target @@ -260,11 +262,11 @@ pub fn simulate_pointer_on_node( // If we find 0 or 2+ primary windows, treat it as if none were found let maybe_primary_window_entity = primary_window_query.get_single(world).ok(); + // Generate the correct render target for the pointer let Some(camera) = world.get::(camera_entity) else { return Err(SimulatedNodePointerError::NoCameraFound); }; - // Generate the correct render target for the pointer let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { return Err(SimulatedNodePointerError::CouldNotComputeRenderTarget); }; @@ -294,10 +296,16 @@ pub fn simulate_pointer_on_node( pub enum SimulatedNodePointerError { /// The entity provided could not be found. /// - /// It must have both a [`TargetCamera`] and a [`GlobalTransform`] component. + /// It must have a [`GlobalTransform`] component, + /// and should have a [`Node`] component. #[error("The entity {0:?} could not be found.")] NodeNotFound(Entity), /// The camera associated with the node could not be found. + /// + /// Did you forget to spawn a camera entity with the [`Camera`] component? + /// + /// The [`TargetCamera`] component can be used to associate a camera with a node, + /// but if it is not present, the [`DefaultUiCamera`] will be used. #[error("No camera could be found for the node.")] NoCameraFound, /// The [`NormalizedRenderTarget`](bevy_render::camera::NormalizedRenderTarget) could not be computed. From 0d10c913aecd5e0efa20d91a76fa7ca45f24ebe8 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 15:32:45 -0800 Subject: [PATCH 09/16] Use new method in example --- examples/ui/directional_navigation.rs | 47 ++++++++------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index 40f17dab2be03..3b675afa76b67 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -12,13 +12,13 @@ use bevy::{ }, InputDispatchPlugin, InputFocus, InputFocusVisible, }, - math::{CompassOctant, FloatOrd}, + math::CompassOctant, picking::{ - backend::HitData, - pointer::{Location, PointerId}, + pointer::{PointerAction, PointerId, PressDirection}, + PickSet, }, prelude::*, - render::camera::NormalizedRenderTarget, + ui::picking_backend::UiPointerMockingExt, utils::{HashMap, HashSet}, }; @@ -39,13 +39,14 @@ fn main() { // Input is generally handled during PreUpdate // We're turning inputs into actions first, then using those actions to determine navigation .add_systems(PreUpdate, (process_inputs, navigate).chain()) + // We're simulating a pointer click on the focused button + // and are running this system at the same time as the other pointer input systems + .add_systems(First, interact_with_focused_button.in_set(PickSet::Input)) .add_systems( Update, ( // We need to show which button is currently focused highlight_focused_element, - // Pressing the "Interact" button while we have a focused element should simulate a click - interact_with_focused_button, // We're doing a tiny animation when the button is interacted with, // so we need a timer and a polling mechanism to reset it reset_button_after_interaction, @@ -372,8 +373,8 @@ fn highlight_focused_element( } } -// By sending a Pointer trigger rather than directly handling button-like interactions, -// we can unify our handling of pointer and keyboard/gamepad interactions +// We're emulating a pointer event sent to the focused button, +// which will be picked up just like any mouse or touch input! fn interact_with_focused_button( action_state: Res, input_focus: Res, @@ -384,32 +385,12 @@ fn interact_with_focused_button( .contains(&DirectionalNavigationAction::Select) { if let Some(focused_entity) = input_focus.0 { - commands.trigger_targets( - Pointer:: { - target: focused_entity, - pointer_id: PointerId::Focus, - // This field isn't used, so we're just setting it to a placeholder value - pointer_location: Location { - target: NormalizedRenderTarget::Image( - bevy_render::camera::ImageRenderTarget { - handle: Handle::default(), - scale_factor: FloatOrd(1.0), - }, - ), - position: Vec2::ZERO, - }, - event: Pressed { - button: PointerButton::Primary, - // This field isn't used, so we're just setting it to a placeholder value - hit: HitData { - camera: Entity::PLACEHOLDER, - depth: 0.0, - position: None, - normal: None, - }, - }, + commands.entity(focused_entity).simulate_pointer_on_node( + PointerId::Focus, + PointerAction::Pressed { + direction: PressDirection::Pressed, + button: PointerButton::Primary, }, - focused_entity, ); } } From b9eb81de2717189a0fe611201e4cfd7e0a1364b6 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 17:48:18 -0800 Subject: [PATCH 10/16] Remove Component derive from Location --- crates/bevy_picking/src/pointer.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/bevy_picking/src/pointer.rs b/crates/bevy_picking/src/pointer.rs index b7c0beba86a95..0e35e89c2325c 100644 --- a/crates/bevy_picking/src/pointer.rs +++ b/crates/bevy_picking/src/pointer.rs @@ -201,13 +201,15 @@ impl PointerLocation { /// The location of a pointer, including the current [`NormalizedRenderTarget`], and the x/y /// position of the pointer on this render target. /// +/// This is stored as a [`PointerLocation`] component on the pointer entity. +/// /// Note that: /// - a pointer can move freely between render targets /// - a pointer is not associated with a [`Camera`] because multiple cameras can target the same /// render target. It is up to picking backends to associate a Pointer's `Location` with a /// specific `Camera`, if any. -#[derive(Debug, Clone, Component, Reflect, PartialEq)] -#[reflect(Component, Debug, PartialEq)] +#[derive(Debug, Clone, Reflect, PartialEq)] +#[reflect(Debug, PartialEq)] pub struct Location { /// The [`NormalizedRenderTarget`] associated with the pointer, usually a window. pub target: NormalizedRenderTarget, From c091388342786b8f1e7b0afc2a9d05cada7cfb48 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 17:57:54 -0800 Subject: [PATCH 11/16] Also release the focus press --- examples/ui/directional_navigation.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index 3b675afa76b67..188f76d37aed6 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -393,5 +393,18 @@ fn interact_with_focused_button( }, ); } + } else { + // If the button was not pressed, we simulate a release event + if let Some(focused_entity) = input_focus.0 { + commands.entity(focused_entity).simulate_pointer_on_node( + PointerId::Focus, + PointerAction::Pressed { + direction: PressDirection::Released, + button: PointerButton::Primary, + }, + ); + } + } +} } } From b4d1b25acccd10fda12b514db58c0dbbe493a2d2 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 18:07:21 -0800 Subject: [PATCH 12/16] Shorter name --- crates/bevy_ui/src/picking_backend.rs | 4 ++-- examples/ui/directional_navigation.rs | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index a75ccebabefa8..0b608c01c8819 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -341,11 +341,11 @@ pub trait UiPointerMockingExt { /// /// If the node is not pickable, or is blocked by a higher node, /// these events may not have any effect, even if sent correctly! - fn simulate_pointer_on_node(&mut self, pointer_id: PointerId, pointer_action: PointerAction); + fn emulate_pointer(&mut self, pointer_id: PointerId, pointer_action: PointerAction); } impl UiPointerMockingExt for EntityCommands<'_> { - fn simulate_pointer_on_node(&mut self, pointer_id: PointerId, pointer_action: PointerAction) { + fn emulate_pointer(&mut self, pointer_id: PointerId, pointer_action: PointerAction) { let node_entity = self.id(); self.commands_mut().queue(move |world: &mut World| { simulate_pointer_on_node(world, pointer_id, pointer_action, node_entity) diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index 188f76d37aed6..c58e56bc4f52c 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -385,7 +385,7 @@ fn interact_with_focused_button( .contains(&DirectionalNavigationAction::Select) { if let Some(focused_entity) = input_focus.0 { - commands.entity(focused_entity).simulate_pointer_on_node( + commands.entity(focused_entity).emulate_pointer( PointerId::Focus, PointerAction::Pressed { direction: PressDirection::Pressed, @@ -396,7 +396,7 @@ fn interact_with_focused_button( } else { // If the button was not pressed, we simulate a release event if let Some(focused_entity) = input_focus.0 { - commands.entity(focused_entity).simulate_pointer_on_node( + commands.entity(focused_entity).emulate_pointer( PointerId::Focus, PointerAction::Pressed { direction: PressDirection::Released, @@ -405,6 +405,4 @@ fn interact_with_focused_button( ); } } -} - } } From cbeed130087adacfdff4481d9b7b964e252860f7 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 18:21:35 -0800 Subject: [PATCH 13/16] Remember to send PointerAction::Moved events --- examples/ui/directional_navigation.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index c58e56bc4f52c..63f168b18f27b 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -39,9 +39,16 @@ fn main() { // Input is generally handled during PreUpdate // We're turning inputs into actions first, then using those actions to determine navigation .add_systems(PreUpdate, (process_inputs, navigate).chain()) - // We're simulating a pointer click on the focused button + // We're simulating pointer clicks on the focused button // and are running this system at the same time as the other pointer input systems - .add_systems(First, interact_with_focused_button.in_set(PickSet::Input)) + .add_systems( + First, + ( + move_pointer_to_focused_element, + interact_with_focused_button, + ) + .in_set(PickSet::Input), + ) .add_systems( Update, ( @@ -355,6 +362,19 @@ fn navigate(action_state: Res, mut directional_navigation: Directio } } +// We need to update the location of the focused element +// so that our virtual pointer presses are sent to the correct location. +// +// Doing this in an ordinary system (rather than just on navigation change) ensures that it is +// initialized correctly, and is updated if the focused element moves or is scaled for any reason. +fn move_pointer_to_focused_element(input_focus: Res, mut commands: Commands) { + if let Some(focused_entity) = input_focus.0 { + commands + .entity(focused_entity) + .emulate_pointer(PointerId::Focus, PointerAction::Moved { delta: Vec2::ZERO }); + } +} + fn highlight_focused_element( input_focus: Res, // While this isn't strictly needed for the example, From 5b0062b025ef649c9383c463d011a17cfeec2f2c Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 18:56:26 -0800 Subject: [PATCH 14/16] Refactor to use a SystemParam instead --- crates/bevy_ui/src/picking_backend.rs | 172 ++++++++++++-------------- examples/ui/directional_navigation.rs | 53 +++++--- 2 files changed, 110 insertions(+), 115 deletions(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index 0b608c01c8819..43f2558f4cf22 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -23,7 +23,7 @@ use crate::{focus::pick_rounded_rect, prelude::*, UiStack}; use bevy_app::prelude::*; -use bevy_ecs::{prelude::*, query::QueryData, system::SystemState}; +use bevy_ecs::{prelude::*, query::QueryData, system::SystemParam}; use bevy_math::{Rect, Vec2, Vec3Swizzles}; use bevy_render::prelude::*; use bevy_transform::prelude::*; @@ -226,69 +226,91 @@ pub fn ui_picking( } } -/// Simulates a pointer event on a UI node. -/// -/// See [`UiPointerMockingExt::simulate_pointer_on_node`] for more information, -/// and a method that wraps this function for use with commands. -pub fn simulate_pointer_on_node( - world: &mut World, - pointer_id: PointerId, - pointer_action: PointerAction, - node_entity: Entity, -) -> Result<(), SimulatedNodePointerError> { - // Figure out which camera this node is associated with - let camera_entity = match world.get::(node_entity) { - // Use the `TargetCamera` component if it exists - Some(explicit_target_camera) => explicit_target_camera.entity(), - // If none is set, fall back to the default UI camera - None => { - // We're reusing the `DefaultUiCamera` system state here for consistency / correctness, - // but there's a few hoops to jump through when working with exclusive world access. - let mut default_ui_camera_system_state = SystemState::::new(world); - // Borrow checker decrees that this can't be one line. - let default_ui_camera = default_ui_camera_system_state.get(world); - - // Now we can finally try and find a default camera to fall back to - match default_ui_camera.get() { +/// A [`SystemParam`] for realistically simulating/mocking pointer events on UI nodes. +#[derive(SystemParam)] +pub struct EmulateNodePointerEvents<'w, 's> { + /// Looks up information about the node that the pointer event should be simulated on. + pub node_query: Query<'w, 's, (&'static GlobalTransform, Option<&'static TargetCamera>)>, + /// Tries to find the default UI camera + pub default_ui_camera: DefaultUiCamera<'w, 's>, + /// Tries to find a primary window entity. + pub primary_window_query: Query<'w, 's, Entity, With>, + /// Looks up the required camera information. + pub camera_query: Query<'w, 's, &'static Camera>, + /// Writes the pointer events to the world. + pub pointer_input_events: EventWriter<'w, PointerInput>, +} + +impl<'w, 's> EmulateNodePointerEvents<'w, 's> { + /// Simulate a [`Pointer`](bevy_picking::events::Pointer) event, + /// at the origin of the provided UI node entity. + /// + /// The entity that represents the [`PointerId`] provided should already exist, + /// as this method does not create it. + /// + /// Under the hood, this generates [`PointerInput`] events, + /// which is read by the [`PointerInput::receive`] system to modify existing pointer entities, + /// and ultimately then processed into UI events by the [`ui_picking`] system. + /// + /// When using [`UiPlugin`](crate::UiPlugin), that system runs in the [`PreUpdate`] schedule, + /// under the [`PickSet::Backend`] set. + /// To ensure that these events are seen at the right time, + /// you should generally call this method in systems scheduled during [`First`], + /// as part of the [`PickSet::Input`] system set. + /// + /// # Warning + /// + /// If the node is not pickable, or is blocked by a higher node, + /// these events may not have any effect, even if sent correctly! + pub fn emulate_pointer( + self: &mut Self, + pointer_id: PointerId, + pointer_action: PointerAction, + entity: Entity, + ) -> Result<(), SimulatedNodePointerError> { + // Look up the node we're trying to send a pointer event to + let Ok((global_transform, maybe_target_camera)) = self.node_query.get(entity) else { + return Err(SimulatedNodePointerError::NodeNotFound(entity)); + }; + + // Figure out which camera this node is associated with + let camera_entity = match maybe_target_camera { + Some(explicit_target_camera) => explicit_target_camera.entity(), + // Fall back to the default UI camera + None => match self.default_ui_camera.get() { Some(default_camera_entity) => default_camera_entity, None => return Err(SimulatedNodePointerError::NoCameraFound), - } - } - }; - - // Find the primary window, needed to normalize the render target - let mut primary_window_query: QueryState> = - world.query_filtered::>(); - // If we find 0 or 2+ primary windows, treat it as if none were found - let maybe_primary_window_entity = primary_window_query.get_single(world).ok(); + }, + }; - // Generate the correct render target for the pointer - let Some(camera) = world.get::(camera_entity) else { - return Err(SimulatedNodePointerError::NoCameraFound); - }; + // Find the primary window, needed to normalize the render target + // If we find 0 or 2+ primary windows, treat it as if none were found + let maybe_primary_window_entity = self.primary_window_query.get_single().ok(); - let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { - return Err(SimulatedNodePointerError::CouldNotComputeRenderTarget); - }; + // Generate the correct render target for the pointer + let Ok(camera) = self.camera_query.get(camera_entity) else { + return Err(SimulatedNodePointerError::NoCameraFound); + }; - // Calculate the pointer position in the render target - // For UI nodes, their final position is stored on their global transform, - // in pixels, with the origin at the top-left corner of the camera's viewport. - let Some(node_global_transform) = world.get::(node_entity) else { - return Err(SimulatedNodePointerError::NodeNotFound(node_entity)); - }; + let Some(target) = camera.target.normalize(maybe_primary_window_entity) else { + return Err(SimulatedNodePointerError::CouldNotComputeRenderTarget); + }; - let position = node_global_transform.translation().xy(); + // Calculate the pointer position in the render target + // For UI nodes, their final position is stored on their global transform, + // in pixels, with the origin at the top-left corner of the camera's viewport. + let position = global_transform.translation().xy(); - let pointer_location = Location { target, position }; + let pointer_location = Location { target, position }; - world.send_event(PointerInput { - pointer_id, - location: pointer_location, - action: pointer_action, - }); + self.pointer_input_events.send(PointerInput { + pointer_id, + location: pointer_location, + action: pointer_action, + }); - Ok(()) + Ok(()) + } } /// An error returned by [`simulate_pointer_on_node`]. @@ -312,43 +334,3 @@ pub enum SimulatedNodePointerError { #[error("Could not compute the normalized render target for the camera.")] CouldNotComputeRenderTarget, } - -/// An extension trait for easily simulating pointer events on UI nodes. -/// -/// This is useful for driving UI interactions from keyboard or gamepad input, -/// but is also valuable for headless testing. -pub trait UiPointerMockingExt { - /// Simulate a [`Pointer`](bevy_picking::events::Pointer) event, - /// at the origin of the provided UI node. - /// - /// Calls [`simulate_pointer_on_node`] when applied to the [`World`]. - /// - /// The entity that represents the [`PointerId`] provided should already exist, - /// as this method does not create it. - /// - /// Under the hood, this generates [`PointerInput`] events, - /// which then spawns a pointer entity with the [`PointerLocation`] and [`PointerId`] components, - /// which is read by the [`PointerInput::receive`] system to modify existing pointer entities, - /// and ultimately then processed into UI events by the [`ui_picking`] system. - /// - /// When using [`UiPlugin`](crate::UiPlugin), that system runs in the [`PreUpdate`] schedule, - /// under the [`PickSet::Backend`] set. - /// To ensure that these events are seen at the right time, - /// you should generally call this method in systems scheduled during [`First`], - /// as part of the [`PickSet::Input`] system set. - /// - /// # Warning - /// - /// If the node is not pickable, or is blocked by a higher node, - /// these events may not have any effect, even if sent correctly! - fn emulate_pointer(&mut self, pointer_id: PointerId, pointer_action: PointerAction); -} - -impl UiPointerMockingExt for EntityCommands<'_> { - fn emulate_pointer(&mut self, pointer_id: PointerId, pointer_action: PointerAction) { - let node_entity = self.id(); - self.commands_mut().queue(move |world: &mut World| { - simulate_pointer_on_node(world, pointer_id, pointer_action, node_entity) - }) - } -} diff --git a/examples/ui/directional_navigation.rs b/examples/ui/directional_navigation.rs index 63f168b18f27b..c2ea35d8b5b05 100644 --- a/examples/ui/directional_navigation.rs +++ b/examples/ui/directional_navigation.rs @@ -18,7 +18,7 @@ use bevy::{ PickSet, }, prelude::*, - ui::picking_backend::UiPointerMockingExt, + ui::picking_backend::EmulateNodePointerEvents, utils::{HashMap, HashSet}, }; @@ -367,11 +367,18 @@ fn navigate(action_state: Res, mut directional_navigation: Directio // // Doing this in an ordinary system (rather than just on navigation change) ensures that it is // initialized correctly, and is updated if the focused element moves or is scaled for any reason. -fn move_pointer_to_focused_element(input_focus: Res, mut commands: Commands) { +fn move_pointer_to_focused_element( + input_focus: Res, + mut emulate_pointer_events: EmulateNodePointerEvents, +) { if let Some(focused_entity) = input_focus.0 { - commands - .entity(focused_entity) - .emulate_pointer(PointerId::Focus, PointerAction::Moved { delta: Vec2::ZERO }); + emulate_pointer_events + .emulate_pointer( + PointerId::Focus, + PointerAction::Moved { delta: Vec2::ZERO }, + focused_entity, + ) + .unwrap_or_else(|e| warn!("Failed to move pointer: {e}")); } } @@ -398,31 +405,37 @@ fn highlight_focused_element( fn interact_with_focused_button( action_state: Res, input_focus: Res, - mut commands: Commands, + mut emulate_pointer_events: EmulateNodePointerEvents, ) { if action_state .pressed_actions .contains(&DirectionalNavigationAction::Select) { if let Some(focused_entity) = input_focus.0 { - commands.entity(focused_entity).emulate_pointer( - PointerId::Focus, - PointerAction::Pressed { - direction: PressDirection::Pressed, - button: PointerButton::Primary, - }, - ); + emulate_pointer_events + .emulate_pointer( + PointerId::Focus, + PointerAction::Pressed { + direction: PressDirection::Pressed, + button: PointerButton::Primary, + }, + focused_entity, + ) + .unwrap_or_else(|e| warn!("Failed to press pointer: {e}")); } } else { // If the button was not pressed, we simulate a release event if let Some(focused_entity) = input_focus.0 { - commands.entity(focused_entity).emulate_pointer( - PointerId::Focus, - PointerAction::Pressed { - direction: PressDirection::Released, - button: PointerButton::Primary, - }, - ); + emulate_pointer_events + .emulate_pointer( + PointerId::Focus, + PointerAction::Pressed { + direction: PressDirection::Released, + button: PointerButton::Primary, + }, + focused_entity, + ) + .unwrap_or_else(|e| warn!("Failed to release pointer: {e}")); } } } From 75fdc097974477dfb5d567fc6e251441215d5d6d Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 19:04:31 -0800 Subject: [PATCH 15/16] Clippy --- crates/bevy_ui/src/picking_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index 43f2558f4cf22..12de878cd80ce 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -263,7 +263,7 @@ impl<'w, 's> EmulateNodePointerEvents<'w, 's> { /// If the node is not pickable, or is blocked by a higher node, /// these events may not have any effect, even if sent correctly! pub fn emulate_pointer( - self: &mut Self, + &mut self, pointer_id: PointerId, pointer_action: PointerAction, entity: Entity, From ef20d18a2239207a01428becad48c091cb3cb220 Mon Sep 17 00:00:00 2001 From: Alice Cecile Date: Wed, 15 Jan 2025 19:41:18 -0800 Subject: [PATCH 16/16] Update doc strings for refactor --- crates/bevy_ui/src/picking_backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs index 12de878cd80ce..13a55f5ba87c6 100644 --- a/crates/bevy_ui/src/picking_backend.rs +++ b/crates/bevy_ui/src/picking_backend.rs @@ -313,7 +313,7 @@ impl<'w, 's> EmulateNodePointerEvents<'w, 's> { } } -/// An error returned by [`simulate_pointer_on_node`]. +/// An error returned by [`EmulateNodePointerEvents`]. #[derive(Debug, PartialEq, Clone, Error)] pub enum SimulatedNodePointerError { /// The entity provided could not be found.