Add bevy_picking sprite backend

Cargo.toml

@@ -3376,6 +3376,17 @@ description = "Demonstrates how to use picking events to spawn simple objects"
 category = "Picking"
 wasm = true
 
+[[example]]
+name = "sprite_picking"
+path = "examples/picking/sprite_picking.rs"
+doc-scrape-examples = true
+
+[package.metadata.example.sprite_picking]
+name = "Sprite Picking"
+description = "Demonstrates picking sprites and sprite atlases"
+category = "Picking"
+wasm = true
+
 [profile.wasm-release]
 inherits = "release"
 opt-level = "z"

crates/bevy_internal/Cargo.toml

@@ -191,7 +191,11 @@ meshlet_processor = ["bevy_pbr?/meshlet_processor"]
 bevy_dev_tools = ["dep:bevy_dev_tools"]
 
 # Provides a picking functionality
-bevy_picking = ["dep:bevy_picking", "bevy_ui?/bevy_picking"]
+bevy_picking = [
+ "dep:bevy_picking",
+ "bevy_ui?/bevy_picking",
+ "bevy_sprite?/bevy_picking",
+]
 
 # Enable support for the ios_simulator by downgrading some rendering capabilities
 ios_simulator = ["bevy_pbr?/ios_simulator", "bevy_render?/ios_simulator"]

crates/bevy_picking/src/backend.rs

@@ -50,6 +50,10 @@ pub mod prelude {
 /// Some backends may only support providing the topmost entity; this is a valid limitation of some
 /// backends. For example, a picking shader might only have data on the topmost rendered output from
 /// its buffer.
+///
+/// Note that systems reading these events in [`PreUpdate`](bevy_app) will not report ordering
+/// ambiguities with picking backends. Take care to ensure such systems are explicitly ordered
+/// against [`PickSet::Backends`](crate), or better, avoid reading `PointerHits` in `PreUpdate`.
 #[derive(Event, Debug, Clone)]
 pub struct PointerHits {
  /// The pointer associated with this hit test.

crates/bevy_picking/src/lib.rs

@@ -207,6 +207,10 @@ impl Plugin for PickingPlugin {
  .add_event::<pointer::InputPress>()
  .add_event::<pointer::InputMove>()
  .add_event::<backend::PointerHits>()
+ // Rather than try to mark all current and future backends as ambiguous with each other,
+ // we allow them to send their hits in any order. These are later sorted, so submission
+ // order doesn't matter. See `PointerHits` docs for caveats.
+ .allow_ambiguous_resource::<Events<backend::PointerHits>>()
  .add_systems(
  PreUpdate,
  (

crates/bevy_sprite/Cargo.toml

@@ -9,6 +9,7 @@ license = "MIT OR Apache-2.0"
 keywords = ["bevy"]
 
 [features]
+bevy_picking = ["dep:bevy_picking", "dep:bevy_window"]
 webgl = []
 webgpu = []
 
@@ -20,12 +21,14 @@ bevy_color = { path = "../bevy_color", version = "0.15.0-dev" }
 bevy_core_pipeline = { path = "../bevy_core_pipeline", version = "0.15.0-dev" }
 bevy_ecs = { path = "../bevy_ecs", version = "0.15.0-dev" }
 bevy_math = { path = "../bevy_math", version = "0.15.0-dev" }
+bevy_picking = { path = "../bevy_picking", version = "0.15.0-dev", optional = true }
 bevy_reflect = { path = "../bevy_reflect", version = "0.15.0-dev", features = [
  "bevy",
 ] }
 bevy_render = { path = "../bevy_render", version = "0.15.0-dev" }
 bevy_transform = { path = "../bevy_transform", version = "0.15.0-dev" }
 bevy_utils = { path = "../bevy_utils", version = "0.15.0-dev" }
+bevy_window = { path = "../bevy_window", version = "0.15.0-dev", optional = true }
 bevy_derive = { path = "../bevy_derive", version = "0.15.0-dev" }
 
 # other

crates/bevy_sprite/src/lib.rs

@@ -11,6 +11,8 @@
 mod bundle;
 mod dynamic_texture_atlas_builder;
 mod mesh2d;
+#[cfg(feature = "bevy_picking")]
+mod picking_backend;
 mod render;
 mod sprite;
 mod texture_atlas;
@@ -133,6 +135,9 @@ impl Plugin for SpritePlugin {
  ),
  );
 
+ #[cfg(feature = "bevy_picking")]
+ app.add_plugins(picking_backend::SpritePickingBackend);
+
  if let Some(render_app) = app.get_sub_app_mut(RenderApp) {
  render_app
  .init_resource::<ImageBindGroups>()

crates/bevy_sprite/src/picking_backend.rs

@@ -0,0 +1,171 @@
+//! A [`bevy_picking`] backend for sprites. Works for simple sprites and sprite atlases. Works for
+//! sprites with arbitrary transforms. Picking is done based on sprite bounds, not visible pixels.
+//! This means a partially transparent sprite is pickable even in its transparent areas.
+
+use std::cmp::Ordering;
+
+use crate::{Sprite, TextureAtlas, TextureAtlasLayout};
+use bevy_app::prelude::*;
+use bevy_asset::prelude::*;
+use bevy_ecs::prelude::*;
+use bevy_math::{prelude::*, FloatExt};
+use bevy_picking::backend::prelude::*;
+use bevy_render::prelude::*;
+use bevy_transform::prelude::*;
+use bevy_window::PrimaryWindow;
+
+#[derive(Clone)]
+pub struct SpritePickingBackend;
+
+impl Plugin for SpritePickingBackend {
+ fn build(&self, app: &mut App) {
+ app.add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend));
+ }
+}
+
+pub fn sprite_picking(
+ pointers: Query<(&PointerId, &PointerLocation)>,
+ cameras: Query<(Entity, &Camera, &GlobalTransform, &OrthographicProjection)>,
+ primary_window: Query<Entity, With<PrimaryWindow>>,
+ images: Res<Assets<Image>>,
+ texture_atlas_layout: Res<Assets<TextureAtlasLayout>>,
+ sprite_query: Query<
+ (
+ Entity,
+ Option<&Sprite>,
+ Option<&TextureAtlas>,
+ Option<&Handle<Image>>,
+ &GlobalTransform,
+ Option<&Pickable>,
+ &ViewVisibility,
+ ),
+ Or<(With<Sprite>, With<TextureAtlas>)>,
+ >,
+ mut output: EventWriter<PointerHits>,
+) {
+ let mut sorted_sprites: Vec<_> = sprite_query.iter().collect();
+ sorted_sprites.sort_by(|a, b| {
+ (b.4.translation().z)
+ .partial_cmp(&a.4.translation().z)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ let primary_window = primary_window.get_single().ok();
+
+ for (pointer, location) in pointers.iter().filter_map(|(pointer, pointer_location)| {
+ pointer_location.location().map(|loc| (pointer, loc))
+ }) {
+ let mut blocked = false;
+ let Some((cam_entity, camera, cam_transform, cam_ortho)) = cameras
+ .iter()
+ .filter(|(_, camera, _, _)| camera.is_active)
+ .find(|(_, camera, _, _)| {
+ camera
+ .target
+ .normalize(primary_window)
+ .map(|x| x == location.target)
+ .unwrap_or(false)
+ })
+ else {
+ continue;
+ };
+
+ let Some(cursor_ray_world) = camera.viewport_to_world(cam_transform, location.position)
+ else {
+ continue;
+ };
+ let cursor_ray_len = cam_ortho.far - cam_ortho.near;
+ let cursor_ray_end = cursor_ray_world.origin + cursor_ray_world.direction * cursor_ray_len;
+
+ let picks: Vec<(Entity, HitData)> = sorted_sprites
+ .iter()
+ .copied()
+ .filter(|(.., visibility)| visibility.get())
+ .filter_map(
+ |(entity, sprite, atlas, image, sprite_transform, pickable, ..)| {
+ if blocked {
+ return None;
+ }
+
+ // Hit box in sprite coordinate system
+ let (extents, anchor) = if let Some((sprite, atlas)) = sprite.zip(atlas) {
+ let extents = sprite.custom_size.or_else(|| {
+ texture_atlas_layout
+ .get(&atlas.layout)
+ .map(|f| f.textures[atlas.index].size().as_vec2())
+ })?;
+ let anchor = sprite.anchor.as_vec();
+ (extents, anchor)
+ } else if let Some((sprite, image)) = sprite.zip(image) {
+ let extents = sprite
+ .custom_size
+ .or_else(|| images.get(image).map(|f| f.size().as_vec2()))?;
+ let anchor = sprite.anchor.as_vec();
+ (extents, anchor)
+ } else {
+ return None;
+ };
+
+ let center = -anchor * extents;
+ let rect = Rect::from_center_half_size(center, extents / 2.0);
+
+ // Transform cursor line segment to sprite coordinate system
+ let world_to_sprite = sprite_transform.affine().inverse();
+ let cursor_start_sprite =
+ world_to_sprite.transform_point3(cursor_ray_world.origin);
+ let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);
+
+ // Find where the cursor segment intersects the plane Z=0 (which is the sprite's
+ // plane in sprite-local space). It may not intersect if, for example, we're
+ // viewing the sprite side-on
+ if cursor_start_sprite.z == cursor_end_sprite.z {
+ // Cursor ray is parallel to the sprite and misses it
+ return None;
+ }
+ let lerp_factor =
+ f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
+ if !(0.0..=1.0).contains(&lerp_factor) {
+ // Lerp factor is out of range, meaning that while an infinite line cast by
+ // the cursor would intersect the sprite, the sprite is not between the
+ // camera's near and far planes
+ return None;
+ }
+ // Otherwise we can interpolate the xy of the start and end positions by the
+ // lerp factor to get the cursor position in sprite space!
+ let cursor_pos_sprite = cursor_start_sprite
+ .lerp(cursor_end_sprite, lerp_factor)
+ .xy();
+
+ let is_cursor_in_sprite = rect.contains(cursor_pos_sprite);
+
+ blocked = is_cursor_in_sprite
+ && pickable.map(|p| p.should_block_lower) != Some(false);
+
+ is_cursor_in_sprite.then(|| {
+ let hit_pos_world =
+ sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
+ // Transform point from world to camera space to get the Z distance
+ let hit_pos_cam = cam_transform
+ .affine()
+ .inverse()
+ .transform_point3(hit_pos_world);
+ // HitData requires a depth as calculated from the camera's near clipping plane
+ let depth = -cam_ortho.near - hit_pos_cam.z;
+ (
+ entity,
+ HitData::new(
+ cam_entity,
+ depth,
+ Some(hit_pos_world),
+ Some(*sprite_transform.back()),
+ ),
+ )
+ })
+ },
+ )
+ .collect();
+
+ let order = camera.order as f32;
+ output.send(PointerHits::new(*pointer, picks, order));
+ }
+}

examples/README.md

@@ -357,6 +357,7 @@ Example | Description
 Example | Description
 --- | ---
 [Showcases simple picking events and usage](../examples/picking/simple_picking.rs) | Demonstrates how to use picking events to spawn simple objects
+[Sprite Picking](../examples/picking/sprite_picking.rs) | Demonstrates picking sprites and sprite atlases
 
 ## Reflection