Add bevy_picking sprite backend (#14757)

# Objective

Add `bevy_picking` sprite backend as part of the `bevy_mod_picking`
upstreamening (#12365).

## Solution

More or less a copy/paste from `bevy_mod_picking`, with the changes
[here](aevyrie/bevy_mod_picking#354). I'm
putting that link here since those changes haven't yet made it through
review, so should probably be reviewed on their own.

## Testing

I couldn't find any sprite-backend-specific tests in `bevy_mod_picking`
and unfortunately I'm not familiar enough with Bevy's testing patterns
to write tests for code that relies on windowing and input. I'm willing
to break the pointer hit system into testable blocks and add some more
modular tests if that's deemed important enough to block, otherwise I
can open an issue for adding tests as follow-up.

## Follow-up work

- More docs/tests
- Ignore pick events on transparent sprite pixels with potential opt-out

---------

Co-authored-by: Aevyrie <aevyrie@gmail.com>

Cargo.toml

@@ -3384,6 +3384,17 @@ description = "Demonstrates how to use picking events to spawn simple objects"
 category = "Picking"
 wasm = true
 
+[[example]]
+name = "sprite_picking"
+path = "examples/picking/sprite_picking.rs"
+doc-scrape-examples = true
+
+[package.metadata.example.sprite_picking]
+name = "Sprite Picking"
+description = "Demonstrates picking sprites and sprite atlases"
+category = "Picking"
+wasm = true
+
 [profile.wasm-release]
 inherits = "release"
 opt-level = "z"

crates/bevy_internal/Cargo.toml

@@ -190,7 +190,11 @@ meshlet_processor = ["bevy_pbr?/meshlet_processor"]
 bevy_dev_tools = ["dep:bevy_dev_tools"]
 
 # Provides a picking functionality
-bevy_picking = ["dep:bevy_picking", "bevy_ui?/bevy_picking"]
+bevy_picking = [
+ "dep:bevy_picking",
+ "bevy_ui?/bevy_picking",
+ "bevy_sprite?/bevy_picking",
+]
 
 # Enable support for the ios_simulator by downgrading some rendering capabilities
 ios_simulator = ["bevy_pbr?/ios_simulator", "bevy_render?/ios_simulator"]

crates/bevy_picking/src/backend.rs

@@ -50,6 +50,10 @@ pub mod prelude {
 /// Some backends may only support providing the topmost entity; this is a valid limitation of some
 /// backends. For example, a picking shader might only have data on the topmost rendered output from
 /// its buffer.
+///
+/// Note that systems reading these events in [`PreUpdate`](bevy_app) will not report ordering
+/// ambiguities with picking backends. Take care to ensure such systems are explicitly ordered
+/// against [`PickSet::Backends`](crate), or better, avoid reading `PointerHits` in `PreUpdate`.
 #[derive(Event, Debug, Clone)]
 pub struct PointerHits {
  /// The pointer associated with this hit test.

crates/bevy_picking/src/lib.rs

@@ -207,6 +207,10 @@ impl Plugin for PickingPlugin {
  .add_event::<pointer::InputPress>()
  .add_event::<pointer::InputMove>()
  .add_event::<backend::PointerHits>()
+ // Rather than try to mark all current and future backends as ambiguous with each other,
+ // we allow them to send their hits in any order. These are later sorted, so submission
+ // order doesn't matter. See `PointerHits` docs for caveats.
+ .allow_ambiguous_resource::<Events<backend::PointerHits>>()
  .add_systems(
  PreUpdate,
  (

crates/bevy_sprite/Cargo.toml

@@ -9,6 +9,7 @@ license = "MIT OR Apache-2.0"
 keywords = ["bevy"]
 
 [features]
+bevy_picking = ["dep:bevy_picking", "dep:bevy_window"]
 webgl = []
 webgpu = []
 
@@ -20,12 +21,14 @@ bevy_color = { path = "../bevy_color", version = "0.15.0-dev" }
 bevy_core_pipeline = { path = "../bevy_core_pipeline", version = "0.15.0-dev" }
 bevy_ecs = { path = "../bevy_ecs", version = "0.15.0-dev" }
 bevy_math = { path = "../bevy_math", version = "0.15.0-dev" }
+bevy_picking = { path = "../bevy_picking", version = "0.15.0-dev", optional = true }
 bevy_reflect = { path = "../bevy_reflect", version = "0.15.0-dev", features = [
  "bevy",
 ] }
 bevy_render = { path = "../bevy_render", version = "0.15.0-dev" }
 bevy_transform = { path = "../bevy_transform", version = "0.15.0-dev" }
 bevy_utils = { path = "../bevy_utils", version = "0.15.0-dev" }
+bevy_window = { path = "../bevy_window", version = "0.15.0-dev", optional = true }
 bevy_derive = { path = "../bevy_derive", version = "0.15.0-dev" }
 
 # other

crates/bevy_sprite/src/lib.rs

@@ -11,6 +11,8 @@
 mod bundle;
 mod dynamic_texture_atlas_builder;
 mod mesh2d;
+#[cfg(feature = "bevy_picking")]
+mod picking_backend;
 mod render;
 mod sprite;
 mod texture_atlas;
@@ -133,6 +135,9 @@ impl Plugin for SpritePlugin {
  ),
  );
 
+ #[cfg(feature = "bevy_picking")]
+ app.add_plugins(picking_backend::SpritePickingBackend);
+
  if let Some(render_app) = app.get_sub_app_mut(RenderApp) {
  render_app
  .init_resource::<ImageBindGroups>()

crates/bevy_sprite/src/picking_backend.rs

@@ -0,0 +1,171 @@
+//! A [`bevy_picking`] backend for sprites. Works for simple sprites and sprite atlases. Works for
+//! sprites with arbitrary transforms. Picking is done based on sprite bounds, not visible pixels.
+//! This means a partially transparent sprite is pickable even in its transparent areas.
+
+use std::cmp::Ordering;
+
+use crate::{Sprite, TextureAtlas, TextureAtlasLayout};
+use bevy_app::prelude::*;
+use bevy_asset::prelude::*;
+use bevy_ecs::prelude::*;
+use bevy_math::{prelude::*, FloatExt};
+use bevy_picking::backend::prelude::*;
+use bevy_render::prelude::*;
+use bevy_transform::prelude::*;
+use bevy_window::PrimaryWindow;
+
+#[derive(Clone)]
+pub struct SpritePickingBackend;
+
+impl Plugin for SpritePickingBackend {
+ fn build(&self, app: &mut App) {
+ app.add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend));
+ }
+}
+
+pub fn sprite_picking(
+ pointers: Query<(&PointerId, &PointerLocation)>,
+ cameras: Query<(Entity, &Camera, &GlobalTransform, &OrthographicProjection)>,
+ primary_window: Query<Entity, With<PrimaryWindow>>,
+ images: Res<Assets<Image>>,
+ texture_atlas_layout: Res<Assets<TextureAtlasLayout>>,
+ sprite_query: Query<
+ (
+ Entity,
+ Option<&Sprite>,
+ Option<&TextureAtlas>,
+ Option<&Handle<Image>>,
+ &GlobalTransform,
+ Option<&Pickable>,
+ &ViewVisibility,
+ ),
+ Or<(With<Sprite>, With<TextureAtlas>)>,
+ >,
+ mut output: EventWriter<PointerHits>,
+) {
+ let mut sorted_sprites: Vec<_> = sprite_query.iter().collect();
+ sorted_sprites.sort_by(|a, b| {
+ (b.4.translation().z)
+ .partial_cmp(&a.4.translation().z)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ let primary_window = primary_window.get_single().ok();
+
+ for (pointer, location) in pointers.iter().filter_map(|(pointer, pointer_location)| {
+ pointer_location.location().map(|loc| (pointer, loc))
+ }) {
+ let mut blocked = false;
+ let Some((cam_entity, camera, cam_transform, cam_ortho)) = cameras
+ .iter()
+ .filter(|(_, camera, _, _)| camera.is_active)
+ .find(|(_, camera, _, _)| {
+ camera
+ .target
+ .normalize(primary_window)
+ .map(|x| x == location.target)
+ .unwrap_or(false)
+ })
+ else {
+ continue;
+ };
+
+ let Some(cursor_ray_world) = camera.viewport_to_world(cam_transform, location.position)
+ else {
+ continue;
+ };
+ let cursor_ray_len = cam_ortho.far - cam_ortho.near;
+ let cursor_ray_end = cursor_ray_world.origin + cursor_ray_world.direction * cursor_ray_len;
+
+ let picks: Vec<(Entity, HitData)> = sorted_sprites
+ .iter()
+ .copied()
+ .filter(|(.., visibility)| visibility.get())
+ .filter_map(
+ |(entity, sprite, atlas, image, sprite_transform, pickable, ..)| {
+ if blocked {
+ return None;
+ }
+
+ // Hit box in sprite coordinate system
+ let (extents, anchor) = if let Some((sprite, atlas)) = sprite.zip(atlas) {
+ let extents = sprite.custom_size.or_else(|| {
+ texture_atlas_layout
+ .get(&atlas.layout)
+ .map(|f| f.textures[atlas.index].size().as_vec2())
+ })?;
+ let anchor = sprite.anchor.as_vec();
+ (extents, anchor)
+ } else if let Some((sprite, image)) = sprite.zip(image) {
+ let extents = sprite
+ .custom_size
+ .or_else(|| images.get(image).map(|f| f.size().as_vec2()))?;
+ let anchor = sprite.anchor.as_vec();
+ (extents, anchor)
+ } else {
+ return None;
+ };
+
+ let center = -anchor * extents;
+ let rect = Rect::from_center_half_size(center, extents / 2.0);
+
+ // Transform cursor line segment to sprite coordinate system
+ let world_to_sprite = sprite_transform.affine().inverse();
+ let cursor_start_sprite =
+ world_to_sprite.transform_point3(cursor_ray_world.origin);
+ let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);
+
+ // Find where the cursor segment intersects the plane Z=0 (which is the sprite's
+ // plane in sprite-local space). It may not intersect if, for example, we're
+ // viewing the sprite side-on
+ if cursor_start_sprite.z == cursor_end_sprite.z {
+ // Cursor ray is parallel to the sprite and misses it
+ return None;
+ }
+ let lerp_factor =
+ f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
+ if !(0.0..=1.0).contains(&lerp_factor) {
+ // Lerp factor is out of range, meaning that while an infinite line cast by
+ // the cursor would intersect the sprite, the sprite is not between the
+ // camera's near and far planes
+ return None;
+ }
+ // Otherwise we can interpolate the xy of the start and end positions by the
+ // lerp factor to get the cursor position in sprite space!
+ let cursor_pos_sprite = cursor_start_sprite
+ .lerp(cursor_end_sprite, lerp_factor)
+ .xy();
+
+ let is_cursor_in_sprite = rect.contains(cursor_pos_sprite);
+
+ blocked = is_cursor_in_sprite
+ && pickable.map(|p| p.should_block_lower) != Some(false);
+
+ is_cursor_in_sprite.then(|| {
+ let hit_pos_world =
+ sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
+ // Transform point from world to camera space to get the Z distance
+ let hit_pos_cam = cam_transform
+ .affine()
+ .inverse()
+ .transform_point3(hit_pos_world);
+ // HitData requires a depth as calculated from the camera's near clipping plane
+ let depth = -cam_ortho.near - hit_pos_cam.z;
+ (
+ entity,
+ HitData::new(
+ cam_entity,
+ depth,
+ Some(hit_pos_world),
+ Some(*sprite_transform.back()),
+ ),
+ )
+ })
+ },
+ )
+ .collect();
+
+ let order = camera.order as f32;
+ output.send(PointerHits::new(*pointer, picks, order));
+ }
+}

examples/README.md

@@ -358,6 +358,7 @@ Example | Description
 Example | Description
 --- | ---
 [Showcases simple picking events and usage](../examples/picking/simple_picking.rs) | Demonstrates how to use picking events to spawn simple objects
+[Sprite Picking](../examples/picking/sprite_picking.rs) | Demonstrates picking sprites and sprite atlases
 
 ## Reflection