From 57c63da9c466f3c782bc9a55c747788d9bcc71fa Mon Sep 17 00:00:00 2001 From: Torstein Grindvik Date: Tue, 20 Dec 2022 16:55:02 +0100 Subject: [PATCH 01/22] Picking proof of concept Signed-off-by: Torstein Grindvik --- Cargo.toml | 11 + .../src/core_2d/main_pass_2d_node.rs | 30 +- crates/bevy_core_pipeline/src/core_2d/mod.rs | 13 +- .../src/core_3d/main_pass_3d_node.rs | 51 ++- crates/bevy_core_pipeline/src/core_3d/mod.rs | 13 +- crates/bevy_core_pipeline/src/lib.rs | 5 +- crates/bevy_core_pipeline/src/picking/mod.rs | 26 ++ crates/bevy_core_pipeline/src/picking/node.rs | 64 +++ crates/bevy_pbr/src/render/mesh.rs | 19 +- crates/bevy_pbr/src/render/mesh_types.wgsl | 1 + crates/bevy_pbr/src/render/pbr.wgsl | 17 +- crates/bevy_render/Cargo.toml | 16 +- crates/bevy_render/src/lib.rs | 1 + crates/bevy_render/src/picking/mod.rs | 408 ++++++++++++++++++ crates/bevy_render/src/render_resource/mod.rs | 20 +- crates/bevy_render/src/view/mod.rs | 2 + .../src/mesh2d/color_material.wgsl | 15 +- crates/bevy_sprite/src/mesh2d/mesh.rs | 19 +- .../bevy_sprite/src/mesh2d/mesh2d_types.wgsl | 1 + crates/bevy_sprite/src/render/mod.rs | 23 +- crates/bevy_sprite/src/render/sprite.wgsl | 21 +- crates/bevy_ui/src/render/mod.rs | 29 +- crates/bevy_ui/src/render/pipeline.rs | 27 +- crates/bevy_ui/src/render/render_pass.rs | 18 +- crates/bevy_ui/src/render/ui.wgsl | 22 +- examples/app/picking.rs | 344 +++++++++++++++ 26 files changed, 1123 insertions(+), 93 deletions(-) create mode 100644 crates/bevy_core_pipeline/src/picking/mod.rs create mode 100644 crates/bevy_core_pipeline/src/picking/node.rs create mode 100644 crates/bevy_render/src/picking/mod.rs create mode 100644 examples/app/picking.rs diff --git a/Cargo.toml b/Cargo.toml index 48b7e55bb5d88..aa2e2f7e8d08a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -670,6 +670,17 @@ description = "An application that runs with default plugins and displays an emp category = "Application" wasm = false +[[example]] +name = "picking" +path = "examples/app/picking.rs" + +[package.metadata.example.picking] +name = "Picking" +description = "An application that show how to pick/select objects and UI elements" +category = "Application" +# TODO, are we wasm compatible? +wasm = false + [[example]] name = "without_winit" path = "examples/app/without_winit.rs" diff --git a/crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs b/crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs index da3cbf3c17242..7d1e07861b7e2 100644 --- a/crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs +++ b/crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs @@ -5,6 +5,7 @@ use crate::{ use bevy_ecs::prelude::*; use bevy_render::{ camera::ExtractedCamera, + picking::PickingTextures, render_graph::{Node, NodeRunError, RenderGraphContext, SlotInfo, SlotType}, render_phase::RenderPhase, render_resource::{LoadOp, Operations, RenderPassDescriptor}, @@ -21,6 +22,7 @@ pub struct MainPass2dNode { &'static RenderPhase, &'static ViewTarget, &'static Camera2d, + &'static PickingTextures, ), With, >, @@ -52,7 +54,7 @@ impl Node for MainPass2dNode { world: &World, ) -> Result<(), NodeRunError> { let view_entity = graph.get_input_entity(Self::IN_VIEW)?; - let (camera, transparent_phase, target, camera_2d) = + let (camera, transparent_phase, target, camera_2d, picking_textures) = if let Ok(result) = self.query.get_manual(world, view_entity) { result } else { @@ -65,16 +67,22 @@ impl Node for MainPass2dNode { let mut render_pass = render_context.begin_tracked_render_pass(RenderPassDescriptor { label: Some("main_pass_2d"), - color_attachments: &[Some(target.get_color_attachment(Operations { - load: match camera_2d.clear_color { - ClearColorConfig::Default => { - LoadOp::Clear(world.resource::().0.into()) - } - ClearColorConfig::Custom(color) => LoadOp::Clear(color.into()), - ClearColorConfig::None => LoadOp::Load, - }, - store: true, - }))], + color_attachments: &[ + Some(target.get_color_attachment(Operations { + load: match camera_2d.clear_color { + ClearColorConfig::Default => { + LoadOp::Clear(world.resource::().0.into()) + } + ClearColorConfig::Custom(color) => LoadOp::Clear(color.into()), + ClearColorConfig::None => LoadOp::Load, + }, + store: true, + })), + Some(picking_textures.get_color_attachment(Operations { + load: LoadOp::Clear(PickingTextures::clear_color()), + store: true, + })), + ], depth_stencil_attachment: None, }); diff --git a/crates/bevy_core_pipeline/src/core_2d/mod.rs b/crates/bevy_core_pipeline/src/core_2d/mod.rs index f391a49496a04..d63f8c648830b 100644 --- a/crates/bevy_core_pipeline/src/core_2d/mod.rs +++ b/crates/bevy_core_pipeline/src/core_2d/mod.rs @@ -11,6 +11,7 @@ pub mod graph { pub const BLOOM: &str = "bloom"; pub const TONEMAPPING: &str = "tonemapping"; pub const FXAA: &str = "fxaa"; + pub const PICKING: &str = "picking"; pub const UPSCALING: &str = "upscaling"; pub const END_MAIN_PASS_POST_PROCESSING: &str = "end_main_pass_post_processing"; } @@ -35,7 +36,7 @@ use bevy_render::{ use bevy_utils::FloatOrd; use std::ops::Range; -use crate::{tonemapping::TonemappingNode, upscaling::UpscalingNode}; +use crate::{picking::node::PickingNode, tonemapping::TonemappingNode, upscaling::UpscalingNode}; pub struct Core2dPlugin; @@ -61,6 +62,8 @@ impl Plugin for Core2dPlugin { let pass_node_2d = MainPass2dNode::new(&mut render_app.world); let tonemapping = TonemappingNode::new(&mut render_app.world); let upscaling = UpscalingNode::new(&mut render_app.world); + let picking = PickingNode::new(&mut render_app.world); + let mut graph = render_app.world.resource_mut::(); let mut draw_2d_graph = RenderGraph::default(); @@ -68,6 +71,7 @@ impl Plugin for Core2dPlugin { draw_2d_graph.add_node(graph::node::TONEMAPPING, tonemapping); draw_2d_graph.add_node(graph::node::END_MAIN_PASS_POST_PROCESSING, EmptyNode); draw_2d_graph.add_node(graph::node::UPSCALING, upscaling); + draw_2d_graph.add_node(graph::node::PICKING, picking); let input_node_id = draw_2d_graph.set_input(vec![SlotInfo::new( graph::input::VIEW_ENTITY, SlotType::Entity, @@ -90,6 +94,12 @@ impl Plugin for Core2dPlugin { graph::node::UPSCALING, UpscalingNode::IN_VIEW, ); + draw_2d_graph.add_slot_edge( + input_node_id, + graph::input::VIEW_ENTITY, + graph::node::PICKING, + PickingNode::IN_VIEW, + ); draw_2d_graph.add_node_edge(graph::node::MAIN_PASS, graph::node::TONEMAPPING); draw_2d_graph.add_node_edge( graph::node::TONEMAPPING, @@ -99,6 +109,7 @@ impl Plugin for Core2dPlugin { graph::node::END_MAIN_PASS_POST_PROCESSING, graph::node::UPSCALING, ); + draw_2d_graph.add_node_edge(graph::node::UPSCALING, graph::node::PICKING); graph.add_sub_graph(graph::NAME, draw_2d_graph); } } diff --git a/crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs b/crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs index a836e0bcbe450..b431cb05162fe 100644 --- a/crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs +++ b/crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs @@ -5,6 +5,7 @@ use crate::{ use bevy_ecs::prelude::*; use bevy_render::{ camera::ExtractedCamera, + picking::PickingTextures, render_graph::{Node, NodeRunError, RenderGraphContext, SlotInfo, SlotType}, render_phase::RenderPhase, render_resource::{LoadOp, Operations, RenderPassDepthStencilAttachment, RenderPassDescriptor}, @@ -24,6 +25,7 @@ pub struct MainPass3dNode { &'static Camera3d, &'static ViewTarget, &'static ViewDepthTexture, + &'static PickingTextures, ), With, >, @@ -55,13 +57,21 @@ impl Node for MainPass3dNode { world: &World, ) -> Result<(), NodeRunError> { let view_entity = graph.get_input_entity(Self::IN_VIEW)?; - let (camera, opaque_phase, alpha_mask_phase, transparent_phase, camera_3d, target, depth) = - match self.query.get_manual(world, view_entity) { - Ok(query) => query, - Err(_) => { - return Ok(()); - } // No window - }; + let ( + camera, + opaque_phase, + alpha_mask_phase, + transparent_phase, + camera_3d, + target, + depth, + picking_textures, + ) = match self.query.get_manual(world, view_entity) { + Ok(query) => query, + Err(_) => { + return Ok(()); + } // No window + }; // Always run opaque pass to ensure screen is cleared { @@ -74,16 +84,23 @@ impl Node for MainPass3dNode { label: Some("main_opaque_pass_3d"), // NOTE: The opaque pass loads the color // buffer as well as writing to it. - color_attachments: &[Some(target.get_color_attachment(Operations { - load: match camera_3d.clear_color { - ClearColorConfig::Default => { - LoadOp::Clear(world.resource::().0.into()) - } - ClearColorConfig::Custom(color) => LoadOp::Clear(color.into()), - ClearColorConfig::None => LoadOp::Load, - }, - store: true, - }))], + color_attachments: &[ + Some(target.get_color_attachment(Operations { + load: match camera_3d.clear_color { + ClearColorConfig::Default => { + LoadOp::Clear(world.resource::().0.into()) + } + ClearColorConfig::Custom(color) => LoadOp::Clear(color.into()), + ClearColorConfig::None => LoadOp::Load, + }, + store: true, + })), + // TODO: Should be based on if picking or not + Some(picking_textures.get_color_attachment(Operations { + load: LoadOp::Clear(PickingTextures::clear_color()), + store: true, + })), + ], depth_stencil_attachment: Some(RenderPassDepthStencilAttachment { view: &depth.view, // NOTE: The opaque main pass loads the depth buffer and possibly overwrites it diff --git a/crates/bevy_core_pipeline/src/core_3d/mod.rs b/crates/bevy_core_pipeline/src/core_3d/mod.rs index 34a430a345398..27e4ad3779dcb 100644 --- a/crates/bevy_core_pipeline/src/core_3d/mod.rs +++ b/crates/bevy_core_pipeline/src/core_3d/mod.rs @@ -11,6 +11,7 @@ pub mod graph { pub const BLOOM: &str = "bloom"; pub const TONEMAPPING: &str = "tonemapping"; pub const FXAA: &str = "fxaa"; + pub const PICKING: &str = "picking"; pub const UPSCALING: &str = "upscaling"; pub const END_MAIN_PASS_POST_PROCESSING: &str = "end_main_pass_post_processing"; } @@ -43,7 +44,7 @@ use bevy_render::{ }; use bevy_utils::{FloatOrd, HashMap}; -use crate::{tonemapping::TonemappingNode, upscaling::UpscalingNode}; +use crate::{picking::node::PickingNode, tonemapping::TonemappingNode, upscaling::UpscalingNode}; pub struct Core3dPlugin; @@ -71,6 +72,8 @@ impl Plugin for Core3dPlugin { let pass_node_3d = MainPass3dNode::new(&mut render_app.world); let tonemapping = TonemappingNode::new(&mut render_app.world); let upscaling = UpscalingNode::new(&mut render_app.world); + let picking = PickingNode::new(&mut render_app.world); + let mut graph = render_app.world.resource_mut::(); let mut draw_3d_graph = RenderGraph::default(); @@ -78,6 +81,7 @@ impl Plugin for Core3dPlugin { draw_3d_graph.add_node(graph::node::TONEMAPPING, tonemapping); draw_3d_graph.add_node(graph::node::END_MAIN_PASS_POST_PROCESSING, EmptyNode); draw_3d_graph.add_node(graph::node::UPSCALING, upscaling); + draw_3d_graph.add_node(graph::node::PICKING, picking); let input_node_id = draw_3d_graph.set_input(vec![SlotInfo::new( graph::input::VIEW_ENTITY, SlotType::Entity, @@ -100,6 +104,12 @@ impl Plugin for Core3dPlugin { graph::node::UPSCALING, UpscalingNode::IN_VIEW, ); + draw_3d_graph.add_slot_edge( + input_node_id, + graph::input::VIEW_ENTITY, + graph::node::PICKING, + PickingNode::IN_VIEW, + ); draw_3d_graph.add_node_edge(graph::node::MAIN_PASS, graph::node::TONEMAPPING); draw_3d_graph.add_node_edge( graph::node::TONEMAPPING, @@ -109,6 +119,7 @@ impl Plugin for Core3dPlugin { graph::node::END_MAIN_PASS_POST_PROCESSING, graph::node::UPSCALING, ); + draw_3d_graph.add_node_edge(graph::node::UPSCALING, graph::node::PICKING); graph.add_sub_graph(graph::NAME, draw_3d_graph); } } diff --git a/crates/bevy_core_pipeline/src/lib.rs b/crates/bevy_core_pipeline/src/lib.rs index adfe9d500f038..bdea2b3fb84f9 100644 --- a/crates/bevy_core_pipeline/src/lib.rs +++ b/crates/bevy_core_pipeline/src/lib.rs @@ -4,6 +4,7 @@ pub mod core_2d; pub mod core_3d; pub mod fullscreen_vertex_shader; pub mod fxaa; +pub mod picking; pub mod tonemapping; pub mod upscaling; @@ -23,6 +24,7 @@ use crate::{ core_3d::Core3dPlugin, fullscreen_vertex_shader::FULLSCREEN_SHADER_HANDLE, fxaa::FxaaPlugin, + picking::PickingPlugin, tonemapping::TonemappingPlugin, upscaling::UpscalingPlugin, }; @@ -51,6 +53,7 @@ impl Plugin for CorePipelinePlugin { .add_plugin(TonemappingPlugin) .add_plugin(UpscalingPlugin) .add_plugin(BloomPlugin) - .add_plugin(FxaaPlugin); + .add_plugin(FxaaPlugin) + .add_plugin(PickingPlugin); } } diff --git a/crates/bevy_core_pipeline/src/picking/mod.rs b/crates/bevy_core_pipeline/src/picking/mod.rs new file mode 100644 index 0000000000000..66a93865b96d4 --- /dev/null +++ b/crates/bevy_core_pipeline/src/picking/mod.rs @@ -0,0 +1,26 @@ +use bevy_app::{CoreStage, Plugin}; +use bevy_render::{ + extract_component::ExtractComponentPlugin, + picking::{self, PickedEvent, Picking}, + RenderApp, RenderStage, +}; + +pub mod node; + +/// Uses the GPU to provide a buffer which allows lookup of entities at a given coordinate. +#[derive(Default)] +pub struct PickingPlugin; + +impl Plugin for PickingPlugin { + fn build(&self, app: &mut bevy_app::App) { + // TODO: Also register type? + app.add_event::() + .add_plugin(ExtractComponentPlugin::::default()) + // In PreUpdate such that written events are ensure not to have a frame delay + // for default user systems + .add_system_to_stage(CoreStage::PreUpdate, picking::picking_events); + + let render_app = app.get_sub_app_mut(RenderApp).unwrap(); + render_app.add_system_to_stage(RenderStage::Prepare, picking::prepare_picking_targets); + } +} diff --git a/crates/bevy_core_pipeline/src/picking/node.rs b/crates/bevy_core_pipeline/src/picking/node.rs new file mode 100644 index 0000000000000..6e97f357cbcb8 --- /dev/null +++ b/crates/bevy_core_pipeline/src/picking/node.rs @@ -0,0 +1,64 @@ +use bevy_ecs::{query::QueryState, world::World}; + +use bevy_render::{ + camera::ExtractedCamera, + picking::{copy_to_buffer, Picking, PickingTextures}, + render_graph::{Node, NodeRunError, RenderGraphContext, SlotInfo, SlotType}, + renderer::RenderContext, +}; +#[cfg(feature = "trace")] +use bevy_utils::tracing::info_span; + +pub struct PickingNode { + query: QueryState<( + &'static ExtractedCamera, + &'static Picking, + &'static PickingTextures, + )>, +} + +impl PickingNode { + pub const IN_VIEW: &'static str = "view"; + + pub fn new(world: &mut World) -> Self { + Self { + query: world.query(), + } + } +} + +impl Node for PickingNode { + fn input(&self) -> Vec { + vec![SlotInfo::new(PickingNode::IN_VIEW, SlotType::Entity)] + } + + fn update(&mut self, world: &mut World) { + self.query.update_archetypes(world); + } + + fn run( + &self, + graph: &mut RenderGraphContext, + render_context: &mut RenderContext, + world: &World, + ) -> Result<(), NodeRunError> { + let view_entity = graph.get_input_entity(Self::IN_VIEW)?; + let (camera, picking, picking_textures) = + if let Ok(result) = self.query.get_manual(world, view_entity) { + result + } else { + // no target + return Ok(()); + }; + { + #[cfg(feature = "trace")] + let _picking_pass = info_span!("picking_pass").entered(); + + if let Some(camera_size) = camera.physical_target_size { + copy_to_buffer(camera_size, picking, picking_textures, render_context); + } + } + + Ok(()) + } +} diff --git a/crates/bevy_pbr/src/render/mesh.rs b/crates/bevy_pbr/src/render/mesh.rs index 92b65621922ac..e8fa425c73fe3 100644 --- a/crates/bevy_pbr/src/render/mesh.rs +++ b/crates/bevy_pbr/src/render/mesh.rs @@ -111,6 +111,7 @@ impl Plugin for MeshRenderPlugin { pub struct MeshUniform { pub transform: Mat4, pub inverse_transpose_model: Mat4, + pub entity_index: u32, pub flags: u32, } @@ -160,6 +161,7 @@ pub fn extract_meshes( flags: flags.bits, transform, inverse_transpose_model: transform.inverse().transpose(), + entity_index: entity.index(), }; if not_caster.is_some() { not_caster_commands.push((entity, (handle.clone_weak(), uniform, NotShadowCaster))); @@ -662,11 +664,18 @@ impl SpecializedMeshPipeline for MeshPipeline { shader: MESH_SHADER_HANDLE.typed::(), shader_defs, entry_point: "fragment".into(), - targets: vec![Some(ColorTargetState { - format, - blend, - write_mask: ColorWrites::ALL, - })], + targets: vec![ + Some(ColorTargetState { + format, + blend, + write_mask: ColorWrites::ALL, + }), + Some(ColorTargetState { + format: TextureFormat::R32Uint, + blend: None, + write_mask: ColorWrites::ALL, + }), + ], }), layout: Some(bind_group_layout), primitive: PrimitiveState { diff --git a/crates/bevy_pbr/src/render/mesh_types.wgsl b/crates/bevy_pbr/src/render/mesh_types.wgsl index d44adbc2bb13f..5a6cf288ccf8c 100644 --- a/crates/bevy_pbr/src/render/mesh_types.wgsl +++ b/crates/bevy_pbr/src/render/mesh_types.wgsl @@ -3,6 +3,7 @@ struct Mesh { model: mat4x4, inverse_transpose_model: mat4x4, + entity_index: u32, // 'flags' is a bit field indicating various options. u32 is 32 bits so we have up to 32 options. flags: u32, }; diff --git a/crates/bevy_pbr/src/render/pbr.wgsl b/crates/bevy_pbr/src/render/pbr.wgsl index cfe825190d2a5..6f25353c568d2 100644 --- a/crates/bevy_pbr/src/render/pbr.wgsl +++ b/crates/bevy_pbr/src/render/pbr.wgsl @@ -14,8 +14,13 @@ struct FragmentInput { #import bevy_pbr::mesh_vertex_output }; +struct FragmentOutput { + @location(0) color: vec4, + @location(1) picking: u32, + } + @fragment -fn fragment(in: FragmentInput) -> @location(0) vec4 { +fn fragment(in: FragmentInput) -> FragmentOutput { var output_color: vec4 = material.base_color; #ifdef VERTEX_COLORS output_color = output_color * in.color; @@ -107,5 +112,13 @@ fn fragment(in: FragmentInput) -> @location(0) vec4 { output_rgb = pow(output_rgb, vec3(2.2)); output_color = vec4(output_rgb, output_color.a); #endif - return output_color; + + let location = vec2(i32(in.frag_coord.x), i32(in.frag_coord.y)); + + var out: FragmentOutput; + + out.color = output_color; + out.picking = mesh.entity_index; + + return out; } diff --git a/crates/bevy_render/Cargo.toml b/crates/bevy_render/Cargo.toml index eaa4d09af6bac..6b38b960a68e1 100644 --- a/crates/bevy_render/Cargo.toml +++ b/crates/bevy_render/Cargo.toml @@ -38,7 +38,9 @@ bevy_hierarchy = { path = "../bevy_hierarchy", version = "0.9.0" } bevy_log = { path = "../bevy_log", version = "0.9.0" } bevy_math = { path = "../bevy_math", version = "0.9.0" } bevy_mikktspace = { path = "../bevy_mikktspace", version = "0.9.0" } -bevy_reflect = { path = "../bevy_reflect", version = "0.9.0", features = ["bevy"] } +bevy_reflect = { path = "../bevy_reflect", version = "0.9.0", features = [ + "bevy", +] } bevy_render_macros = { path = "macros", version = "0.9.0" } bevy_time = { path = "../bevy_time", version = "0.9.0" } bevy_transform = { path = "../bevy_transform", version = "0.9.0" } @@ -52,7 +54,13 @@ image = { version = "0.24", default-features = false } wgpu = { version = "0.14.0", features = ["spirv"] } wgpu-hal = "0.14.1" codespan-reporting = "0.11.0" -naga = { version = "0.10.0", features = ["glsl-in", "spv-in", "spv-out", "wgsl-in", "wgsl-out"] } +naga = { version = "0.10.0", features = [ + "glsl-in", + "spv-in", + "spv-out", + "wgsl-in", + "wgsl-out", +] } serde = { version = "1", features = ["derive"] } bitflags = "1.2.1" smallvec = { version = "1.6", features = ["union", "const_generics"] } @@ -75,4 +83,6 @@ ruzstd = { version = "0.2.4", optional = true } basis-universal = { version = "0.2.0", optional = true } encase = { version = "0.4", features = ["glam"] } # For wgpu profiling using tracing. Use `RUST_LOG=info` to also capture the wgpu spans. -profiling = { version = "1", features = ["profile-with-tracing"], optional = true } +profiling = { version = "1", features = [ + "profile-with-tracing", +], optional = true } diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 5456c7208accc..c36ee45f19e90 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -10,6 +10,7 @@ mod extract_param; pub mod extract_resource; pub mod globals; pub mod mesh; +pub mod picking; pub mod primitives; pub mod render_asset; pub mod render_graph; diff --git a/crates/bevy_render/src/picking/mod.rs b/crates/bevy_render/src/picking/mod.rs new file mode 100644 index 0000000000000..afa3aafd7096d --- /dev/null +++ b/crates/bevy_render/src/picking/mod.rs @@ -0,0 +1,408 @@ +use std::{ + mem::size_of, + sync::{Arc, Mutex}, +}; + +use bevy_ecs::{prelude::*, query::QueryItem}; +use bevy_math::{UVec2, Vec2}; +use bevy_utils::HashMap; +use bevy_window::CursorMoved; +use wgpu::{ + BufferDescriptor, BufferUsages, BufferView, Extent3d, ImageCopyBuffer, ImageDataLayout, + Maintain, MapMode, Operations, RenderPassColorAttachment, TextureDescriptor, TextureDimension, + TextureFormat, TextureUsages, +}; + +use crate::{ + camera::{Camera, ExtractedCamera}, + extract_component::ExtractComponent, + prelude::Color, + render_resource::Buffer, + renderer::{RenderContext, RenderDevice}, + texture::{CachedTexture, TextureCache}, + view::Msaa, +}; + +pub fn copy_to_buffer( + camera_size: UVec2, + picking: &Picking, + picking_textures: &PickingTextures, + render_context: &mut RenderContext, +) { + let picking_buffer_size = PickingBufferSize::from(camera_size); + + let (buffer, size) = picking + .buffer + .try_lock() + .expect("TODO: Can we lock here?") + .as_ref() + .expect("Buffer should have been prepared") + .clone(); + + render_context.command_encoder.copy_texture_to_buffer( + picking_textures.main.texture.as_image_copy(), + ImageCopyBuffer { + buffer: &buffer, + layout: ImageDataLayout { + offset: 0, + bytes_per_row: Some( + std::num::NonZeroU32::new(size.padded_bytes_per_row as u32).unwrap(), + ), + rows_per_image: None, + }, + }, + Extent3d { + width: picking_buffer_size.texture_size.x, + height: picking_buffer_size.texture_size.y, + depth_or_array_layers: 1, + }, + ); +} + +/// Add this to a camera in order for the camera +/// to generate [`PickedEntityIndex`] events when the cursor hovers over entities. +#[derive(Component, Debug, Clone, Default)] +pub struct Picking { + pub buffer: Arc>>, +} + +#[derive(Debug, Clone, Default)] +pub struct PickingBufferSize { + pub texture_size: UVec2, + pub padded_bytes_per_row: usize, +} + +impl PickingBufferSize { + pub fn new(width: u32, height: u32) -> Self { + // See: https://github.com/gfx-rs/wgpu/blob/master/wgpu/examples/capture/main.rs#L193 + let bytes_per_pixel = size_of::(); + let unpadded_bytes_per_row = width as usize * bytes_per_pixel; + let align = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT as usize; + let padded_bytes_per_row_padding = (align - (unpadded_bytes_per_row % align)) % align; + let padded_bytes_per_row = unpadded_bytes_per_row + padded_bytes_per_row_padding; + + Self { + texture_size: UVec2 { + x: width, + y: height, + }, + padded_bytes_per_row, + } + } + + pub fn total_needed_bytes(&self) -> u64 { + (self.texture_size.y as usize * self.padded_bytes_per_row) as u64 + } +} + +impl From for PickingBufferSize { + fn from(texture_extent: Extent3d) -> Self { + Self::new(texture_extent.width, texture_extent.height) + } +} + +impl From for PickingBufferSize { + fn from(texture_extent: UVec2) -> Self { + Self::new(texture_extent.x, texture_extent.y) + } +} + +impl ExtractComponent for Picking { + type Query = &'static Self; + type Filter = With; + type Out = Self; + + fn extract_component(item: QueryItem<'_, Self::Query>) -> Option { + Some(item.clone()) + } +} + +#[derive(Component, Clone)] +pub struct PickingTextures { + pub main: CachedTexture, + pub sampled: Option, +} + +impl PickingTextures { + // Same logic as [`ViewTarget`]. + + /// The clear color which should be used to clear picking textures. + /// Picking textures use a single u32 value for each pixel. + /// This color clears that with u32::MAX. + /// This allows all entity index values below u32::MAX to be valid. + pub fn clear_color() -> wgpu::Color { + Color::Rgba { + red: f32::MAX, + green: f32::MAX, + blue: f32::MAX, + alpha: f32::MAX, + } + .into() + } + + /// Retrieve this target's color attachment. This will use [`Self::sampled_main_texture`] and resolve to [`Self::main_texture`] if + /// the target has sampling enabled. Otherwise it will use [`Self::main_texture`] directly. + pub fn get_color_attachment(&self, ops: Operations) -> RenderPassColorAttachment { + match &self.sampled { + Some(sampled_texture) => RenderPassColorAttachment { + view: &sampled_texture.default_view, + resolve_target: Some(&self.main.default_view), + ops, + }, + None => RenderPassColorAttachment { + view: &self.main.default_view, + resolve_target: None, + ops, + }, + } + } + + pub fn get_unsampled_color_attachment( + &self, + ops: Operations, + ) -> RenderPassColorAttachment { + RenderPassColorAttachment { + view: &self.main.default_view, + resolve_target: None, + ops, + } + } +} + +#[derive(Debug, Resource, PartialEq, Eq, PartialOrd, Ord)] +pub enum PickedEventVariant { + /// The given entity is now picked/hovered. + // TODO: Perhaps it's useful to provide the coords as well? + Picked, + + /// The given entity is no longer picked/hovered. + Unpicked, +} + +#[derive(Debug, Resource, PartialEq, Eq, PartialOrd, Ord)] +pub struct PickedEvent { + /// Which entity triggered the event. + pub entity: Entity, + + /// Which event variant occurred. + pub event: PickedEventVariant, +} + +impl PickedEvent { + fn new(entity_index: u32, event: PickedEventVariant) -> Self { + Self { + entity: Entity::from_raw(entity_index), + event, + } + } + + fn new_picked(entity_index: u32) -> Self { + Self::new(entity_index, PickedEventVariant::Picked) + } + + fn new_unpicked(entity_index: u32) -> Self { + Self::new(entity_index, PickedEventVariant::Unpicked) + } +} + +fn cursor_coords_to_entity_index( + cursor: Vec2, + camera_size: UVec2, + picking_buffer_size: &PickingBufferSize, + buffer_view: &BufferView, +) -> u32 { + // The GPU image has a top-left origin, + // but the cursor has a bottom-left origin. + // Therefore we must flip the vertical axis. + let x = cursor.x as usize; + let y = camera_size.y as usize - cursor.y as usize; + + // We know the coordinates, but in order to find the true position of the 4 bytes + // we're interested in, we have to know how wide a single line in the GPU written buffer is. + // Due to alignment requirements this may be wider than the physical camera size because + // of padding. + let padded_width = picking_buffer_size.padded_bytes_per_row; + + let pixel_size = std::mem::size_of::(); + + let start = (y * padded_width) + (x * pixel_size); + let end = start + pixel_size; + + let bytes = &buffer_view[start..end]; + + u32::from_le_bytes(bytes.try_into().unwrap()) +} + +pub fn picking_events( + // TODO: Must be hashmap to have per-camera + // Maybe use the entity of the below query for that? + mut picked: Local>, + + query: Query<(&Picking, &Camera)>, + // TODO: Ensure we get events on the same frame as this guy + // UPDATE: These events are issued via winit, so presumably they arrive very early/first in the frame? + mut cursor_moved_events: EventReader, + mut events: EventWriter, + render_device: Res, +) { + #[cfg(feature = "trace")] + let _picking_span = info_span!("picking", name = "picking").entered(); + + for (picking, camera) in query.iter() { + let Some(camera_size) = camera.physical_target_size() else { continue }; + + if camera_size.x == 0 || camera_size.y == 0 { + continue; + } + + // TODO: Is it possible the GPU tries this at the same time as us? + let guard = picking.buffer.try_lock().unwrap(); + + let Some((buffer, picking_buffer_size)) = guard.as_ref() else { continue }; + + let buffer_slice = buffer.slice(..); + + buffer_slice.map_async(MapMode::Read, move |result| { + if let Err(e) = result { + panic!("{e}"); + } + }); + // For the above mapping to complete + render_device.poll(Maintain::Wait); + + let buffer_view = buffer_slice.get_mapped_range(); + + for event in cursor_moved_events.iter() { + let picked_index = cursor_coords_to_entity_index( + event.position, + camera_size, + picking_buffer_size, + &buffer_view, + ); + + match *picked { + Some(cached_index) if picked_index == u32::MAX => { + // No entity + events.send(PickedEvent::new_unpicked(cached_index)); + *picked = None; + } + Some(cached_index) if cached_index == picked_index => { + // Nothing to report, the same entity is being hovered/picked + } + Some(cached_index) => { + // The cursor moved straight between two entities + events.send(PickedEvent::new_unpicked(cached_index)); + + *picked = Some(picked_index); + events.send(PickedEvent::new_picked(picked_index)); + } + None if picked_index == u32::MAX => { + // Nothing to report, this index is reserved to mean "nothing picked" + } + None => { + *picked = Some(picked_index); + events.send(PickedEvent::new_picked(picked_index)); + } + } + } + + drop(buffer_view); + buffer.unmap(); + } +} + +pub fn prepare_picking_targets( + mut commands: Commands, + msaa: Res, + render_device: Res, + mut texture_cache: ResMut, + cameras: Query<(Entity, &ExtractedCamera, &Picking)>, +) { + #[cfg(feature = "trace")] + let _picking_span = info_span!("picking_prepare", name = "picking_prepare").entered(); + + let mut textures = HashMap::default(); + for (entity, camera, picking) in cameras.iter() { + if let Some(target_size) = camera.physical_target_size { + let size = Extent3d { + width: target_size.x, + height: target_size.y, + depth_or_array_layers: 1, + }; + let picking_buffer_dimensions = PickingBufferSize::from(size); + let needed_buffer_size = picking_buffer_dimensions.total_needed_bytes(); + + let mut buffer = picking + .buffer + .try_lock() + .expect("TODO: Are we ok to lock here?"); + + let make_buffer = || { + #[cfg(feature = "trace")] + bevy_utils::tracing::debug!("Creating new picking buffer"); + + render_device.create_buffer(&BufferDescriptor { + label: Some("Picking buffer"), + size: needed_buffer_size, + usage: BufferUsages::COPY_DST | BufferUsages::MAP_READ, + mapped_at_creation: false, + }) + }; + + // If either the buffer has never been created or + // the size of the current one has changed (e.g. due to window resize) + // we have to create a new one. + match buffer.as_mut() { + Some((buffer, contained_size)) => { + if buffer.size() != needed_buffer_size { + *buffer = make_buffer(); + *contained_size = size.into(); + } + } + None => *buffer = Some((make_buffer(), size.into())), + } + + // We want to store entity indices, which are u32s. + // We therefore only need a single u32 channel. + let picking_texture_format = TextureFormat::R32Uint; + + let picking_textures = textures.entry(camera.target.clone()).or_insert_with(|| { + let descriptor = TextureDescriptor { + label: None, + size, + mip_level_count: 1, + sample_count: 1, + dimension: TextureDimension::D2, + format: picking_texture_format, + usage: TextureUsages::RENDER_ATTACHMENT | TextureUsages::COPY_SRC, + }; + + PickingTextures { + main: texture_cache.get( + &render_device, + TextureDescriptor { + label: Some("main_picking_texture"), + ..descriptor + }, + ), + sampled: (msaa.samples > 1).then(|| { + texture_cache.get( + &render_device, + TextureDescriptor { + label: Some("main_picking_texture_sampled"), + size, + mip_level_count: 1, + sample_count: msaa.samples, + dimension: TextureDimension::D2, + format: picking_texture_format, + usage: TextureUsages::RENDER_ATTACHMENT, + }, + ) + }), + } + }); + + commands.entity(entity).insert(picking_textures.clone()); + } + } +} diff --git a/crates/bevy_render/src/render_resource/mod.rs b/crates/bevy_render/src/render_resource/mod.rs index 33a010ff502d1..feccbf9863a5a 100644 --- a/crates/bevy_render/src/render_resource/mod.rs +++ b/crates/bevy_render/src/render_resource/mod.rs @@ -33,16 +33,16 @@ pub use wgpu::{ ComputePipelineDescriptor as RawComputePipelineDescriptor, DepthBiasState, DepthStencilState, Extent3d, Face, Features as WgpuFeatures, FilterMode, FragmentState as RawFragmentState, FrontFace, ImageCopyBuffer, ImageCopyBufferBase, ImageCopyTexture, ImageCopyTextureBase, - ImageDataLayout, ImageSubresourceRange, IndexFormat, Limits as WgpuLimits, LoadOp, MapMode, - MultisampleState, Operations, Origin3d, PipelineLayout, PipelineLayoutDescriptor, PolygonMode, - PrimitiveState, PrimitiveTopology, RenderPassColorAttachment, RenderPassDepthStencilAttachment, - RenderPassDescriptor, RenderPipelineDescriptor as RawRenderPipelineDescriptor, - SamplerBindingType, SamplerDescriptor, ShaderModule, ShaderModuleDescriptor, ShaderSource, - ShaderStages, StencilFaceState, StencilOperation, StencilState, StorageTextureAccess, - TextureAspect, TextureDescriptor, TextureDimension, TextureFormat, TextureSampleType, - TextureUsages, TextureViewDescriptor, TextureViewDimension, VertexAttribute, - VertexBufferLayout as RawVertexBufferLayout, VertexFormat, VertexState as RawVertexState, - VertexStepMode, + ImageDataLayout, ImageSubresourceRange, IndexFormat, Limits as WgpuLimits, LoadOp, Maintain, + MapMode, MultisampleState, Operations, Origin3d, PipelineLayout, PipelineLayoutDescriptor, + PolygonMode, PrimitiveState, PrimitiveTopology, RenderPassColorAttachment, + RenderPassDepthStencilAttachment, RenderPassDescriptor, + RenderPipelineDescriptor as RawRenderPipelineDescriptor, SamplerBindingType, SamplerDescriptor, + ShaderModule, ShaderModuleDescriptor, ShaderSource, ShaderStages, StencilFaceState, + StencilOperation, StencilState, StorageTextureAccess, TextureAspect, TextureDescriptor, + TextureDimension, TextureFormat, TextureSampleType, TextureUsages, TextureViewDescriptor, + TextureViewDimension, VertexAttribute, VertexBufferLayout as RawVertexBufferLayout, + VertexFormat, VertexState as RawVertexState, VertexStepMode, }; pub mod encase { diff --git a/crates/bevy_render/src/view/mod.rs b/crates/bevy_render/src/view/mod.rs index 3bb6479c86e89..e676a0c1c4bd0 100644 --- a/crates/bevy_render/src/view/mod.rs +++ b/crates/bevy_render/src/view/mod.rs @@ -110,6 +110,7 @@ pub struct ViewUniform { world_position: Vec3, // viewport(x_origin, y_origin, width, height) viewport: Vec4, + entity_index: u32, } #[derive(Resource, Default)] @@ -255,6 +256,7 @@ fn prepare_view_uniforms( inverse_projection, world_position: camera.transform.translation(), viewport: camera.viewport.as_vec4(), + entity_index: entity.index(), }), }; diff --git a/crates/bevy_sprite/src/mesh2d/color_material.wgsl b/crates/bevy_sprite/src/mesh2d/color_material.wgsl index 0e83d1487dcf2..a2a24086e59a5 100644 --- a/crates/bevy_sprite/src/mesh2d/color_material.wgsl +++ b/crates/bevy_sprite/src/mesh2d/color_material.wgsl @@ -22,8 +22,13 @@ struct FragmentInput { #import bevy_sprite::mesh2d_vertex_output }; +struct FragmentOutput { + @location(0) color: vec4, + @location(1) picking: u32, + } + @fragment -fn fragment(in: FragmentInput) -> @location(0) vec4 { +fn fragment(in: FragmentInput) -> FragmentOutput { var output_color: vec4 = material.color; #ifdef VERTEX_COLORS output_color = output_color * in.color; @@ -31,5 +36,11 @@ fn fragment(in: FragmentInput) -> @location(0) vec4 { if ((material.flags & COLOR_MATERIAL_FLAGS_TEXTURE_BIT) != 0u) { output_color = output_color * textureSample(texture, texture_sampler, in.uv); } - return output_color; + + var out: FragmentOutput; + + out.color = output_color; + out.picking = mesh.entity_index; + + return out; } diff --git a/crates/bevy_sprite/src/mesh2d/mesh.rs b/crates/bevy_sprite/src/mesh2d/mesh.rs index 03e9f468aaa21..5fcfed4dc6979 100644 --- a/crates/bevy_sprite/src/mesh2d/mesh.rs +++ b/crates/bevy_sprite/src/mesh2d/mesh.rs @@ -113,6 +113,7 @@ impl Plugin for Mesh2dRenderPlugin { pub struct Mesh2dUniform { pub transform: Mat4, pub inverse_transpose_model: Mat4, + pub entity_index: u32, pub flags: u32, } @@ -144,6 +145,7 @@ pub fn extract_mesh2d( flags: MeshFlags::empty().bits, transform, inverse_transpose_model: transform.inverse().transpose(), + entity_index: entity.index(), }, ), )); @@ -403,11 +405,18 @@ impl SpecializedMeshPipeline for Mesh2dPipeline { shader: MESH2D_SHADER_HANDLE.typed::(), shader_defs, entry_point: "fragment".into(), - targets: vec![Some(ColorTargetState { - format, - blend: Some(BlendState::ALPHA_BLENDING), - write_mask: ColorWrites::ALL, - })], + targets: vec![ + Some(ColorTargetState { + format, + blend: Some(BlendState::ALPHA_BLENDING), + write_mask: ColorWrites::ALL, + }), + Some(ColorTargetState { + format: TextureFormat::R32Uint, + blend: None, + write_mask: ColorWrites::ALL, + }), + ], }), layout: Some(vec![self.view_layout.clone(), self.mesh_layout.clone()]), primitive: PrimitiveState { diff --git a/crates/bevy_sprite/src/mesh2d/mesh2d_types.wgsl b/crates/bevy_sprite/src/mesh2d/mesh2d_types.wgsl index 1de0218112a47..9c2c4140facbc 100644 --- a/crates/bevy_sprite/src/mesh2d/mesh2d_types.wgsl +++ b/crates/bevy_sprite/src/mesh2d/mesh2d_types.wgsl @@ -3,6 +3,7 @@ struct Mesh2d { model: mat4x4, inverse_transpose_model: mat4x4, + entity_index: u32, // 'flags' is a bit field indicating various options. u32 is 32 bits so we have up to 32 options. flags: u32, }; diff --git a/crates/bevy_sprite/src/render/mod.rs b/crates/bevy_sprite/src/render/mod.rs index 09c386b4064c2..4b52de2525d61 100644 --- a/crates/bevy_sprite/src/render/mod.rs +++ b/crates/bevy_sprite/src/render/mod.rs @@ -200,6 +200,8 @@ impl SpecializedRenderPipeline for SpritePipeline { VertexFormat::Float32x3, // uv VertexFormat::Float32x2, + // entity index + VertexFormat::Uint32, ]; if key.contains(SpritePipelineKey::COLORED) { @@ -240,11 +242,18 @@ impl SpecializedRenderPipeline for SpritePipeline { shader: SPRITE_SHADER_HANDLE.typed::(), shader_defs, entry_point: "fragment".into(), - targets: vec![Some(ColorTargetState { - format, - blend: Some(BlendState::ALPHA_BLENDING), - write_mask: ColorWrites::ALL, - })], + targets: vec![ + Some(ColorTargetState { + format, + blend: Some(BlendState::ALPHA_BLENDING), + write_mask: ColorWrites::ALL, + }), + Some(ColorTargetState { + format: TextureFormat::R32Uint, + blend: None, + write_mask: ColorWrites::ALL, + }), + ], }), layout: Some(vec![self.view_layout.clone(), self.material_layout.clone()]), primitive: PrimitiveState { @@ -386,6 +395,7 @@ pub fn extract_sprites( struct SpriteVertex { pub position: [f32; 3], pub uv: [f32; 2], + pub entity_index: u32, } #[repr(C)] @@ -393,6 +403,7 @@ struct SpriteVertex { struct ColoredSpriteVertex { pub position: [f32; 3], pub uv: [f32; 2], + pub entity_index: u32, pub color: [f32; 4], } @@ -645,6 +656,7 @@ pub fn queue_sprites( position: positions[i], uv: uvs[i].into(), color: extracted_sprite.color.as_linear_rgba_f32(), + entity_index: extracted_sprite.entity.index(), }); } let item_start = colored_index; @@ -663,6 +675,7 @@ pub fn queue_sprites( sprite_meta.vertices.push(SpriteVertex { position: positions[i], uv: uvs[i].into(), + entity_index: extracted_sprite.entity.index(), }); } let item_start = index; diff --git a/crates/bevy_sprite/src/render/sprite.wgsl b/crates/bevy_sprite/src/render/sprite.wgsl index d1097e61af6fc..2086d45c2a364 100644 --- a/crates/bevy_sprite/src/render/sprite.wgsl +++ b/crates/bevy_sprite/src/render/sprite.wgsl @@ -18,8 +18,9 @@ var view: View; struct VertexOutput { @location(0) uv: vec2, + @location(1) entity_index: u32, #ifdef COLORED - @location(1) color: vec4, + @location(2) color: vec4, #endif @builtin(position) position: vec4, }; @@ -28,12 +29,14 @@ struct VertexOutput { fn vertex( @location(0) vertex_position: vec3, @location(1) vertex_uv: vec2, + @location(2) entity_index: u32, #ifdef COLORED - @location(2) vertex_color: vec4, + @location(3) vertex_color: vec4, #endif ) -> VertexOutput { var out: VertexOutput; out.uv = vertex_uv; + out.entity_index = entity_index; out.position = view.view_proj * vec4(vertex_position, 1.0); #ifdef COLORED out.color = vertex_color; @@ -46,8 +49,13 @@ var sprite_texture: texture_2d; @group(1) @binding(1) var sprite_sampler: sampler; +struct FragmentOutput { + @location(0) color: vec4, + @location(1) picking: u32, + } + @fragment -fn fragment(in: VertexOutput) -> @location(0) vec4 { +fn fragment(in: VertexOutput) -> FragmentOutput { var color = textureSample(sprite_texture, sprite_sampler, in.uv); #ifdef COLORED color = in.color * color; @@ -57,5 +65,10 @@ fn fragment(in: VertexOutput) -> @location(0) vec4 { color = vec4(reinhard_luminance(color.rgb), color.a); #endif - return color; + var out: FragmentOutput; + + out.color = color; + out.picking = in.entity_index; + + return out; } diff --git a/crates/bevy_ui/src/render/mod.rs b/crates/bevy_ui/src/render/mod.rs index 0a6d93285ac4e..63a1a24605c8d 100644 --- a/crates/bevy_ui/src/render/mod.rs +++ b/crates/bevy_ui/src/render/mod.rs @@ -102,16 +102,16 @@ pub fn build_ui_render(app: &mut App) { draw_ui_graph::node::UI_PASS, RunGraphOnViewNode::new(draw_ui_graph::NAME), ); - graph_2d.add_node_edge( - bevy_core_pipeline::core_2d::graph::node::MAIN_PASS, - draw_ui_graph::node::UI_PASS, - ); graph_2d.add_slot_edge( graph_2d.input_node().id, bevy_core_pipeline::core_2d::graph::input::VIEW_ENTITY, draw_ui_graph::node::UI_PASS, RunGraphOnViewNode::IN_VIEW, ); + graph_2d.add_node_edge( + bevy_core_pipeline::core_2d::graph::node::MAIN_PASS, + draw_ui_graph::node::UI_PASS, + ); graph_2d.add_node_edge( bevy_core_pipeline::core_2d::graph::node::END_MAIN_PASS_POST_PROCESSING, draw_ui_graph::node::UI_PASS, @@ -120,6 +120,10 @@ pub fn build_ui_render(app: &mut App) { draw_ui_graph::node::UI_PASS, bevy_core_pipeline::core_2d::graph::node::UPSCALING, ); + graph_2d.add_node_edge( + draw_ui_graph::node::UI_PASS, + bevy_core_pipeline::core_2d::graph::node::PICKING, + ); } if let Some(graph_3d) = graph.get_sub_graph_mut(bevy_core_pipeline::core_3d::graph::NAME) { @@ -128,6 +132,12 @@ pub fn build_ui_render(app: &mut App) { draw_ui_graph::node::UI_PASS, RunGraphOnViewNode::new(draw_ui_graph::NAME), ); + graph_3d.add_slot_edge( + graph_3d.input_node().id, + bevy_core_pipeline::core_3d::graph::input::VIEW_ENTITY, + draw_ui_graph::node::UI_PASS, + RunGraphOnViewNode::IN_VIEW, + ); graph_3d.add_node_edge( bevy_core_pipeline::core_3d::graph::node::MAIN_PASS, draw_ui_graph::node::UI_PASS, @@ -140,11 +150,9 @@ pub fn build_ui_render(app: &mut App) { draw_ui_graph::node::UI_PASS, bevy_core_pipeline::core_3d::graph::node::UPSCALING, ); - graph_3d.add_slot_edge( - graph_3d.input_node().id, - bevy_core_pipeline::core_3d::graph::input::VIEW_ENTITY, + graph_3d.add_node_edge( draw_ui_graph::node::UI_PASS, - RunGraphOnViewNode::IN_VIEW, + bevy_core_pipeline::core_3d::graph::node::PICKING, ); } } @@ -167,6 +175,7 @@ fn get_ui_graph(render_app: &mut App) -> RenderGraph { } pub struct ExtractedUiNode { + pub entity: Entity, pub stack_index: usize, pub transform: Mat4, pub background_color: Color, @@ -233,6 +242,7 @@ pub fn extract_uinodes( clip: clip.map(|clip| clip.clip), flip_x, flip_y, + entity: *entity, }); } } @@ -360,6 +370,7 @@ pub fn extract_text_uinodes( clip: clip.map(|clip| clip.clip), flip_x: false, flip_y: false, + entity: *entity, }); } } @@ -371,6 +382,7 @@ pub fn extract_text_uinodes( struct UiVertex { pub position: [f32; 3], pub uv: [f32; 2], + pub entity_index: u32, pub color: [f32; 4], } @@ -525,6 +537,7 @@ pub fn prepare_uinodes( position: positions_clipped[i].into(), uv: uvs[i].into(), color: extracted_uinode.background_color.as_linear_rgba_f32(), + entity_index: extracted_uinode.entity.index(), }); } diff --git a/crates/bevy_ui/src/render/pipeline.rs b/crates/bevy_ui/src/render/pipeline.rs index 1a86d577ab417..676395aa48f61 100644 --- a/crates/bevy_ui/src/render/pipeline.rs +++ b/crates/bevy_ui/src/render/pipeline.rs @@ -75,6 +75,8 @@ impl SpecializedRenderPipeline for UiPipeline { VertexFormat::Float32x3, // uv VertexFormat::Float32x2, + // entity index + VertexFormat::Uint32, // color VertexFormat::Float32x4, ], @@ -92,15 +94,22 @@ impl SpecializedRenderPipeline for UiPipeline { shader: super::UI_SHADER_HANDLE.typed::(), shader_defs, entry_point: "fragment".into(), - targets: vec![Some(ColorTargetState { - format: if key.hdr { - ViewTarget::TEXTURE_FORMAT_HDR - } else { - TextureFormat::bevy_default() - }, - blend: Some(BlendState::ALPHA_BLENDING), - write_mask: ColorWrites::ALL, - })], + targets: vec![ + Some(ColorTargetState { + format: if key.hdr { + ViewTarget::TEXTURE_FORMAT_HDR + } else { + TextureFormat::bevy_default() + }, + blend: Some(BlendState::ALPHA_BLENDING), + write_mask: ColorWrites::ALL, + }), + Some(ColorTargetState { + format: TextureFormat::R32Uint, + blend: None, + write_mask: ColorWrites::ALL, + }), + ], }), layout: Some(vec![self.view_layout.clone(), self.image_layout.clone()]), primitive: PrimitiveState { diff --git a/crates/bevy_ui/src/render/render_pass.rs b/crates/bevy_ui/src/render/render_pass.rs index 63ff0a47f989c..dbfe77ea43522 100644 --- a/crates/bevy_ui/src/render/render_pass.rs +++ b/crates/bevy_ui/src/render/render_pass.rs @@ -5,6 +5,7 @@ use bevy_ecs::{ system::{lifetimeless::*, SystemParamItem}, }; use bevy_render::{ + picking::PickingTextures, render_graph::*, render_phase::*, render_resource::{CachedRenderPipelineId, LoadOp, Operations, RenderPassDescriptor}, @@ -18,6 +19,7 @@ pub struct UiPassNode { ( &'static RenderPhase, &'static ViewTarget, + &'static PickingTextures, Option<&'static UiCameraConfig>, ), With, @@ -54,7 +56,7 @@ impl Node for UiPassNode { ) -> Result<(), NodeRunError> { let input_view_entity = graph.get_input_entity(Self::IN_VIEW)?; - let Ok((transparent_phase, target, camera_ui)) = + let Ok((transparent_phase, target, picking_textures, camera_ui)) = self.ui_view_query.get_manual(world, input_view_entity) else { return Ok(()); @@ -76,12 +78,18 @@ impl Node for UiPassNode { } else { input_view_entity }; + + let ops = Operations { + load: LoadOp::Load, + store: true, + }; + let mut render_pass = render_context.begin_tracked_render_pass(RenderPassDescriptor { label: Some("ui_pass"), - color_attachments: &[Some(target.get_unsampled_color_attachment(Operations { - load: LoadOp::Load, - store: true, - }))], + color_attachments: &[ + Some(target.get_unsampled_color_attachment(ops)), + Some(picking_textures.get_unsampled_color_attachment(ops)), + ], depth_stencil_attachment: None, }); diff --git a/crates/bevy_ui/src/render/ui.wgsl b/crates/bevy_ui/src/render/ui.wgsl index 0cf8ed8aa784c..01598f59eaf70 100644 --- a/crates/bevy_ui/src/render/ui.wgsl +++ b/crates/bevy_ui/src/render/ui.wgsl @@ -14,7 +14,8 @@ var view: View; struct VertexOutput { @location(0) uv: vec2, - @location(1) color: vec4, + @location(1) entity_index: u32, + @location(2) color: vec4, @builtin(position) position: vec4, }; @@ -22,11 +23,13 @@ struct VertexOutput { fn vertex( @location(0) vertex_position: vec3, @location(1) vertex_uv: vec2, - @location(2) vertex_color: vec4, + @location(2) entity_index: u32, + @location(3) vertex_color: vec4, ) -> VertexOutput { var out: VertexOutput; out.uv = vertex_uv; out.position = view.view_proj * vec4(vertex_position, 1.0); + out.entity_index = entity_index; out.color = vertex_color; return out; } @@ -36,9 +39,20 @@ var sprite_texture: texture_2d; @group(1) @binding(1) var sprite_sampler: sampler; +struct FragmentOutput { + @location(0) color: vec4, + @location(1) picking: u32, + } + @fragment -fn fragment(in: VertexOutput) -> @location(0) vec4 { +fn fragment(in: VertexOutput) -> FragmentOutput { var color = textureSample(sprite_texture, sprite_sampler, in.uv); color = in.color * color; - return color; + + var out: FragmentOutput; + + out.color = color; + out.picking = in.entity_index; + + return out; } diff --git a/examples/app/picking.rs b/examples/app/picking.rs new file mode 100644 index 0000000000000..1758bcf0562e9 --- /dev/null +++ b/examples/app/picking.rs @@ -0,0 +1,344 @@ +//! An example that shows how both 3D meshes and UI entities may be "picked" by +//! using the cursor. +//! +//! Combines parts of the 3D shapes example and the UI example. + +use std::f32::consts::PI; + +use bevy::{ + input::mouse::{MouseScrollUnit, MouseWheel}, + prelude::*, + render::picking::{PickedEvent, PickedEventVariant, Picking}, +}; + +fn main() { + App::new() + .add_plugins(DefaultPlugins) + .add_startup_system(setup) + .add_startup_system(setup_ui) + .add_system(rotate_shapes) + .add_system(mouse_scroll) + .add_system(picking_shapes) + .add_system(picking_logo) + .add_system(picking_text) + .run(); +} + +/// A marker component for our shapes so we can query them separately from the ground plane +#[derive(Component)] +struct Shape; + +const X_EXTENT: f32 = 14.5; + +const LOGO_NORMAL: f32 = 500.0; +const LOGO_HOVERED: f32 = 600.0; + +const COLOR_NORMAL: Color = Color::WHITE; +const COLOR_HOVERED: Color = Color::GOLD; + +#[derive(Resource, Deref, DerefMut)] +struct NormalMaterial(Handle); + +#[derive(Resource, Deref, DerefMut)] +struct HoveredMaterial(Handle); + +#[derive(Resource, Deref, DerefMut)] +struct SelectedMaterial(Handle); + +fn setup( + mut commands: Commands, + mut meshes: ResMut>, + mut materials: ResMut>, +) { + let normal = materials.add(COLOR_NORMAL.into()); + + commands.insert_resource(NormalMaterial(normal.clone())); + commands.insert_resource(HoveredMaterial(materials.add(COLOR_HOVERED.into()))); + + let shapes = [ + meshes.add(shape::Cube::default().into()), + meshes.add(shape::Box::default().into()), + meshes.add(shape::Capsule::default().into()), + meshes.add(shape::Torus::default().into()), + meshes.add(shape::Cylinder::default().into()), + meshes.add(shape::Icosphere::default().try_into().unwrap()), + meshes.add(shape::UVSphere::default().into()), + ]; + + let num_shapes = shapes.len(); + + for (i, shape) in shapes.into_iter().enumerate() { + commands.spawn(( + PbrBundle { + mesh: shape, + material: normal.clone(), + transform: Transform::from_xyz( + -X_EXTENT / 2. + i as f32 / (num_shapes - 1) as f32 * X_EXTENT, + 2.0, + 0.0, + ) + .with_rotation(Quat::from_rotation_x(-PI / 4.)), + ..default() + }, + Shape, + )); + } + + commands.spawn(PointLightBundle { + point_light: PointLight { + intensity: 9000.0, + range: 100., + shadows_enabled: true, + ..default() + }, + transform: Transform::from_xyz(8.0, 16.0, 8.0), + ..default() + }); + + // ground plane + commands.spawn(PbrBundle { + mesh: meshes.add(shape::Plane { size: 50. }.into()), + material: materials.add(Color::SILVER.into()), + ..default() + }); + + commands.spawn(( + Camera3dBundle { + transform: Transform::from_xyz(0.0, 6., 12.0) + .looking_at(Vec3::new(0., 1., 0.), Vec3::Y), + ..default() + }, + Picking::default(), + )); +} + +fn setup_ui(mut commands: Commands, asset_server: Res) { + // root node + commands + .spawn(NodeBundle { + style: Style { + size: Size::new(Val::Percent(100.0), Val::Percent(100.0)), + justify_content: JustifyContent::SpaceBetween, + ..default() + }, + ..default() + }) + .with_children(|parent| { + // right vertical fill + parent + .spawn(NodeBundle { + style: Style { + flex_direction: FlexDirection::Column, + justify_content: JustifyContent::Center, + size: Size::new(Val::Px(200.0), Val::Percent(100.0)), + ..default() + }, + background_color: Color::rgb(0.15, 0.15, 0.15).into(), + ..default() + }) + .with_children(|parent| { + // Title + parent.spawn( + TextBundle::from_section( + "Scrolling list", + TextStyle { + font: asset_server.load("fonts/FiraSans-Bold.ttf"), + font_size: 25., + color: Color::WHITE, + }, + ) + .with_style(Style { + size: Size::new(Val::Undefined, Val::Px(25.)), + margin: UiRect { + left: Val::Auto, + right: Val::Auto, + ..default() + }, + ..default() + }), + ); + // List with hidden overflow + parent + .spawn(NodeBundle { + style: Style { + flex_direction: FlexDirection::Column, + align_self: AlignSelf::Center, + size: Size::new(Val::Percent(100.0), Val::Percent(50.0)), + overflow: Overflow::Hidden, + ..default() + }, + background_color: Color::rgb(0.10, 0.10, 0.10).into(), + ..default() + }) + .with_children(|parent| { + // Moving panel + parent + .spawn(( + NodeBundle { + style: Style { + flex_direction: FlexDirection::Column, + flex_grow: 1.0, + max_size: Size::UNDEFINED, + ..default() + }, + ..default() + }, + ScrollingList::default(), + )) + .with_children(|parent| { + // List items + for i in 0..50 { + parent.spawn( + TextBundle::from_section( + format!("Item {i}"), + TextStyle { + font: asset_server + .load("fonts/FiraSans-Bold.ttf"), + font_size: 30., + color: COLOR_NORMAL, + }, + ) + .with_style(Style { + flex_shrink: 0., + size: Size::new(Val::Undefined, Val::Px(20.)), + margin: UiRect { + left: Val::Auto, + right: Val::Auto, + ..default() + }, + ..default() + }), + ); + } + }); + }); + }); + // bevy logo (flex center) + parent + .spawn(NodeBundle { + style: Style { + size: Size::new(Val::Percent(100.0), Val::Percent(100.0)), + position_type: PositionType::Absolute, + justify_content: JustifyContent::Center, + align_items: AlignItems::FlexStart, + ..default() + }, + ..default() + }) + .with_children(|parent| { + // bevy logo (image) + parent.spawn(ImageBundle { + style: Style { + size: Size::new(Val::Px(LOGO_NORMAL), Val::Auto), + ..default() + }, + image: asset_server.load("branding/bevy_logo_dark_big.png").into(), + ..default() + }); + }); + }); +} + +#[derive(Component, Default)] +struct ScrollingList { + position: f32, +} + +fn mouse_scroll( + mut mouse_wheel_events: EventReader, + mut query_list: Query<(&mut ScrollingList, &mut Style, &Children, &Node)>, + query_item: Query<&Node>, +) { + for mouse_wheel_event in mouse_wheel_events.iter() { + for (mut scrolling_list, mut style, children, uinode) in &mut query_list { + let items_height: f32 = children + .iter() + .map(|entity| query_item.get(*entity).unwrap().size().y) + .sum(); + let panel_height = uinode.size().y; + let max_scroll = (items_height - panel_height).max(0.); + let dy = match mouse_wheel_event.unit { + MouseScrollUnit::Line => mouse_wheel_event.y * 20., + MouseScrollUnit::Pixel => mouse_wheel_event.y, + }; + scrolling_list.position += dy; + scrolling_list.position = scrolling_list.position.clamp(-max_scroll, 0.); + style.position.top = Val::Px(scrolling_list.position); + } + } +} + +fn rotate_shapes(mut query: Query<&mut Transform, With>, time: Res