GPU Picking v2

Cargo.toml

@@ -1444,6 +1444,16 @@ description = "Shows how to rumble a gamepad using force feedback"
 category = "Input"
 wasm = false
 
+[[example]]
+name = "gpu_picking"
+path = "examples/input/gpu_picking.rs"
+
+[package.metadata.example.gpu_picking]
+name = "GPU picking"
+description = "Mouse picking using the gpu"
+category = "Input"
+wasm = true
+
 [[example]]
 name = "keyboard_input"
 path = "examples/input/keyboard_input.rs"

assets/shaders/gpu_picking_material.wgsl

@@ -0,0 +1,28 @@
+// This shader shows how to enable the gpu picking feature for a material
+
+// You'll need the mesh binding because that's where the entity index is
+#import bevy_pbr::mesh_bindings mesh
+#import bevy_pbr::mesh_vertex_output MeshVertexOutput
+
+@group(1) @binding(0)
+var<uniform> color: vec4<f32>;
+
+// Gpu picking uses multiple fragment output
+struct FragmentOutput {
+ @location(0) color: vec4<f32>,
+// You can detect the feature with this flag
+#ifdef GPU_PICKING
+ @location(1) mesh_id: u32,
+#endif
+};
+
+@fragment
+fn fragment(in: MeshVertexOutput) -> FragmentOutput {
+ var out: FragmentOutput;
+ out.color = color;
+// make sure to output the entity index for gpu picking to work correctly
+#ifdef GPU_PICKING
+ out.mesh_id = mesh[in.instance_index].id;
+#endif
+ return out;
+}

crates/bevy_core_pipeline/Cargo.toml

@@ -33,3 +33,4 @@ bevy_utils = { path = "../bevy_utils", version = "0.12.0-dev" }
 serde = { version = "1", features = ["derive"] }
 bitflags = "2.3"
 radsort = "0.1"
+smallvec = "1.6"

crates/bevy_core_pipeline/src/core_3d/main_opaque_pass_3d_node.rs

@@ -7,16 +7,19 @@ use crate::{
 use bevy_ecs::{prelude::*, query::QueryItem};
 use bevy_render::{
  camera::ExtractedCamera,
+ picking::{ExtractedGpuPickingCamera, VisibleMeshIdTextures},
  render_graph::{NodeRunError, RenderGraphContext, ViewNode},
  render_phase::RenderPhase,
  render_resource::{
- LoadOp, Operations, PipelineCache, RenderPassDepthStencilAttachment, RenderPassDescriptor,
+ LoadOp, Operations, PipelineCache, RenderPassColorAttachment,
+ RenderPassDepthStencilAttachment, RenderPassDescriptor,
  },
  renderer::RenderContext,
  view::{ViewDepthTexture, ViewTarget, ViewUniformOffset},
 };
 #[cfg(feature = "trace")]
 use bevy_utils::tracing::info_span;
+use smallvec::{smallvec, SmallVec};
 
 use super::{AlphaMask3d, Camera3dDepthLoadOp};
 
@@ -34,8 +37,10 @@ impl ViewNode for MainOpaquePass3dNode {
  Option<&'static DepthPrepass>,
  Option<&'static NormalPrepass>,
  Option<&'static MotionVectorPrepass>,
+ Option<&'static ExtractedGpuPickingCamera>,
  Option<&'static SkyboxPipelineId>,
  Option<&'static SkyboxBindGroup>,
+ Option<&'static VisibleMeshIdTextures>,
  &'static ViewUniformOffset,
  );
 
@@ -53,8 +58,10 @@ impl ViewNode for MainOpaquePass3dNode {
  depth_prepass,
  normal_prepass,
  motion_vector_prepass,
+ gpu_picking_camera,
  skybox_pipeline,
  skybox_bind_group,
+ mesh_id_textures,
  view_uniform_offset,
  ): QueryItem<Self::ViewQuery>,
  world: &World,
@@ -64,21 +71,36 @@ impl ViewNode for MainOpaquePass3dNode {
  #[cfg(feature = "trace")]
  let _main_opaque_pass_3d_span = info_span!("main_opaque_pass_3d").entered();
 
- // Setup render pass
- let mut render_pass = render_context.begin_tracked_render_pass(RenderPassDescriptor {
- label: Some("main_opaque_pass_3d"),
- // NOTE: The opaque pass loads the color
- // buffer as well as writing to it.
- color_attachments: &[Some(target.get_color_attachment(Operations {
+ let mut color_attachments: SmallVec<[Option<RenderPassColorAttachment>; 2]> =
+ smallvec![Some(target.get_color_attachment(Operations {
  load: match camera_3d.clear_color {
- ClearColorConfig::Default => {
- LoadOp::Clear(world.resource::<ClearColor>().0.into())
- }
+ ClearColorConfig::Default =>
+ LoadOp::Clear(world.resource::<ClearColor>().0.into()),
  ClearColorConfig::Custom(color) => LoadOp::Clear(color.into()),
  ClearColorConfig::None => LoadOp::Load,
  },
  store: true,
- }))],
+ }))];
+
+ if gpu_picking_camera.is_some() {
+ if let Some(mesh_id_textures) = mesh_id_textures {
+ color_attachments.push(Some(mesh_id_textures.get_color_attachment(Operations {
+ load: match camera_3d.clear_color {
+ ClearColorConfig::None => LoadOp::Load,
+ // TODO clear this earlier?
+ _ => LoadOp::Clear(VisibleMeshIdTextures::CLEAR_COLOR),
+ },
+ store: true,
+ })));
+ }
+ }
+
+ // Setup render pass
+ let mut render_pass = render_context.begin_tracked_render_pass(RenderPassDescriptor {
+ label: Some("main_opaque_pass_3d"),
+ // NOTE: The opaque pass loads the color
+ // buffer as well as writing to it.
+ color_attachments: &color_attachments,
  depth_stencil_attachment: Some(RenderPassDepthStencilAttachment {
  view: &depth.view,
  // NOTE: The opaque main pass loads the depth buffer and possibly overwrites it

crates/bevy_core_pipeline/src/core_3d/main_transparent_pass_3d_node.rs

@@ -2,6 +2,7 @@ use crate::core_3d::Transparent3d;
 use bevy_ecs::{prelude::*, query::QueryItem};
 use bevy_render::{
  camera::ExtractedCamera,
+ picking::{ExtractedGpuPickingCamera, VisibleMeshIdTextures},
  render_graph::{NodeRunError, RenderGraphContext, ViewNode},
  render_phase::RenderPhase,
  render_resource::{LoadOp, Operations, RenderPassDepthStencilAttachment, RenderPassDescriptor},
@@ -21,12 +22,16 @@ impl ViewNode for MainTransparentPass3dNode {
  &'static RenderPhase<Transparent3d>,
  &'static ViewTarget,
  &'static ViewDepthTexture,
+ Option<&'static ExtractedGpuPickingCamera>,
+ Option<&'static VisibleMeshIdTextures>,
  );
  fn run(
  &self,
  graph: &mut RenderGraphContext,
  render_context: &mut RenderContext,
- (camera, transparent_phase, target, depth): QueryItem<Self::ViewQuery>,
+ (camera, transparent_phase, target, depth, gpu_picking_camera, mesh_id_textures): QueryItem<
+ Self::ViewQuery,
+ >,
  world: &World,
  ) -> Result<(), NodeRunError> {
  let view_entity = graph.view_entity();
@@ -37,13 +42,27 @@ impl ViewNode for MainTransparentPass3dNode {
  #[cfg(feature = "trace")]
  let _main_transparent_pass_3d_span = info_span!("main_transparent_pass_3d").entered();
 
+ // NOTE: The transparent pass loads the color buffer as well as overwriting it where appropriate.
+ let mut color_attachments = vec![Some(target.get_color_attachment(Operations {
+ load: LoadOp::Load,
+ store: true,
+ }))];
+
+ if gpu_picking_camera.is_some() {
+ if let Some(mesh_id_textures) = mesh_id_textures {
+ color_attachments.push(Some(mesh_id_textures.get_color_attachment(
+ Operations {
+ // The texture is already cleared in the opaque pass
+ load: LoadOp::Load,
+ store: true,
+ },
+ )));
+ }
+ }
+
  let mut render_pass = render_context.begin_tracked_render_pass(RenderPassDescriptor {
  label: Some("main_transparent_pass_3d"),
- // NOTE: The transparent pass loads the color buffer as well as overwriting it where appropriate.
- color_attachments: &[Some(target.get_color_attachment(Operations {
- load: LoadOp::Load,
- store: true,
- }))],
+ color_attachments: &color_attachments,
  depth_stencil_attachment: Some(RenderPassDepthStencilAttachment {
  view: &depth.view,
  // NOTE: For the transparent pass we load the depth buffer. There should be no

crates/bevy_core_pipeline/src/core_3d/mod.rs

@@ -14,6 +14,7 @@ pub mod graph {
  pub const MAIN_OPAQUE_PASS: &str = "main_opaque_pass";
  pub const MAIN_TRANSPARENT_PASS: &str = "main_transparent_pass";
  pub const END_MAIN_PASS: &str = "end_main_pass";
+ pub const ENTITY_INDEX_BUFFER_COPY: &str = "entity_index_buffer_copy";
  pub const BLOOM: &str = "bloom";
  pub const TONEMAPPING: &str = "tonemapping";
  pub const FXAA: &str = "fxaa";
@@ -35,6 +36,7 @@ use bevy_ecs::prelude::*;
 use bevy_render::{
  camera::{Camera, ExtractedCamera},
  extract_component::ExtractComponentPlugin,
+ picking::{ExtractedGpuPickingCamera, VisibleMeshIdTextures, MESH_ID_TEXTURE_FORMAT},
  prelude::Msaa,
  render_graph::{EmptyNode, RenderGraphApp, ViewNodeRunner},
  render_phase::{
@@ -94,6 +96,7 @@ impl Plugin for Core3dPlugin {
  sort_phase_system::<AlphaMask3dPrepass>.in_set(RenderSet::PhaseSort),
  prepare_core_3d_depth_textures.in_set(RenderSet::PrepareResources),
  prepare_prepass_textures.in_set(RenderSet::PrepareResources),
+ prepare_entity_textures.in_set(RenderSet::PrepareResources),
  ),
  );
 
@@ -504,3 +507,63 @@ pub fn prepare_prepass_textures(
  });
  }
 }
+
+/// Create the required textures based on the camera size
+pub fn prepare_entity_textures(
+ mut commands: Commands,
+ mut texture_cache: ResMut<TextureCache>,
+ msaa: Res<Msaa>,
+ render_device: Res<RenderDevice>,
+ views_3d: Query<
+ (Entity, &ExtractedCamera),
+ (
+ With<ExtractedGpuPickingCamera>,
+ With<RenderPhase<Opaque3d>>,
+ With<RenderPhase<AlphaMask3d>>,
+ ),
+ >,
+) {
+ for (entity, camera) in &views_3d {
+ let Some(physical_target_size) = camera.physical_target_size else {
+ continue;
+ };
+
+ let size = Extent3d {
+ depth_or_array_layers: 1,
+ width: physical_target_size.x,
+ height: physical_target_size.y,
+ };
+
+ let descriptor = TextureDescriptor {
+ label: None,
+ size,
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: TextureDimension::D2,
+ format: MESH_ID_TEXTURE_FORMAT,
+ usage: TextureUsages::RENDER_ATTACHMENT | TextureUsages::COPY_SRC,
+ view_formats: &[],
+ };
+
+ let mesh_id_textures = VisibleMeshIdTextures {
+ main: texture_cache.get(
+ &render_device,
+ TextureDescriptor {
+ label: Some("main_entity_texture"),
+ ..descriptor
+ },
+ ),
+ sampled: (msaa.samples() > 1).then(|| {
+ texture_cache.get(
+ &render_device,
+ TextureDescriptor {
+ label: Some("main_entity_texture_sampled"),
+ sample_count: msaa.samples(),
+ ..descriptor
+ },
+ )
+ }),
+ };
+ commands.entity(entity).insert(mesh_id_textures);
+ }
+}

crates/bevy_core_pipeline/src/entity_index_buffer_copy/mod.rs

@@ -0,0 +1,53 @@
+use bevy_app::Plugin;
+use bevy_ecs::{query::QueryItem, world::World};
+use bevy_render::{
+ picking::{CurrentGpuPickingBufferIndex, ExtractedGpuPickingCamera, VisibleMeshIdTextures},
+ render_graph::{RenderGraphApp, RenderGraphContext, ViewNode, ViewNodeRunner},
+ renderer::RenderContext,
+ RenderApp,
+};
+
+use crate::core_3d::CORE_3D;
+
+#[derive(Default)]
+pub struct EntityIndexBufferCopyNode;
+impl ViewNode for EntityIndexBufferCopyNode {
+ type ViewQuery = (
+ &'static VisibleMeshIdTextures,
+ &'static ExtractedGpuPickingCamera,
+ );
+
+ fn run(
+ &self,
+ _graph: &mut RenderGraphContext,
+ render_context: &mut RenderContext,
+ (mesh_id_textures, gpu_picking_camera): QueryItem<Self::ViewQuery>,
+ world: &World,
+ ) -> Result<(), bevy_render::render_graph::NodeRunError> {
+ let current_buffer_index = world.resource::<CurrentGpuPickingBufferIndex>();
+ gpu_picking_camera.run_node(
+ render_context.command_encoder(),
+ &mesh_id_textures.main.texture,
+ current_buffer_index,
+ );
+ Ok(())
+ }
+}
+
+pub struct EntityIndexBufferCopyPlugin;
+impl Plugin for EntityIndexBufferCopyPlugin {
+ fn build(&self, app: &mut bevy_app::App) {
+ let Ok(render_app) = app.get_sub_app_mut(RenderApp) else {
+ return;
+ };
+
+ // 3D
+ use crate::core_3d::graph::node::*;
+ render_app
+ .add_render_graph_node::<ViewNodeRunner<EntityIndexBufferCopyNode>>(
+ CORE_3D,
+ ENTITY_INDEX_BUFFER_COPY,
+ )
+ .add_render_graph_edge(CORE_3D, UPSCALING, ENTITY_INDEX_BUFFER_COPY);
+ }
+}

crates/bevy_core_pipeline/src/lib.rs

@@ -6,6 +6,7 @@ pub mod clear_color;
 pub mod contrast_adaptive_sharpening;
 pub mod core_2d;
 pub mod core_3d;
+pub mod entity_index_buffer_copy;
 pub mod fullscreen_vertex_shader;
 pub mod fxaa;
 pub mod msaa_writeback;
@@ -40,6 +41,7 @@ use crate::{
  contrast_adaptive_sharpening::CASPlugin,
  core_2d::Core2dPlugin,
  core_3d::Core3dPlugin,
+ entity_index_buffer_copy::EntityIndexBufferCopyPlugin,
  fullscreen_vertex_shader::FULLSCREEN_SHADER_HANDLE,
  fxaa::FxaaPlugin,
  msaa_writeback::MsaaWritebackPlugin,
@@ -79,6 +81,7 @@ impl Plugin for CorePipelinePlugin {
  BloomPlugin,
  FxaaPlugin,
  CASPlugin,
+ EntityIndexBufferCopyPlugin,
  ));
  }
 }