Add Order Independent Transparency (#14876)

# Objective

- Alpha blending can easily fail in many situations and requires sorting
on the cpu

## Solution

- Implement order independent transparency (OIT) as an alternative to
alpha blending
- The implementation uses 2 passes
- The first pass records all the fragments colors and position to a
buffer that is the size of N layers * the render target resolution.
- The second pass sorts the fragments, blends them and draws them to the
screen. It also currently does manual depth testing because early-z
fails in too many cases in the first pass.

## Testing

- We've been using this implementation at foresight in production for
many months now and we haven't had any issues related to OIT.

---

## Showcase


![image](https://github.com/user-attachments/assets/157f3e32-adaf-4782-b25b-c10313b9bc43)

![image](https://github.com/user-attachments/assets/bef23258-0c22-4b67-a0b8-48a9f571c44f)

## Future work

- Add an example showing how to use OIT for a custom material
- Next step would be to implement a per-pixel linked list to reduce
memory use
- I'd also like to investigate using a BinnedRenderPhase instead of a
SortedRenderPhase. If it works, it would make the transparent pass
significantly faster.

---------

Co-authored-by: Kristoffer Søholm <k.soeholm@gmail.com>
Co-authored-by: JMS55 <47158642+JMS55@users.noreply.github.com>
Co-authored-by: Charlotte McElwain <charlotte.c.mcelwain@gmail.com>

Cargo.toml

@@ -973,6 +973,17 @@ description = "Demonstrates per-pixel motion blur"
 category = "3D Rendering"
 wasm = false
 
+[[example]]
+name = "order_independent_transparency"
+path = "examples/3d/order_independent_transparency.rs"
+doc-scrape-examples = true
+
+[package.metadata.example.order_independent_transparency]
+name = "Order Independent Transparency"
+description = "Demonstrates how to use OIT"
+category = "3D Rendering"
+wasm = false
+
 [[example]]
 name = "tonemapping"
 path = "examples/3d/tonemapping.rs"

crates/bevy_core_pipeline/Cargo.toml

@@ -34,6 +34,7 @@ bevy_render = { path = "../bevy_render", version = "0.15.0-dev" }
 bevy_transform = { path = "../bevy_transform", version = "0.15.0-dev" }
 bevy_math = { path = "../bevy_math", version = "0.15.0-dev" }
 bevy_utils = { path = "../bevy_utils", version = "0.15.0-dev" }
+bevy_window = { path = "../bevy_window", version = "0.15.0-dev" }
 
 serde = { version = "1", features = ["derive"] }
 bitflags = "2.3"

crates/bevy_core_pipeline/src/lib.rs

@@ -19,6 +19,7 @@ pub mod fullscreen_vertex_shader;
 pub mod fxaa;
 pub mod motion_blur;
 pub mod msaa_writeback;
+pub mod oit;
 pub mod post_process;
 pub mod prepass;
 mod skybox;
@@ -75,6 +76,8 @@ use crate::{
 use bevy_app::{App, Plugin};
 use bevy_asset::load_internal_asset;
 use bevy_render::prelude::Shader;
+#[cfg(not(feature = "webgl"))]
+use oit::OrderIndependentTransparencyPlugin;
 
 #[derive(Default)]
 pub struct CorePipelinePlugin;
@@ -107,6 +110,9 @@ impl Plugin for CorePipelinePlugin {
  DepthOfFieldPlugin,
  SmaaPlugin,
  PostProcessingPlugin,
+ // DownlevelFlags::FRAGMENT_WRITABLE_STORAGE is required for OIT
+ #[cfg(not(feature = "webgl"))]
+ OrderIndependentTransparencyPlugin,
  ));
  }
 }

crates/bevy_core_pipeline/src/oit/mod.rs

@@ -0,0 +1,283 @@
+//! Order Independent Transparency (OIT) for 3d rendering. See [`OrderIndependentTransparencyPlugin`] for more details.
+
+use bevy_app::prelude::*;
+use bevy_asset::{load_internal_asset, Handle};
+use bevy_ecs::prelude::*;
+use bevy_math::UVec2;
+use bevy_render::{
+ camera::{Camera, ExtractedCamera},
+ extract_component::{ExtractComponent, ExtractComponentPlugin},
+ render_graph::{RenderGraphApp, ViewNodeRunner},
+ render_resource::{BufferUsages, BufferVec, DynamicUniformBuffer, Shader, TextureUsages},
+ renderer::{RenderDevice, RenderQueue},
+ view::Msaa,
+ Render, RenderApp, RenderSet,
+};
+use bevy_utils::{tracing::trace, HashSet, Instant};
+use bevy_window::PrimaryWindow;
+use resolve::{
+ node::{OitResolveNode, OitResolvePass},
+ OitResolvePlugin,
+};
+
+use crate::core_3d::{
+ graph::{Core3d, Node3d},
+ Camera3d,
+};
+
+/// Module that defines the necesasry systems to resolve the OIT buffer and render it to the screen.
+pub mod resolve;
+
+/// Shader handle for the shader that draws the transparent meshes to the OIT layers buffer.
+pub const OIT_DRAW_SHADER_HANDLE: Handle<Shader> = Handle::weak_from_u128(4042527984320512);
+
+/// Used to identify which camera will use OIT to render transparent meshes
+/// and to configure OIT.
+// TODO consider supporting multiple OIT techniques like WBOIT, Moment Based OIT,
+// depth peeling, stochastic transparency, ray tracing etc.
+// This should probably be done by adding an enum to this component.
+#[derive(Component, Clone, Copy, ExtractComponent)]
+pub struct OrderIndependentTransparencySettings {
+ /// Controls how many layers will be used to compute the blending.
+ /// The more layers you use the more memory it will use but it will also give better results.
+ /// 8 is generally recommended, going above 16 is probably not worth it in the vast majority of cases
+ pub layer_count: u8,
+}
+
+impl Default for OrderIndependentTransparencySettings {
+ fn default() -> Self {
+ Self { layer_count: 8 }
+ }
+}
+
+/// A plugin that adds support for Order Independent Transparency (OIT).
+/// This can correctly render some scenes that would otherwise have artifacts due to alpha blending, but uses more memory.
+///
+/// To enable OIT for a camera you need to add the [`OrderIndependentTransparencySettings`] component to it.
+///
+/// If you want to use OIT for your custom material you need to call `oit_draw(position, color)` in your fragment shader.
+/// You also need to make sure that your fragment shader doesn't output any colors.
+///
+/// # Implementation details
+/// This implementation uses 2 passes.
+///
+/// The first pass writes the depth and color of all the fragments to a big buffer.
+/// The buffer contains N layers for each pixel, where N can be set with [`OrderIndependentTransparencySettings::layer_count`].
+/// This pass is essentially a forward pass.
+///
+/// The second pass is a single fullscreen triangle pass that sorts all the fragments then blends them together
+/// and outputs the result to the screen.
+pub struct OrderIndependentTransparencyPlugin;
+impl Plugin for OrderIndependentTransparencyPlugin {
+ fn build(&self, app: &mut App) {
+ load_internal_asset!(
+ app,
+ OIT_DRAW_SHADER_HANDLE,
+ "oit_draw.wgsl",
+ Shader::from_wgsl
+ );
+
+ app.add_plugins((
+ ExtractComponentPlugin::<OrderIndependentTransparencySettings>::default(),
+ OitResolvePlugin,
+ ))
+ .add_systems(Update, check_msaa)
+ .add_systems(Last, configure_depth_texture_usages);
+
+ let Some(render_app) = app.get_sub_app_mut(RenderApp) else {
+ return;
+ };
+
+ render_app.add_systems(
+ Render,
+ prepare_oit_buffers.in_set(RenderSet::PrepareResources),
+ );
+
+ render_app
+ .add_render_graph_node::<ViewNodeRunner<OitResolveNode>>(Core3d, OitResolvePass)
+ .add_render_graph_edges(
+ Core3d,
+ (
+ Node3d::MainTransparentPass,
+ OitResolvePass,
+ Node3d::EndMainPass,
+ ),
+ );
+ }
+
+ fn finish(&self, app: &mut App) {
+ let Some(render_app) = app.get_sub_app_mut(RenderApp) else {
+ return;
+ };
+
+ render_app.init_resource::<OitBuffers>();
+ }
+}
+
+// WARN This should only happen for cameras with the [`OrderIndependentTransparencySettings`] component
+// but when multiple cameras are present on the same window
+// bevy reuses the same depth texture so we need to set this on all cameras with the same render target.
+fn configure_depth_texture_usages(
+ p: Query<Entity, With<PrimaryWindow>>,
+ cameras: Query<(&Camera, Has<OrderIndependentTransparencySettings>)>,
+ mut new_cameras: Query<(&mut Camera3d, &Camera), Added<Camera3d>>,
+) {
+ if new_cameras.is_empty() {
+ return;
+ }
+
+ // Find all the render target that potentially uses OIT
+ let primary_window = p.get_single().ok();
+ let mut render_target_has_oit = HashSet::new();
+ for (camera, has_oit) in &cameras {
+ if has_oit {
+ render_target_has_oit.insert(camera.target.normalize(primary_window));
+ }
+ }
+
+ // Update the depth texture usage for cameras with a render target that has OIT
+ for (mut camera_3d, camera) in &mut new_cameras {
+ if render_target_has_oit.contains(&camera.target.normalize(primary_window)) {
+ let mut usages = TextureUsages::from(camera_3d.depth_texture_usages);
+ usages |= TextureUsages::RENDER_ATTACHMENT | TextureUsages::TEXTURE_BINDING;
+ camera_3d.depth_texture_usages = usages.into();
+ }
+ }
+}
+
+fn check_msaa(cameras: Query<&Msaa, With<OrderIndependentTransparencySettings>>) {
+ for msaa in &cameras {
+ if msaa.samples() > 1 {
+ panic!("MSAA is not supported when using OrderIndependentTransparency");
+ }
+ }
+}
+
+/// Holds the buffers that contain the data of all OIT layers.
+/// We use one big buffer for the entire app. Each camaera will reuse it so it will
+/// always be the size of the biggest OIT enabled camera.
+#[derive(Resource)]
+pub struct OitBuffers {
+ /// The OIT layers containing depth and color for each fragments.
+ /// This is essentially used as a 3d array where xy is the screen coordinate and z is
+ /// the list of fragments rendered with OIT.
+ pub layers: BufferVec<UVec2>,
+ /// Buffer containing the index of the last layer that was written for each fragment.
+ pub layer_ids: BufferVec<i32>,
+ pub layers_count_uniforms: DynamicUniformBuffer<i32>,
+}
+
+impl FromWorld for OitBuffers {
+ fn from_world(world: &mut World) -> Self {
+ let render_device = world.resource::<RenderDevice>();
+ let render_queue = world.resource::<RenderQueue>();
+
+ // initialize buffers with something so there's a valid binding
+
+ let mut layers = BufferVec::new(BufferUsages::COPY_DST | BufferUsages::STORAGE);
+ layers.set_label(Some("oit_layers"));
+ layers.reserve(1, render_device);
+ layers.write_buffer(render_device, render_queue);
+
+ let mut layer_ids = BufferVec::new(BufferUsages::COPY_DST | BufferUsages::STORAGE);
+ layer_ids.set_label(Some("oit_layer_ids"));
+ layer_ids.reserve(1, render_device);
+ layer_ids.write_buffer(render_device, render_queue);
+
+ let mut layers_count_uniforms = DynamicUniformBuffer::default();
+ layers_count_uniforms.set_label(Some("oit_layers_count"));
+
+ Self {
+ layers,
+ layer_ids,
+ layers_count_uniforms,
+ }
+ }
+}
+
+#[derive(Component)]
+pub struct OitLayersCountOffset {
+ pub offset: u32,
+}
+
+/// This creates or resizes the oit buffers for each camera.
+/// It will always create one big buffer that's as big as the biggest buffer needed.
+/// Cameras with smaller viewports or less layers will simply use the big buffer and ignore the rest.
+#[allow(clippy::type_complexity)]
+pub fn prepare_oit_buffers(
+ mut commands: Commands,
+ render_device: Res<RenderDevice>,
+ render_queue: Res<RenderQueue>,
+ cameras: Query<
+ (&ExtractedCamera, &OrderIndependentTransparencySettings),
+ (
+ Changed<ExtractedCamera>,
+ Changed<OrderIndependentTransparencySettings>,
+ ),
+ >,
+ camera_oit_uniforms: Query<(Entity, &OrderIndependentTransparencySettings)>,
+ mut buffers: ResMut<OitBuffers>,
+) {
+ // Get the max buffer size for any OIT enabled camera
+ let mut max_layer_ids_size = usize::MIN;
+ let mut max_layers_size = usize::MIN;
+ for (camera, settings) in &cameras {
+ let Some(size) = camera.physical_target_size else {
+ continue;
+ };
+
+ let layer_count = settings.layer_count as usize;
+ let size = (size.x * size.y) as usize;
+ max_layer_ids_size = max_layer_ids_size.max(size);
+ max_layers_size = max_layers_size.max(size * layer_count);
+ }
+
+ // Create or update the layers buffer based on the max size
+ if buffers.layers.capacity() < max_layers_size {
+ let start = Instant::now();
+ buffers.layers.reserve(max_layers_size, &render_device);
+ let remaining = max_layers_size - buffers.layers.capacity();
+ for _ in 0..remaining {
+ buffers.layers.push(UVec2::ZERO);
+ }
+ buffers.layers.write_buffer(&render_device, &render_queue);
+ trace!(
+ "OIT layers buffer updated in {:.01}ms with total size {} MiB",
+ start.elapsed().as_millis(),
+ buffers.layers.capacity() * size_of::<UVec2>() / 1024 / 1024,
+ );
+ }
+
+ // Create or update the layer_ids buffer based on the max size
+ if buffers.layer_ids.capacity() < max_layer_ids_size {
+ let start = Instant::now();
+ buffers
+ .layer_ids
+ .reserve(max_layer_ids_size, &render_device);
+ let remaining = max_layer_ids_size - buffers.layer_ids.capacity();
+ for _ in 0..remaining {
+ buffers.layer_ids.push(0);
+ }
+ buffers
+ .layer_ids
+ .write_buffer(&render_device, &render_queue);
+ trace!(
+ "OIT layer ids buffer updated in {:.01}ms with total size {} MiB",
+ start.elapsed().as_millis(),
+ buffers.layer_ids.capacity() * size_of::<UVec2>() / 1024 / 1024,
+ );
+ }
+
+ if let Some(mut writer) = buffers.layers_count_uniforms.get_writer(
+ camera_oit_uniforms.iter().len(),
+ &render_device,
+ &render_queue,
+ ) {
+ for (entity, settings) in &camera_oit_uniforms {
+ let offset = writer.write(&(settings.layer_count as i32));
+ commands
+ .entity(entity)
+ .insert(OitLayersCountOffset { offset });
+ }
+ }
+}

crates/bevy_core_pipeline/src/oit/oit_draw.wgsl

@@ -0,0 +1,44 @@
+#define_import_path bevy_core_pipeline::oit
+
+#import bevy_pbr::mesh_view_bindings::{view, oit_layers, oit_layer_ids, oit_layers_count}
+
+#ifdef OIT_ENABLED
+// Add the fragment to the oit buffer
+fn oit_draw(position: vec4f, color: vec4f) -> vec4f {
+ // get the index of the current fragment relative to the screen size
+ let screen_index = i32(floor(position.x) + floor(position.y) * view.viewport.z);
+ // get the size of the buffer.
+ // It's always the size of the screen
+ let buffer_size = i32(view.viewport.z * view.viewport.w);
+
+ // gets the layer index of the current fragment
+ var layer_id = atomicAdd(&oit_layer_ids[screen_index], 1);
+ // exit early if we've reached the maximum amount of fragments per layer
+ if layer_id >= oit_layers_count {
+ // force to store the oit_layers_count to make sure we don't
+ // accidentally increase the index above the maximum value
+ atomicStore(&oit_layer_ids[screen_index], oit_layers_count);
+ // TODO for tail blending we should return the color here
+ discard;
+ }
+
+ // get the layer_index from the screen
+ let layer_index = screen_index + layer_id * buffer_size;
+ let rgb9e5_color = bevy_pbr::rgb9e5::vec3_to_rgb9e5_(color.rgb);
+ let depth_alpha = pack_24bit_depth_8bit_alpha(position.z, color.a);
+ oit_layers[layer_index] = vec2(rgb9e5_color, depth_alpha);
+ discard;
+}
+#endif // OIT_ENABLED
+
+fn pack_24bit_depth_8bit_alpha(depth: f32, alpha: f32) -> u32 {
+ let depth_bits = u32(saturate(depth) * f32(0xFFFFFFu) + 0.5);
+ let alpha_bits = u32(saturate(alpha) * f32(0xFFu) + 0.5);
+ return (depth_bits & 0xFFFFFFu) | ((alpha_bits & 0xFFu) << 24u);
+}
+
+fn unpack_24bit_depth_8bit_alpha(packed: u32) -> vec2<f32> {
+ let depth_bits = packed & 0xFFFFFFu;
+ let alpha_bits = (packed >> 24u) & 0xFFu;
+ return vec2(f32(depth_bits) / f32(0xFFFFFFu), f32(alpha_bits) / f32(0xFFu));
+}