From 7029d5b8f3c1f93d9461948e7feceb2e3bb7d197 Mon Sep 17 00:00:00 2001 From: Hyunseok Yang <yanghyunseok@me.com> Date: Sat, 23 Nov 2024 11:51:57 +0900 Subject: [PATCH] Fix wrong value in Segmentation Camera - Disable PostProcessig in SegmentationRenderer -> cause wrong pixel info - Remove Encode16BitsToRG() and Encode16BitsToGR() in ColorEncoding - Store classId value only in MPB for SegmentationShader - Modify SegmentationShader to handle class id value and encode class id value to 16bits and store R/G value in little-endian order - Modify SegmentationCamera Device - targetColorFormat, GraphicsFormat.R8G8_UNorm -> R8G8B8A8_UNorm --- .../SegmentationRenderer.asset | 2 +- Assets/Resources/Shader/Segmentation.shader | 22 ++++++++++--------- Assets/Scripts/Core/Modules/ColorEncoding.cs | 18 ++------------- .../Scripts/Core/Modules/SegmentationTag.cs | 15 ++++--------- Assets/Scripts/Devices/SegmentationCamera.cs | 15 ++++--------- 5 files changed, 23 insertions(+), 49 deletions(-) diff --git a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset index 5fe7ffbb..eac0c1a0 100644 --- a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset +++ b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset @@ -62,7 +62,7 @@ MonoBehaviour: - {fileID: -5373475574186842855} m_RendererFeatureMap: 19e55a676f946db5 m_UseNativeRenderPass: 0 - postProcessData: {fileID: 11400000, guid: 41439944d30ece34e96484bdb6645b55, type: 2} + postProcessData: {fileID: 0} shaders: blitPS: {fileID: 4800000, guid: c17132b1f77d20942aa75f8429c0f8bc, type: 3} copyDepthPS: {fileID: 4800000, guid: d6dae50ee9e1bfa4db75f19f99355220, type: 3} diff --git a/Assets/Resources/Shader/Segmentation.shader b/Assets/Resources/Shader/Segmentation.shader index b8ca6e2c..73974b48 100644 --- a/Assets/Resources/Shader/Segmentation.shader +++ b/Assets/Resources/Shader/Segmentation.shader @@ -2,9 +2,7 @@ Shader "Sensor/Segmentation" { Properties { - _SegmentationColor ("Segmentation Color", Color) = (1, 1, 1, 1) - _SegmentationClassId ("Segmentation Class ID Value in 16bits", Color) = (0, 0, 0, 1) - [Toggle] _DisableColor ("Disable Color method", int) = 0 + _SegmentationValue ("Segmentation Value", int) = 0 [Toggle] _Hide ("Hide this label", int) = 0 } @@ -24,9 +22,7 @@ Shader "Sensor/Segmentation" #include "Packages/com.unity.render-pipelines.universal/ShaderLibrary/Core.hlsl" CBUFFER_START(UnityPerMaterial) - half4 _SegmentationColor; - half4 _SegmentationClassId; - int _DisableColor; + int _SegmentationValue; int _Hide; CBUFFER_END @@ -38,18 +34,18 @@ Shader "Sensor/Segmentation" HLSLPROGRAM - #pragma target 4.5 + #pragma target 4.6 #pragma vertex vert #pragma fragment frag struct Attributes { - float4 positionOS : POSITION; + half4 positionOS : POSITION; }; struct Varyings { - float4 positionCS : SV_POSITION; + half4 positionCS : SV_POSITION; }; Varyings vert(Attributes i) @@ -65,7 +61,13 @@ Shader "Sensor/Segmentation" half4 segColor = half4(0, 0, 0, 0); if (_Hide == 0) { - segColor = (_DisableColor == 0)? _SegmentationColor : _SegmentationClassId; + // Encode 16Bits To RG + float R = ((_SegmentationValue >> 8) & 0xFF) / 255.0; + float G = (_SegmentationValue & 0xFF) / 255.0; + + // Due to little-endian data, SWAP + segColor.r = G; + segColor.g = R; } return segColor; } diff --git a/Assets/Scripts/Core/Modules/ColorEncoding.cs b/Assets/Scripts/Core/Modules/ColorEncoding.cs index 5285d9ac..833db6d3 100644 --- a/Assets/Scripts/Core/Modules/ColorEncoding.cs +++ b/Assets/Scripts/Core/Modules/ColorEncoding.cs @@ -56,8 +56,8 @@ public static Color EncodeTagAsColor(in string tag) var r = (byte)(hash >> 16); var g = (byte)(hash >> 8); var b = (byte)(hash); - // Debug.Log($"EncodeTagAsColor({tag}): {r} {g} {b}"); - return new Color32(r, g, b, 255); + // Debug.Log($"EncodeTagAsColor({tag}): {r} {g} {b} {a}"); + return new Color32(r, g, b, a); } public static Color EncodeLayerAsColor(in int layer) @@ -84,18 +84,4 @@ public static Color EncodeLayerAsColor(in int layer) return color; } - - public static Color Encode16BitsToRG(in UInt16 value) - { - var classIdR = (byte)((value >> 8) & 0xFF); - var classIdG = (byte)(value & 0xFF); - return new Color32(classIdR, classIdG, 0, 0); - } - - // for little endian order - public static Color Encode16BitsToGR(in UInt16 value) - { - var rgColor = Encode16BitsToRG(value); - return new Color(rgColor.g, rgColor.r, 0, 0); - } } \ No newline at end of file diff --git a/Assets/Scripts/Core/Modules/SegmentationTag.cs b/Assets/Scripts/Core/Modules/SegmentationTag.cs index 5fe26d74..a4a6a19d 100644 --- a/Assets/Scripts/Core/Modules/SegmentationTag.cs +++ b/Assets/Scripts/Core/Modules/SegmentationTag.cs @@ -85,19 +85,12 @@ public void Refresh() break; } - var grayscale = color.grayscale; - - _classId = (UInt16)(grayscale * UInt16.MaxValue); - - var classValue = ColorEncoding.Encode16BitsToGR(_classId); - - mpb.SetColor("_SegmentationColor", color); - mpb.SetColor("_SegmentationClassId", classValue); + _classId = (UInt16)(color.grayscale * UInt16.MaxValue); + + mpb.SetInt("_SegmentationValue", (int)_classId); mpb.SetInt("_Hide", 0); - // Debug.Log(TagName + ": mode=" + Main.SegmentationManager.Mode + - // " color=" + color + - // " calssId=" + classValue.r + " " + classValue.g); + // Debug.Log($"{TagName}: mode={Main.SegmentationManager.Mode} color={color} {_classId}"); // Debug.Log($"{TagName} : {grayscale} > {_classId}"); AllocateMaterialPropertyBlock(mpb); diff --git a/Assets/Scripts/Devices/SegmentationCamera.cs b/Assets/Scripts/Devices/SegmentationCamera.cs index 6a16b543..328ffc5d 100644 --- a/Assets/Scripts/Devices/SegmentationCamera.cs +++ b/Assets/Scripts/Devices/SegmentationCamera.cs @@ -36,23 +36,15 @@ protected override void SetupTexture() } // for Unsigned 16-bit - _targetColorFormat = GraphicsFormat.R8G8_UNorm; + _targetColorFormat = GraphicsFormat.R8G8B8A8_UNorm; _readbackDstFormat = GraphicsFormat.R8G8_UNorm; - + _camImageData = new CameraData.Image(_camParam.image.width, _camParam.image.height, pixelFormat); } protected override void SetupCamera() { - // Debug.Log("Segmenataion Setup Camera"); - if (!_camParam.segmentation_type.Equals("semantic")) - { - Debug.Log("Only support semantic segmentation"); - } - _camSensor.backgroundColor = Color.black; - _camSensor.clearFlags = CameraClearFlags.SolidColor; - _camSensor.allowHDR = false; - _camSensor.allowMSAA = true; + // Debug.Log("Segmenataion Setup"); // Refer to SegmentationRenderer (Universal Renderer Data) _universalCamData.SetRenderer(1); @@ -99,6 +91,7 @@ protected override void ImageProcessing(ref NativeArray<byte> readbackData, in f if (imageData != null) { image.Data = imageData; + // Debug.LogFormat($"{image.Data[0]}|{image.Data[1]}|{image.Data[2]}|{image.Data[3]}"); if (_camParam.save_enabled && _startCameraWork) { var saveName = name + "_" + Time.time;