From b5ee2df361bc72b3f26265add36b900650e731fd Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 14 Feb 2024 16:43:25 +0900 Subject: [PATCH 01/47] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 435e1d05..03409d8d 100644 --- a/README.md +++ b/README.md @@ -165,12 +165,13 @@ In terms of branch, 'main' is release(stable) version, and 'develop' is used for ### If you want to build a project -Please visit here [build guide](https://github.com/lge-ros2/cloisim/wiki/Build-Guide). +Please visit here [build guide](https://github.com/lge-ros2/cloisim/wiki/Build-Guide) in Wiki pages. ## Usage ![multi-robots](https://user-images.githubusercontent.com/21001946/82773215-75572480-9e7c-11ea-85a2-a3838fa1e190.png) +Any problem during launching the simultion, please refer to [Wiki Page/Usage](https://github.com/lge-ros2/cloisim/wiki/Usage)' section. ### Run 'CLOiSim' @@ -218,7 +219,7 @@ websocket service path: ***ws://127.0.0.1:8080/{service-name}*** You can add markers like line, text, box, or sphere point and reset simulation by just sending a request data as a JSON format. -Read [detail guide](https://github.com/lge-ros2/cloisim/wiki/Usage#control-service) +Read [detail guide](https://github.com/lge-ros2/cloisim/wiki/Usage#control-service) in Wiki pages. ### CLOiSim + nav2 From 6d2009280ee5e787384c15ef1d85e33784cecb5e Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Fri, 16 Feb 2024 08:11:45 +0900 Subject: [PATCH 02/47] Upgrade Unity editor version -> 2022.3.20f1 (LTS) --- ProjectSettings/ProjectVersion.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ProjectSettings/ProjectVersion.txt b/ProjectSettings/ProjectVersion.txt index 307ccd35..224decfe 100644 --- a/ProjectSettings/ProjectVersion.txt +++ b/ProjectSettings/ProjectVersion.txt @@ -1,2 +1,2 @@ -m_EditorVersion: 2022.3.19f1 -m_EditorVersionWithRevision: 2022.3.19f1 (244b723c30a6) +m_EditorVersion: 2022.3.20f1 +m_EditorVersionWithRevision: 2022.3.20f1 (61c2feb0970d) From 1c0e1b27377ef52f0be921c890eb92132bc5b883 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Mon, 19 Feb 2024 17:18:04 +0900 Subject: [PATCH 03/47] Member variable renamed in SDF.Parser world Add underscore as a prefix for private member variables --- Assets/Scripts/Main.cs | 38 ++++++++++++------------ Assets/Scripts/Tools/SDF/Helper/Base.cs | 30 +++++++++---------- Assets/Scripts/Tools/SDF/Parser/World.cs | 6 ++-- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Assets/Scripts/Main.cs b/Assets/Scripts/Main.cs index 522f56ee..0e4f9949 100644 --- a/Assets/Scripts/Main.cs +++ b/Assets/Scripts/Main.cs @@ -31,8 +31,8 @@ public class Main : MonoBehaviour private FollowingTargetList followingList = null; private static GameObject _core = null; - private static GameObject propsRoot = null; - private static GameObject worldRoot = null; + private static GameObject _propsRoot = null; + private static GameObject _worldRoot = null; private static GameObject lightsRoot = null; private static GameObject _roadsRoot = null; private static GameObject uiRoot = null; @@ -52,8 +52,8 @@ public class Main : MonoBehaviour private static bool isResetting = false; private static bool resetTriggered = false; - public static GameObject PropsRoot => propsRoot; - public static GameObject WorldRoot => worldRoot; + public static GameObject PropsRoot => _propsRoot; + public static GameObject WorldRoot => _worldRoot; public static GameObject RoadsRoot => _roadsRoot; public static GameObject CoreObject => _core; public static GameObject UIObject => uiRoot; @@ -73,10 +73,10 @@ public class Main : MonoBehaviour private void CleanAllModels() { - foreach (var child in worldRoot.GetComponentsInChildren()) + foreach (var child in _worldRoot.GetComponentsInChildren()) { // skip root gameobject - if (child == null || child.gameObject == null || child.gameObject == worldRoot) + if (child == null || child.gameObject == null || child.gameObject == _worldRoot) { continue; } @@ -107,11 +107,11 @@ private void CleanAllResources() private void ResetRootModelsTransform() { - if (worldRoot != null) + if (_worldRoot != null) { - worldRoot.transform.localRotation = Quaternion.identity; - worldRoot.transform.localPosition = Vector3.zero; - worldRoot.transform.localScale = Vector3.one; + _worldRoot.transform.localRotation = Quaternion.identity; + _worldRoot.transform.localPosition = Vector3.zero; + _worldRoot.transform.localScale = Vector3.one; } } @@ -213,8 +213,8 @@ void Awake() Debug.LogError("Failed to Find 'Core'!!!!"); } - propsRoot = GameObject.Find("Props"); - worldRoot = GameObject.Find("World"); + _propsRoot = GameObject.Find("Props"); + _worldRoot = GameObject.Find("World"); lightsRoot = GameObject.Find("Lights"); _roadsRoot = GameObject.Find("Roads"); uiRoot = GameObject.Find("UI"); @@ -231,9 +231,9 @@ void Awake() cameraControl = mainCamera.GetComponent(); - worldNavMeshBuilder = worldRoot.GetComponent(); + worldNavMeshBuilder = _worldRoot.GetComponent(); - var simWorld = worldRoot.AddComponent(); + var simWorld = _worldRoot.AddComponent(); DeviceHelper.SetGlobalClock(simWorld.GetClock()); Main.bridgeManager = new BridgeManager(); @@ -324,7 +324,7 @@ private void UpdateUIModelList() private string GetClonedModelName(in string modelName) { - var worldTrnasform = worldRoot.transform; + var worldTrnasform = _worldRoot.transform; var numbering = 0; var tmpModelName = modelName; for (var i = 0; i < worldTrnasform.childCount; i++) @@ -349,7 +349,7 @@ private IEnumerator LoadModel(string modelPath, string modelFileName) yield return StartCoroutine(_sdfLoader.StartImport(model)); - var targetObject = worldRoot.transform.Find(model.Name); + var targetObject = _worldRoot.transform.Find(model.Name); var addingModel = uiMainCanvasRoot.GetComponentInChildren(); addingModel.SetAddingModelForDeploy(targetObject); @@ -380,7 +380,7 @@ private IEnumerator LoadWorld() if (_sdfRoot.DoParse(out var world, worldFileName)) { _sdfLoader = new SDF.Import.Loader(); - _sdfLoader.SetRootModels(worldRoot); + _sdfLoader.SetRootModels(_worldRoot); _sdfLoader.SetRootLights(lightsRoot); _sdfLoader.SetRootRoads(_roadsRoot); @@ -441,12 +441,12 @@ public static bool TriggerResetService() void Reset() { - foreach (var helper in worldRoot.GetComponentsInChildren()) + foreach (var helper in _worldRoot.GetComponentsInChildren()) { helper.Reset(); } - foreach (var plugin in worldRoot.GetComponentsInChildren()) + foreach (var plugin in _worldRoot.GetComponentsInChildren()) { plugin.Reset(); } diff --git a/Assets/Scripts/Tools/SDF/Helper/Base.cs b/Assets/Scripts/Tools/SDF/Helper/Base.cs index 4566788f..acce68f0 100644 --- a/Assets/Scripts/Tools/SDF/Helper/Base.cs +++ b/Assets/Scripts/Tools/SDF/Helper/Base.cs @@ -12,16 +12,16 @@ namespace Helper { public class Base : UE.MonoBehaviour { - private PoseControl poseControl = null; + private PoseControl _poseControl = null; - private bool isFirstChild = false; + private bool _isFirstChild = false; - public bool IsFirstChild => isFirstChild; // root model + public bool IsFirstChild => _isFirstChild; // root model protected void Awake() { - isFirstChild = SDF2Unity.IsRootModel(this.gameObject); - poseControl = new PoseControl(this.transform); + _isFirstChild = SDF2Unity.IsRootModel(this.gameObject); + _poseControl = new PoseControl(this.transform); Reset(); } @@ -32,17 +32,17 @@ public void Reset() public void ClearPose() { - if (poseControl != null) + if (_poseControl != null) { - poseControl.Clear(); + _poseControl.Clear(); } } public void SetJointPoseTarget(in float targetAxis1, in float targetAxis2, in int targetFrame = 0) { - if (poseControl != null) + if (_poseControl != null) { - poseControl.SetJointTarget(targetAxis1, targetAxis2, targetFrame); + _poseControl.SetJointTarget(targetAxis1, targetAxis2, targetFrame); } } @@ -53,28 +53,28 @@ public void SetPose(in UE.Pose pose, in int targetFrame = 0) public void SetPose(in UE.Vector3 position, in UE.Quaternion rotation, in int targetFrame = 0) { - if (poseControl != null) + if (_poseControl != null) { - poseControl.Set(position, rotation, targetFrame); + _poseControl.Set(position, rotation, targetFrame); } } public void ResetPose() { - if (poseControl != null) + if (_poseControl != null) { - poseControl.Reset(); + _poseControl.Reset(); } } public UE.Pose GetPose(in int targetFrame = 0) { - return (poseControl != null) ? poseControl.Get(targetFrame) : UE.Pose.identity; + return (_poseControl != null) ? _poseControl.Get(targetFrame) : UE.Pose.identity; } public int GetPoseCount() { - return (poseControl != null) ? poseControl.Count : 0; + return (_poseControl != null) ? _poseControl.Count : 0; } } } diff --git a/Assets/Scripts/Tools/SDF/Parser/World.cs b/Assets/Scripts/Tools/SDF/Parser/World.cs index 5d254411..4e7f0196 100644 --- a/Assets/Scripts/Tools/SDF/Parser/World.cs +++ b/Assets/Scripts/Tools/SDF/Parser/World.cs @@ -222,7 +222,7 @@ public class Distribution private Models models = null; private Actors actors = null; - private List roads_ = new List(); + private List _roads = new List(); public SphericalCoordinates spherical_coordinates = null; @@ -302,7 +302,7 @@ protected override void ParseElements() foreach (XmlNode roadNode in GetNodes("road")) { var road = new Road(roadNode); - roads_.Add(road); + _roads.Add(road); } if (IsValidNode("spherical_coordinates")) @@ -373,7 +373,7 @@ public List GetLights() public List GetRoads() { - return roads_; + return _roads; } } } \ No newline at end of file From cd8fe45f1d8036437b0070e4a913ed211fbcb3ec Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Mon, 19 Feb 2024 17:19:06 +0900 Subject: [PATCH 04/47] Parsing segmentation camera in SDF sensor model Implement segmentaion camera --- Assets/Resources/Materials/Segmentation.mat | 47 ++++++++ .../Resources/Materials/Segmentation.mat.meta | 8 ++ .../SegmentationRenderer.asset | 104 ++++++++++++++++++ .../SegmentationRenderer.asset.meta | 8 ++ .../UniversalRenderPipelineAsset.asset | 1 + Assets/Resources/Shader/Segmentation.shader | 89 +++++++++++++++ .../Resources/Shader/Segmentation.shader.meta | 9 ++ Assets/Scripts/Core/Modules/ColorEncoding.cs | 84 ++++++++++++++ .../Core/Modules/ColorEncoding.cs.meta | 11 ++ Assets/Scripts/Core/ObjectSpawning.cs | 6 + Assets/Scripts/Core/SegmentationManager.cs | 77 +++++++++++++ .../Scripts/Core/SegmentationManager.cs.meta | 11 ++ Assets/Scripts/Devices/Camera.cs | 48 ++++---- Assets/Scripts/Devices/DepthCamera.cs | 23 ++-- Assets/Scripts/Devices/SegmentationCamera.cs | 76 +++++++++++++ .../Devices/SegmentationCamera.cs.meta | 11 ++ .../Tools/SDF/Implement/Implement.Sensor.cs | 38 +++++++ .../Scripts/Tools/SDF/Import/Import.Sensor.cs | 6 + .../Scripts/Tools/SDF/Parser/Sensor.Parse.cs | 5 + Assets/Scripts/Tools/SDF/Parser/Sensor.cs | 3 +- ProjectSettings/GraphicsSettings.asset | 1 + 21 files changed, 634 insertions(+), 32 deletions(-) create mode 100644 Assets/Resources/Materials/Segmentation.mat create mode 100644 Assets/Resources/Materials/Segmentation.mat.meta create mode 100644 Assets/Resources/RenderPipelines/SegmentationRenderer.asset create mode 100644 Assets/Resources/RenderPipelines/SegmentationRenderer.asset.meta create mode 100644 Assets/Resources/Shader/Segmentation.shader create mode 100644 Assets/Resources/Shader/Segmentation.shader.meta create mode 100644 Assets/Scripts/Core/Modules/ColorEncoding.cs create mode 100644 Assets/Scripts/Core/Modules/ColorEncoding.cs.meta create mode 100644 Assets/Scripts/Core/SegmentationManager.cs create mode 100644 Assets/Scripts/Core/SegmentationManager.cs.meta create mode 100644 Assets/Scripts/Devices/SegmentationCamera.cs create mode 100644 Assets/Scripts/Devices/SegmentationCamera.cs.meta diff --git a/Assets/Resources/Materials/Segmentation.mat b/Assets/Resources/Materials/Segmentation.mat new file mode 100644 index 00000000..f52df8c1 --- /dev/null +++ b/Assets/Resources/Materials/Segmentation.mat @@ -0,0 +1,47 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!21 &2100000 +Material: + serializedVersion: 8 + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_Name: Segmentation + m_Shader: {fileID: 4800000, guid: 419a697b1455619dea4cc9d729b3d0c8, type: 3} + m_Parent: {fileID: 0} + m_ModifiedSerializedProperties: 0 + m_ValidKeywords: [] + m_InvalidKeywords: [] + m_LightmapFlags: 4 + m_EnableInstancingVariants: 0 + m_DoubleSidedGI: 0 + m_CustomRenderQueue: -1 + stringTagMap: {} + disabledShaderPasses: [] + m_LockedProperties: + m_SavedProperties: + serializedVersion: 3 + m_TexEnvs: [] + m_Ints: [] + m_Floats: + - _OutputMode: -1 + m_Colors: + - _SegmentationIdColor: {r: 0.29999998, g: 0.5, b: 1, a: 1} + - _SegmentationLayerColor: {r: 0.5, g: 0.9, b: 1, a: 1} + - _SegmentationNameColor: {r: 0.39999998, g: 0.7, b: 1, a: 1} + - _UnsupportedColor: {r: 0.09999997, g: 0.09999997, b: 0.09999997, a: 1} + m_BuildTextureStacks: [] +--- !u!114 &6202412140821847777 +MonoBehaviour: + m_ObjectHideFlags: 11 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d0353a89b1f911e48b9e16bdc9f2e058, type: 3} + m_Name: + m_EditorClassIdentifier: + version: 7 diff --git a/Assets/Resources/Materials/Segmentation.mat.meta b/Assets/Resources/Materials/Segmentation.mat.meta new file mode 100644 index 00000000..73cce21c --- /dev/null +++ b/Assets/Resources/Materials/Segmentation.mat.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 748cb5dd9e7f8a9f7b5d0712426cd62c +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 2100000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset new file mode 100644 index 00000000..b4f27546 --- /dev/null +++ b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset @@ -0,0 +1,104 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!114 &-5373475574186842855 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 6b3d386ba5cd94485973aee1479b272e, type: 3} + m_Name: RenderObjects + m_EditorClassIdentifier: + m_Active: 1 + settings: + passTag: RenderObjects + Event: 250 + filterSettings: + RenderQueueType: 0 + LayerMask: + serializedVersion: 2 + m_Bits: 9 + PassNames: [] + overrideMaterial: {fileID: 2100000, guid: 748cb5dd9e7f8a9f7b5d0712426cd62c, type: 2} + overrideMaterialPassIndex: 0 + overrideShader: {fileID: 4800000, guid: 419a697b1455619dea4cc9d729b3d0c8, type: 3} + overrideShaderPassIndex: 0 + overrideMode: 1 + overrideDepthState: 0 + depthCompareFunction: 5 + enableWrite: 0 + stencilSettings: + overrideStencilState: 0 + stencilReference: 5 + stencilCompareFunction: 5 + passOperation: 0 + failOperation: 1 + zFailOperation: 1 + cameraSettings: + overrideCamera: 0 + restoreCamera: 1 + offset: {x: 0, y: 0, z: 0, w: 0} + cameraFieldOfView: 25 +--- !u!114 &11400000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: de640fe3d0db1804a85f9fc8f5cadab6, type: 3} + m_Name: SegmentationRenderer + m_EditorClassIdentifier: + debugShaders: + debugReplacementPS: {fileID: 4800000, guid: cf852408f2e174538bcd9b7fda1c5ae7, + type: 3} + hdrDebugViewPS: {fileID: 4800000, guid: 573620ae32aec764abd4d728906d2587, type: 3} + m_RendererFeatures: + - {fileID: -5373475574186842855} + m_RendererFeatureMap: 19e55a676f946db5 + m_UseNativeRenderPass: 0 + postProcessData: {fileID: 11400000, guid: 41439944d30ece34e96484bdb6645b55, type: 2} + shaders: + blitPS: {fileID: 4800000, guid: c17132b1f77d20942aa75f8429c0f8bc, type: 3} + copyDepthPS: {fileID: 4800000, guid: d6dae50ee9e1bfa4db75f19f99355220, type: 3} + screenSpaceShadowPS: {fileID: 0} + samplingPS: {fileID: 4800000, guid: 04c410c9937594faa893a11dceb85f7e, type: 3} + stencilDeferredPS: {fileID: 4800000, guid: e9155b26e1bc55942a41e518703fe304, type: 3} + fallbackErrorPS: {fileID: 4800000, guid: e6e9a19c3678ded42a3bc431ebef7dbd, type: 3} + fallbackLoadingPS: {fileID: 4800000, guid: 7f888aff2ac86494babad1c2c5daeee2, type: 3} + materialErrorPS: {fileID: 4800000, guid: 5fd9a8feb75a4b5894c241777f519d4e, type: 3} + coreBlitPS: {fileID: 4800000, guid: 93446b5c5339d4f00b85c159e1159b7c, type: 3} + coreBlitColorAndDepthPS: {fileID: 4800000, guid: d104b2fc1ca6445babb8e90b0758136b, + type: 3} + blitHDROverlay: {fileID: 4800000, guid: a89bee29cffa951418fc1e2da94d1959, type: 3} + cameraMotionVector: {fileID: 4800000, guid: c56b7e0d4c7cb484e959caeeedae9bbf, + type: 3} + objectMotionVector: {fileID: 4800000, guid: 7b3ede40266cd49a395def176e1bc486, + type: 3} + dataDrivenLensFlare: {fileID: 4800000, guid: 6cda457ac28612740adb23da5d39ea92, + type: 3} + m_AssetVersion: 2 + m_OpaqueLayerMask: + serializedVersion: 2 + m_Bits: 9 + m_TransparentLayerMask: + serializedVersion: 2 + m_Bits: 0 + m_DefaultStencilState: + overrideStencilState: 0 + stencilReference: 0 + stencilCompareFunction: 8 + passOperation: 2 + failOperation: 0 + zFailOperation: 0 + m_ShadowTransparentReceive: 0 + m_RenderingMode: 1 + m_DepthPrimingMode: 0 + m_CopyDepthMode: 0 + m_AccurateGbufferNormals: 0 + m_IntermediateTextureMode: 1 diff --git a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset.meta b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset.meta new file mode 100644 index 00000000..ff87f66a --- /dev/null +++ b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e7e5daae73c962ecbbb5651a07d8ffe4 +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Resources/RenderPipelines/UniversalRenderPipelineAsset.asset b/Assets/Resources/RenderPipelines/UniversalRenderPipelineAsset.asset index ad6be46a..c2b47054 100644 --- a/Assets/Resources/RenderPipelines/UniversalRenderPipelineAsset.asset +++ b/Assets/Resources/RenderPipelines/UniversalRenderPipelineAsset.asset @@ -18,6 +18,7 @@ MonoBehaviour: m_RendererData: {fileID: 0} m_RendererDataList: - {fileID: 11400000, guid: a079301ff0f3b5c48ac29edc3797d4c7, type: 2} + - {fileID: 11400000, guid: e7e5daae73c962ecbbb5651a07d8ffe4, type: 2} m_DefaultRendererIndex: 0 m_RequireDepthTexture: 0 m_RequireOpaqueTexture: 0 diff --git a/Assets/Resources/Shader/Segmentation.shader b/Assets/Resources/Shader/Segmentation.shader new file mode 100644 index 00000000..a048b315 --- /dev/null +++ b/Assets/Resources/Shader/Segmentation.shader @@ -0,0 +1,89 @@ +Shader "Sensor/Segmentation" +{ + Properties + { + _SegmentationIdColor ("Id Color", Color) = (0.3, 0.5, 1, 1) + _SegmentationNameColor ("Name Color", Color) = (0.4, 0.7, 1, 1) + _SegmentationLayerColor ("Layer Color", Color) = (0.5, 0.9, 1, 1) + _UnsupportedColor ("Unsupported Color", Color) = (0.1, 0.1, 0.1, 1) + _OutputMode ("Output Mode", int) = -1 + } + + SubShader + { + Tags { + "RenderType" = "Opaque" + } + + HLSLINCLUDE + ENDHLSL + + Pass + { + Name "Segmentation" + + HLSLPROGRAM + #include "Packages/com.unity.render-pipelines.universal/ShaderLibrary/Core.hlsl" + + #pragma vertex vert + #pragma fragment frag + + CBUFFER_START(UnityPerMaterial) + half4 _SegmentationIdColor; + half4 _SegmentationNameColor; + half4 _SegmentationLayerColor; + half4 _UnsupportedColor; + int _OutputMode; + CBUFFER_END + + + half4 replacement_output() + { + /* + public enum ReplacementMode + { + ObjectId = 0, + ObjectName = 1, + ObjectLayer = 2 + }; + */ + switch (_OutputMode) + { + case 0: + return _SegmentationIdColor; + case 1: + return _SegmentationNameColor; + case 2: + return _SegmentationLayerColor; + } + return _UnsupportedColor; + } + + struct Attributes + { + half4 positionOS : POSITION; + }; + + struct Varyings + { + half4 positionCS : SV_POSITION; + half4 color : COLOR; + }; + + Varyings vert(Attributes input) + { + Varyings output = (Varyings)0; + VertexPositionInputs vertexInput = GetVertexPositionInputs(input.positionOS.xyz); + output.positionCS = vertexInput.positionCS; + output.color = replacement_output(); + return output; + } + + half4 frag(Varyings i) : SV_Target + { + return replacement_output(); + } + ENDHLSL + } + } +} \ No newline at end of file diff --git a/Assets/Resources/Shader/Segmentation.shader.meta b/Assets/Resources/Shader/Segmentation.shader.meta new file mode 100644 index 00000000..43ac5826 --- /dev/null +++ b/Assets/Resources/Shader/Segmentation.shader.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 419a697b1455619dea4cc9d729b3d0c8 +ShaderImporter: + externalObjects: {} + defaultTextures: [] + nonModifiableTextures: [] + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/Core/Modules/ColorEncoding.cs b/Assets/Scripts/Core/Modules/ColorEncoding.cs new file mode 100644 index 00000000..2b4dac53 --- /dev/null +++ b/Assets/Scripts/Core/Modules/ColorEncoding.cs @@ -0,0 +1,84 @@ +using UnityEngine; + +public class ColorEncoding +{ + // public static byte ReverseBits(byte value) + // { + // return (byte)((value * 0x0202020202 & 0x010884422010) % 1023); + // } + + public static int SparsifyBits(byte value, in int sparse) + { + int retVal = 0; + for (int bits = 0; bits < 8; bits++, value >>= 1) + { + retVal |= (value & 1); + retVal <<= sparse; + } + return retVal >> sparse; + } + + public static Color EncodeIDAsColor(in int instanceId) + { + var uid = instanceId * 2; + if (uid < 0) + uid = -uid + 1; + + var sid = + (SparsifyBits((byte)(uid >> 16), 3) << 2) | + (SparsifyBits((byte)(uid >> 8), 3) << 1) | + SparsifyBits((byte)(uid), 3); + //Debug.Log(uid + " >>> " + System.Convert.ToString(sid, 2).PadLeft(24, '0')); + + var r = (byte)(sid >> 8); + var g = (byte)(sid >> 16); + var b = (byte)(sid); + + //Debug.Log(r + " " + g + " " + b); + return new Color32(r, g, b, 255); + } + + public static Color EncodeNameAsColor(in string name) + { + return EncodeTagAsColor(name); + } + + public static Color EncodeTagAsColor(in string tag) + { + var hash = tag.GetHashCode(); + var a = (byte)(hash >> 24); + var r = (byte)(hash >> 16); + var g = (byte)(hash >> 8); + var b = (byte)(hash); + return new Color32(r, g, b, 255); + } + + public static Color EncodeLayerAsColor(in int layer) + { + // Following value must be in the range (0.5 .. 1.0) + // in order to avoid color overlaps when using 'divider' in this func + var z = .7f; + + // First 8 layers are Unity Builtin layers + // Unity supports up to 32 layers in total + + // Lets create palette of unique 16 colors + var uniqueColors = new Color[] { + new Color(1,1,1,1), new Color(z,z,z,1), // 0 + new Color(1,1,z,1), new Color(1,z,1,1), new Color(z,1,1,1), // + new Color(1,z,0,1), new Color(z,0,1,1), new Color(0,1,z,1), // 7 + + new Color(1,0,0,1), new Color(0,1,0,1), new Color(0,0,1,1), // 8 + new Color(1,1,0,1), new Color(1,0,1,1), new Color(0,1,1,1), // + new Color(1,z,z,1), new Color(z,1,z,1) // 15 + }; + + // Create as many colors as necessary by using base 16 color palette + // To create more than 16 - will simply adjust brightness with 'divider' + var color = uniqueColors[layer % uniqueColors.Length]; + var divider = 1.0f + Mathf.Floor(layer / uniqueColors.Length); + color /= divider; + + return color; + } +} diff --git a/Assets/Scripts/Core/Modules/ColorEncoding.cs.meta b/Assets/Scripts/Core/Modules/ColorEncoding.cs.meta new file mode 100644 index 00000000..6de3eaf2 --- /dev/null +++ b/Assets/Scripts/Core/Modules/ColorEncoding.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b27dae911b41a0538a9460e05376f313 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/Core/ObjectSpawning.cs b/Assets/Scripts/Core/ObjectSpawning.cs index 195dd461..cbaef16a 100644 --- a/Assets/Scripts/Core/ObjectSpawning.cs +++ b/Assets/Scripts/Core/ObjectSpawning.cs @@ -1,3 +1,9 @@ +/* + * Copyright (c) 2021 LG Electronics Inc. + * + * SPDX-License-Identifier: MIT + */ + using System.Collections; using System.Collections.Generic; using UnityEngine.AI; diff --git a/Assets/Scripts/Core/SegmentationManager.cs b/Assets/Scripts/Core/SegmentationManager.cs new file mode 100644 index 00000000..a0e5fd66 --- /dev/null +++ b/Assets/Scripts/Core/SegmentationManager.cs @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024 LG Electronics Inc. + * + * SPDX-License-Identifier: MIT + */ + +using UnityEngine; +// using UnityEngine.Rendering.Universal; +// using UnityEngine.Experimental.Rendering; +// using Unity.Collections; +using System.Collections.Generic; + +public class SegmentationManager : MonoBehaviour +{ + private enum ReplacementMode + { + ObjectId = 0, + ObjectName = 1, + LayerId = 2 + }; + + private static readonly ReplacementMode ReplaceMode = ReplacementMode.ObjectName; + private static Material SegmentationMaterial = null; + + + void OnEnable() + { + if (SegmentationMaterial == null) + { + SegmentationMaterial = Resources.Load("Materials/Segmentation"); + } + + if (SegmentationMaterial != null) + { + SegmentationMaterial.SetInt("_OutputMode", (int)ReplaceMode); + } + } + + void OnDisable() + { + if (SegmentationMaterial != null) + { + SegmentationMaterial.SetInt("_OutputMode", (int)-1); + } + } + + + + public void OnSceneChanged() + { + var renderersWorld = Main.WorldRoot.GetComponentsInChildren(); + var renderersProps = Main.PropsRoot.GetComponentsInChildren(); + + var combineRenderers = new List(); + combineRenderers.AddRange(renderersWorld); + combineRenderers.AddRange(renderersProps); + + var mpb = new MaterialPropertyBlock(); + foreach (var renderer in combineRenderers.ToArray()) + { + var go = renderer.gameObject; + + var id = go.GetInstanceID(); + var layer = go.layer; + var name = go.name; + + mpb.SetColor("_SegmentationIdColor", ColorEncoding.EncodeIDAsColor(id)); + mpb.SetColor("_SegmentationNameColor", ColorEncoding.EncodeNameAsColor(name)); + mpb.SetColor("_SegmentationLayerColor", ColorEncoding.EncodeLayerAsColor(layer)); + // Debug.Log(id + " " + name + " " + mpb.GetColor("_SegmentationIdColor")); + // Debug.Log(id + " " + name + " " + mpb.GetColor("_SegmentationNameColor")); + // Debug.Log(id + " " + name + " " + mpb.GetColor("_SegmentationLayerColor")); + + renderer.SetPropertyBlock(mpb); + } + } +} \ No newline at end of file diff --git a/Assets/Scripts/Core/SegmentationManager.cs.meta b/Assets/Scripts/Core/SegmentationManager.cs.meta new file mode 100644 index 00000000..b0f2f8de --- /dev/null +++ b/Assets/Scripts/Core/SegmentationManager.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 71b3c95390c30a9628935696c38169e9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/Devices/Camera.cs b/Assets/Scripts/Devices/Camera.cs index 9ba74d9b..0ff0427d 100644 --- a/Assets/Scripts/Devices/Camera.cs +++ b/Assets/Scripts/Devices/Camera.cs @@ -30,11 +30,11 @@ public class Camera : Device protected UnityEngine.Camera camSensor = null; protected UniversalAdditionalCameraData _universalCamData = null; - protected string targetRTname; - protected GraphicsFormat targetColorFormat; - protected GraphicsFormat readbackDstFormat; + protected string _targetRTname; + protected GraphicsFormat _targetColorFormat; + protected GraphicsFormat _readbackDstFormat; - private CameraData.Image camImageData; + protected CameraData.Image _camImageData; private List _readbackList = new List(); public Noise noise = null; protected bool _startCameraWork = false; @@ -104,24 +104,24 @@ protected virtual void SetupTexture() _universalCamData.renderShadows = true; // Debug.Log("This is not a Depth Camera!"); - targetRTname = "CameraColorTexture"; + _targetRTname = "CameraColorTexture"; var pixelFormat = CameraData.GetPixelFormat(camParameter.image.format); switch (pixelFormat) { case CameraData.PixelFormat.L_INT8: - targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; - readbackDstFormat = GraphicsFormat.R8_SRGB; + _targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; + _readbackDstFormat = GraphicsFormat.R8_SRGB; break; case CameraData.PixelFormat.RGB_INT8: default: - targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; - readbackDstFormat = GraphicsFormat.R8G8B8_SRGB; + _targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; + _readbackDstFormat = GraphicsFormat.R8G8B8_SRGB; break; } - camImageData = new CameraData.Image(camParameter.image.width, camParameter.image.height, pixelFormat); + _camImageData = new CameraData.Image(camParameter.image.width, camParameter.image.height, pixelFormat); } protected override void InitializeMessages() @@ -167,6 +167,19 @@ protected override void SetupMessages() } } + private void OnEnable() + { + RenderPipelineManager.beginCameraRendering += OnBeginCameraRendering; + RenderPipelineManager.endCameraRendering += OnEndCameraRendering; + } + + private void OnDisable() + { + RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering; + RenderPipelineManager.endCameraRendering -= OnEndCameraRendering; + } + + // protected virtual void SetupCamera() private void SetupCamera() { camSensor.ResetWorldToCameraMatrix(); @@ -188,7 +201,7 @@ private void SetupCamera() height: camParameter.image.height, slices: 1, depthBufferBits: DepthBits.None, - colorFormat: targetColorFormat, + colorFormat: _targetColorFormat, filterMode: FilterMode.Bilinear, wrapMode: TextureWrapMode.Clamp, dimension: TextureDimension.Tex2D, @@ -202,7 +215,7 @@ private void SetupCamera() bindTextureMS: false, useDynamicScale: true, memoryless: RenderTextureMemoryless.None, - name: targetRTname); + name: _targetRTname); camSensor.targetTexture = _rtHandle.rt; @@ -225,9 +238,6 @@ private void SetupCamera() _universalCamData.cameraStack.Clear(); camSensor.enabled = false; - RenderPipelineManager.beginCameraRendering += OnBeginCameraRendering; - RenderPipelineManager.endCameraRendering += OnEndCameraRendering; - // camSensor.hideFlags |= HideFlags.NotEditable; } @@ -263,7 +273,7 @@ private void CameraWorker() { camSensor.Render(); - var readbackRequest = AsyncGPUReadback.Request(camSensor.targetTexture, 0, readbackDstFormat, OnCompleteAsyncReadback); + var readbackRequest = AsyncGPUReadback.Request(camSensor.targetTexture, 0, _readbackDstFormat, OnCompleteAsyncReadback); lock (_readbackList) { @@ -308,16 +318,16 @@ protected override void GenerateMessage() protected virtual void ImageProcessing(ref NativeArray readbackData) { var image = imageStamped.Image; - camImageData.SetTextureBufferData(readbackData); + _camImageData.SetTextureBufferData(readbackData); - var imageData = camImageData.GetImageData(image.Data.Length); + var imageData = _camImageData.GetImageData(image.Data.Length); if (imageData != null) { image.Data = imageData; if (camParameter.save_enabled && _startCameraWork) { var saveName = name + "_" + Time.time; - camImageData.SaveRawImageData(camParameter.save_path, saveName); + _camImageData.SaveRawImageData(camParameter.save_path, saveName); // Debug.LogFormat("{0}|{1} captured", camParameter.save_path, saveName); } } diff --git a/Assets/Scripts/Devices/DepthCamera.cs b/Assets/Scripts/Devices/DepthCamera.cs index 1d834d2a..051c8c12 100644 --- a/Assets/Scripts/Devices/DepthCamera.cs +++ b/Assets/Scripts/Devices/DepthCamera.cs @@ -14,6 +14,7 @@ namespace SensorDevices { + [RequireComponent(typeof(UnityEngine.Camera))] public class DepthCamera : Camera { #region "For Compute Shader" @@ -38,7 +39,6 @@ public class DepthCamera : Camera private byte[] _computedBufferOutput; private const uint OutputUnitSize = 4; private int _computedBufferOutputUnitLength; - private CameraData.Image _depthCamImage; private Texture2D _textureForCapture; @@ -120,9 +120,9 @@ protected override void SetupTexture() _universalCamData.requiresDepthTexture = true; _universalCamData.renderShadows = false; - targetRTname = "CameraDepthTexture"; - targetColorFormat = GraphicsFormat.R8G8B8A8_UNorm; - readbackDstFormat = GraphicsFormat.R8G8B8A8_UNorm; + _targetRTname = "CameraDepthTexture"; + _targetColorFormat = GraphicsFormat.R8G8B8A8_UNorm; + _readbackDstFormat = GraphicsFormat.R8G8B8A8_UNorm; var cb = new CommandBuffer(); cb.name = "CommandBufferForDepthShading"; @@ -150,14 +150,11 @@ protected override void SetupTexture() graphicFormat = GraphicsFormat.R16_UNorm; break; } - _textureForCapture = new Texture2D(width, height, graphicFormat, 0, TextureCreationFlags.None); _depthCamBuffer = new DepthData.CamBuffer(width, height); _computedBufferOutputUnitLength = width * height; _computedBufferOutput = new byte[_computedBufferOutputUnitLength * OutputUnitSize]; - _depthCamImage = new CameraData.Image(width, height, format); - if (computeShader != null) { _imageDepth = CameraData.GetImageDepth(format); @@ -171,6 +168,9 @@ protected override void SetupTexture() _threadGroupX = width / ThreadGroupsX; _threadGroupY = height / ThreadGroupsY; + + _textureForCapture = new Texture2D(width, height, graphicFormat, 0, TextureCreationFlags.None); + _textureForCapture.filterMode = FilterMode.Point; } protected override void ImageProcessing(ref NativeArray readbackData) @@ -205,20 +205,19 @@ protected override void ImageProcessing(ref NativeArray readbackData) if (camParameter.save_enabled && _startCameraWork) { - SaveRawImageData(); + var saveName = name + "_" + Time.time; + SaveRawImageData(camParameter.save_path, saveName); } } _depthCamBuffer.Deallocate(); } - private void SaveRawImageData() + private void SaveRawImageData(in string path, in string name) { - var saveName = name + "_" + Time.time; _textureForCapture.SetPixelData(imageStamped.Image.Data, 0); - _textureForCapture.filterMode = FilterMode.Point; _textureForCapture.Apply(); var bytes = _textureForCapture.EncodeToJPG(); - var fileName = string.Format("{0}/{1}.jpg", camParameter.save_path, saveName); + var fileName = string.Format("{0}/{1}.jpg", path, name); System.IO.File.WriteAllBytes(fileName, bytes); // Debug.LogFormat("{0}|{1} captured", camParameter.save_path, saveName); } diff --git a/Assets/Scripts/Devices/SegmentationCamera.cs b/Assets/Scripts/Devices/SegmentationCamera.cs new file mode 100644 index 00000000..6e6e88ed --- /dev/null +++ b/Assets/Scripts/Devices/SegmentationCamera.cs @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024 LG Electronics Inc. + * + * SPDX-License-Identifier: MIT + */ + +using UnityEngine; +using UnityEngine.Rendering.Universal; +using UnityEngine.Experimental.Rendering; +using Unity.Collections; + +namespace SensorDevices +{ + [RequireComponent(typeof(UnityEngine.Camera))] + public class SegmentationCamera : Camera + { + protected override void SetupTexture() + { + if (!camParameter.segmentation_type.Equals("semantic")) + { + Debug.Log("Only support semantic segmentation"); + } + + camSensor.backgroundColor = Color.black; + camSensor.clearFlags = CameraClearFlags.SolidColor; + camSensor.depthTextureMode = DepthTextureMode.None; + camSensor.allowHDR = false; + camSensor.allowMSAA = true; + + // Refer to SegmentationRenderer (Universal Renderer Data) + _universalCamData.SetRenderer(1); + _universalCamData.renderPostProcessing = true; + _universalCamData.requiresColorOption = CameraOverrideOption.On; + _universalCamData.requiresDepthOption = CameraOverrideOption.Off; + _universalCamData.requiresColorTexture = true; + _universalCamData.requiresDepthTexture = false; + _universalCamData.renderShadows = true; + + _targetRTname = "SegmentationTexture"; + + var pixelFormat = CameraData.GetPixelFormat(camParameter.image.format); + if (pixelFormat != CameraData.PixelFormat.L_INT16) + { + Debug.Log("Only support INT16 format"); + } + + _targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; + _readbackDstFormat = GraphicsFormat.R16_UNorm; + // _readbackDstFormat = GraphicsFormat.R16_SFloat; + + _camImageData = new CameraData.Image(camParameter.image.width, camParameter.image.height, pixelFormat); + } + + protected override void ImageProcessing(ref NativeArray readbackData) + { + var image = imageStamped.Image; + _camImageData.SetTextureBufferData(readbackData); + + var imageData = _camImageData.GetImageData(image.Data.Length); + if (imageData != null) + { + image.Data = imageData; + if (camParameter.save_enabled && _startCameraWork) + { + var saveName = name + "_" + Time.time; + _camImageData.SaveRawImageData(camParameter.save_path, saveName); + // Debug.LogFormat("{0}|{1} captured", camParameter.save_path, saveName); + } + } + else + { + Debug.LogWarningFormat("{0}: Failed to get image Data", name); + } + } + } +} \ No newline at end of file diff --git a/Assets/Scripts/Devices/SegmentationCamera.cs.meta b/Assets/Scripts/Devices/SegmentationCamera.cs.meta new file mode 100644 index 00000000..148bce41 --- /dev/null +++ b/Assets/Scripts/Devices/SegmentationCamera.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b55d9fe82ae5d49d6913608370365457 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/Tools/SDF/Implement/Implement.Sensor.cs b/Assets/Scripts/Tools/SDF/Implement/Implement.Sensor.cs index 04bd0739..44ea1da6 100644 --- a/Assets/Scripts/Tools/SDF/Implement/Implement.Sensor.cs +++ b/Assets/Scripts/Tools/SDF/Implement/Implement.Sensor.cs @@ -97,6 +97,44 @@ public static Device AddCamera(in SDF.Camera element, in GameObject targetObject return camera; } + public static Device AddSegmentaionCamera(in SDF.Camera element, in GameObject targetObject) + { + var newSensorObject = new GameObject(); + AttachSensor(newSensorObject, targetObject, element.Pose); + + var segmentationCamera = newSensorObject.AddComponent(); + segmentationCamera.DeviceName = GetFrameName(newSensorObject); + + switch (element.image.format) + { + case "L16": + case "L_UINT16": + // Debug.Log("Supporting data type for Depth camera"); + break; + + default: + if (element.image.format.Equals(string.Empty)) + { + Debug.LogWarningFormat("'L16' will be set for Depth camera({0})'s image format", element.name); + } + else + { + Debug.LogWarningFormat("Not supporting data type({0}) for Depth camera", element.image.format); + } + element.image.format = "L16"; + break; + } + + segmentationCamera.SetCamParameter(element); + + if (element.noise != null) + { + segmentationCamera.noise = new SensorDevices.Noise(element.noise, element.type); + } + + return segmentationCamera; + } + public static Device AddDepthCamera(in SDF.Camera element, in GameObject targetObject) { var newSensorObject = new GameObject(); diff --git a/Assets/Scripts/Tools/SDF/Import/Import.Sensor.cs b/Assets/Scripts/Tools/SDF/Import/Import.Sensor.cs index 29220ff6..0181b38a 100644 --- a/Assets/Scripts/Tools/SDF/Import/Import.Sensor.cs +++ b/Assets/Scripts/Tools/SDF/Import/Import.Sensor.cs @@ -55,6 +55,12 @@ protected override System.Object ImportSensor(in SDF.Sensor sensor, in System.Ob device = Implement.Sensor.AddCamera(camera, targetObject); break; + case "segmentation_camera": + case "segmentation": + var segmentationCamera = sensor.GetSensor() as SDF.Camera; + device = Implement.Sensor.AddSegmentaionCamera(segmentationCamera, targetObject); + break; + case "rgbd_camera": case "rgbd": case "multicamera": diff --git a/Assets/Scripts/Tools/SDF/Parser/Sensor.Parse.cs b/Assets/Scripts/Tools/SDF/Parser/Sensor.Parse.cs index 6fa09802..dfeb2071 100644 --- a/Assets/Scripts/Tools/SDF/Parser/Sensor.Parse.cs +++ b/Assets/Scripts/Tools/SDF/Parser/Sensor.Parse.cs @@ -114,6 +114,11 @@ private Camera ParseCamera(in int index = 1) } } + if (IsValidNode(cameraElement + "/segmentation_type")) + { + camera.segmentation_type = GetValue(cameraElement + "/segmentation_type", "semantic"); + } + if (IsValidNode(cameraElement + "/noise")) { camera.noise = new Noise("gaussian"); diff --git a/Assets/Scripts/Tools/SDF/Parser/Sensor.cs b/Assets/Scripts/Tools/SDF/Parser/Sensor.cs index af9c2f2c..9cab77b2 100644 --- a/Assets/Scripts/Tools/SDF/Parser/Sensor.cs +++ b/Assets/Scripts/Tools/SDF/Parser/Sensor.cs @@ -109,6 +109,8 @@ protected override void ParseElements() case "camera": case "depth_camera": case "depth": + case "segmentation_camera": + case "segmentation": if (IsValidNode("camera")) { sensor = ParseCamera(); @@ -140,7 +142,6 @@ protected override void ParseElements() case "force_torque": case "logical_camera": case "boundingbox_camera": - case "segmentation_camera": case "magnetometer": case "rfid": case "rfidtag": diff --git a/ProjectSettings/GraphicsSettings.asset b/ProjectSettings/GraphicsSettings.asset index 1ffd1b13..0959380e 100644 --- a/ProjectSettings/GraphicsSettings.asset +++ b/ProjectSettings/GraphicsSettings.asset @@ -29,6 +29,7 @@ GraphicsSettings: m_AlwaysIncludedShaders: - {fileID: 10770, guid: 0000000000000000f000000000000000, type: 0} - {fileID: 4800000, guid: 580493fad5661e266b759ad2bd7f2c54, type: 3} + - {fileID: 4800000, guid: f717059fba45966649cc75584b98b436, type: 3} m_PreloadedShaders: - {fileID: 20000000, guid: 444c3b2060d68fa04a081d66c72c2066, type: 2} m_PreloadShadersBatchTimeLimit: -1 From 52d630d6a15e660b15b7a1d6e9ae987713fb8d9a Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 21 Feb 2024 15:03:12 +0900 Subject: [PATCH 05/47] Update .vscode settings --- .vscode/settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index c9f71335..94a9211a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,7 +4,10 @@ "**/*.meta": true, "Library": true, "Temp": true, - "UserSettings": true + "UserSettings": true, + "obj": true, + "**/TextMesh Pro": true, + "**/VSCode": true }, "editor.detectIndentation": false, "editor.indentSize": "tabSize", From 417adbba0257e4709143c2e9805df6adda5d5bba Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 21 Feb 2024 15:03:29 +0900 Subject: [PATCH 06/47] Apply RequireComponent --- Assets/Scenes/MainScene.unity | 2 +- Assets/Scripts/Main.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Assets/Scenes/MainScene.unity b/Assets/Scenes/MainScene.unity index 797223aa..4b6e15c0 100644 --- a/Assets/Scenes/MainScene.unity +++ b/Assets/Scenes/MainScene.unity @@ -5207,7 +5207,7 @@ Camera: m_Enabled: 1 serializedVersion: 2 m_ClearFlags: 1 - m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0} + m_BackGroundColor: {r: 0, g: 0, b: 0, a: 1} m_projectionMatrixMode: 2 m_GateFitMode: 2 m_FOVAxisMode: 0 diff --git a/Assets/Scripts/Main.cs b/Assets/Scripts/Main.cs index 0e4f9949..c27f7ce5 100644 --- a/Assets/Scripts/Main.cs +++ b/Assets/Scripts/Main.cs @@ -13,6 +13,7 @@ using UnityEngine.UI; [DefaultExecutionOrder(30)] +[RequireComponent(typeof(ObjectSpawning), typeof(SegmentationManager))] public class Main : MonoBehaviour { [Header("Block Loading SDF")] @@ -243,8 +244,6 @@ void Awake() DeviceHelper.SetGlobalSphericalCoordinates(sphericalCoordinates); SensorDevices.DepthCamera.LoadComputeShader(); - - gameObject.AddComponent(); } void Start() @@ -503,5 +502,7 @@ void OnDestroy() { Assimp.Unmanaged.AssimpLibrary.Instance.FreeLibrary(); } + + } } \ No newline at end of file From 4fd007f1b61648badda78dc84647c2ec68dbdeed Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 21 Feb 2024 15:17:20 +0900 Subject: [PATCH 07/47] Rearrange Folder structures for Resoruces --- Assets/Resources/PhysicsMaterials.meta | 8 ++++++++ .../DefaultZero.physicMaterial | 0 .../DefaultZero.physicMaterial.meta | 0 .../Props.physicMaterial | 0 .../Props.physicMaterial.meta | 0 .../temp_material.physicMaterial | 0 .../temp_material.physicMaterial.meta | 0 Assets/Resources/{Materials => }/icons.meta | 0 .../icons/android-chrome-192x192.png | Bin .../icons/android-chrome-192x192.png.meta | 0 .../icons/android-chrome-512x512.png | Bin .../icons/android-chrome-512x512.png.meta | 0 .../{Materials => }/icons/apple-touch-icon.png | Bin .../{Materials => }/icons/apple-touch-icon.png.meta | 0 Assets/Resources/{Materials => }/icons/cursor.png | Bin .../Resources/{Materials => }/icons/cursor.png.meta | 0 .../{Materials => }/icons/favicon-16x16.png | Bin .../{Materials => }/icons/favicon-16x16.png.meta | 0 .../{Materials => }/icons/favicon-32x32.png | Bin .../{Materials => }/icons/favicon-32x32.png.meta | 0 Assets/Resources/{Materials => }/icons/favicon.ico | Bin .../{Materials => }/icons/favicon.ico.meta | 0 22 files changed, 8 insertions(+) create mode 100644 Assets/Resources/PhysicsMaterials.meta rename Assets/Resources/{Materials => PhysicsMaterials}/DefaultZero.physicMaterial (100%) rename Assets/Resources/{Materials => PhysicsMaterials}/DefaultZero.physicMaterial.meta (100%) rename Assets/Resources/{Materials => PhysicsMaterials}/Props.physicMaterial (100%) rename Assets/Resources/{Materials => PhysicsMaterials}/Props.physicMaterial.meta (100%) rename Assets/Resources/{Materials => PhysicsMaterials}/temp_material.physicMaterial (100%) rename Assets/Resources/{Materials => PhysicsMaterials}/temp_material.physicMaterial.meta (100%) rename Assets/Resources/{Materials => }/icons.meta (100%) rename Assets/Resources/{Materials => }/icons/android-chrome-192x192.png (100%) rename Assets/Resources/{Materials => }/icons/android-chrome-192x192.png.meta (100%) rename Assets/Resources/{Materials => }/icons/android-chrome-512x512.png (100%) rename Assets/Resources/{Materials => }/icons/android-chrome-512x512.png.meta (100%) rename Assets/Resources/{Materials => }/icons/apple-touch-icon.png (100%) rename Assets/Resources/{Materials => }/icons/apple-touch-icon.png.meta (100%) rename Assets/Resources/{Materials => }/icons/cursor.png (100%) rename Assets/Resources/{Materials => }/icons/cursor.png.meta (100%) rename Assets/Resources/{Materials => }/icons/favicon-16x16.png (100%) rename Assets/Resources/{Materials => }/icons/favicon-16x16.png.meta (100%) rename Assets/Resources/{Materials => }/icons/favicon-32x32.png (100%) rename Assets/Resources/{Materials => }/icons/favicon-32x32.png.meta (100%) rename Assets/Resources/{Materials => }/icons/favicon.ico (100%) rename Assets/Resources/{Materials => }/icons/favicon.ico.meta (100%) diff --git a/Assets/Resources/PhysicsMaterials.meta b/Assets/Resources/PhysicsMaterials.meta new file mode 100644 index 00000000..a21f52f7 --- /dev/null +++ b/Assets/Resources/PhysicsMaterials.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: cac3813512007112193b6f905a198160 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Resources/Materials/DefaultZero.physicMaterial b/Assets/Resources/PhysicsMaterials/DefaultZero.physicMaterial similarity index 100% rename from Assets/Resources/Materials/DefaultZero.physicMaterial rename to Assets/Resources/PhysicsMaterials/DefaultZero.physicMaterial diff --git a/Assets/Resources/Materials/DefaultZero.physicMaterial.meta b/Assets/Resources/PhysicsMaterials/DefaultZero.physicMaterial.meta similarity index 100% rename from Assets/Resources/Materials/DefaultZero.physicMaterial.meta rename to Assets/Resources/PhysicsMaterials/DefaultZero.physicMaterial.meta diff --git a/Assets/Resources/Materials/Props.physicMaterial b/Assets/Resources/PhysicsMaterials/Props.physicMaterial similarity index 100% rename from Assets/Resources/Materials/Props.physicMaterial rename to Assets/Resources/PhysicsMaterials/Props.physicMaterial diff --git a/Assets/Resources/Materials/Props.physicMaterial.meta b/Assets/Resources/PhysicsMaterials/Props.physicMaterial.meta similarity index 100% rename from Assets/Resources/Materials/Props.physicMaterial.meta rename to Assets/Resources/PhysicsMaterials/Props.physicMaterial.meta diff --git a/Assets/Resources/Materials/temp_material.physicMaterial b/Assets/Resources/PhysicsMaterials/temp_material.physicMaterial similarity index 100% rename from Assets/Resources/Materials/temp_material.physicMaterial rename to Assets/Resources/PhysicsMaterials/temp_material.physicMaterial diff --git a/Assets/Resources/Materials/temp_material.physicMaterial.meta b/Assets/Resources/PhysicsMaterials/temp_material.physicMaterial.meta similarity index 100% rename from Assets/Resources/Materials/temp_material.physicMaterial.meta rename to Assets/Resources/PhysicsMaterials/temp_material.physicMaterial.meta diff --git a/Assets/Resources/Materials/icons.meta b/Assets/Resources/icons.meta similarity index 100% rename from Assets/Resources/Materials/icons.meta rename to Assets/Resources/icons.meta diff --git a/Assets/Resources/Materials/icons/android-chrome-192x192.png b/Assets/Resources/icons/android-chrome-192x192.png similarity index 100% rename from Assets/Resources/Materials/icons/android-chrome-192x192.png rename to Assets/Resources/icons/android-chrome-192x192.png diff --git a/Assets/Resources/Materials/icons/android-chrome-192x192.png.meta b/Assets/Resources/icons/android-chrome-192x192.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/android-chrome-192x192.png.meta rename to Assets/Resources/icons/android-chrome-192x192.png.meta diff --git a/Assets/Resources/Materials/icons/android-chrome-512x512.png b/Assets/Resources/icons/android-chrome-512x512.png similarity index 100% rename from Assets/Resources/Materials/icons/android-chrome-512x512.png rename to Assets/Resources/icons/android-chrome-512x512.png diff --git a/Assets/Resources/Materials/icons/android-chrome-512x512.png.meta b/Assets/Resources/icons/android-chrome-512x512.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/android-chrome-512x512.png.meta rename to Assets/Resources/icons/android-chrome-512x512.png.meta diff --git a/Assets/Resources/Materials/icons/apple-touch-icon.png b/Assets/Resources/icons/apple-touch-icon.png similarity index 100% rename from Assets/Resources/Materials/icons/apple-touch-icon.png rename to Assets/Resources/icons/apple-touch-icon.png diff --git a/Assets/Resources/Materials/icons/apple-touch-icon.png.meta b/Assets/Resources/icons/apple-touch-icon.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/apple-touch-icon.png.meta rename to Assets/Resources/icons/apple-touch-icon.png.meta diff --git a/Assets/Resources/Materials/icons/cursor.png b/Assets/Resources/icons/cursor.png similarity index 100% rename from Assets/Resources/Materials/icons/cursor.png rename to Assets/Resources/icons/cursor.png diff --git a/Assets/Resources/Materials/icons/cursor.png.meta b/Assets/Resources/icons/cursor.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/cursor.png.meta rename to Assets/Resources/icons/cursor.png.meta diff --git a/Assets/Resources/Materials/icons/favicon-16x16.png b/Assets/Resources/icons/favicon-16x16.png similarity index 100% rename from Assets/Resources/Materials/icons/favicon-16x16.png rename to Assets/Resources/icons/favicon-16x16.png diff --git a/Assets/Resources/Materials/icons/favicon-16x16.png.meta b/Assets/Resources/icons/favicon-16x16.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/favicon-16x16.png.meta rename to Assets/Resources/icons/favicon-16x16.png.meta diff --git a/Assets/Resources/Materials/icons/favicon-32x32.png b/Assets/Resources/icons/favicon-32x32.png similarity index 100% rename from Assets/Resources/Materials/icons/favicon-32x32.png rename to Assets/Resources/icons/favicon-32x32.png diff --git a/Assets/Resources/Materials/icons/favicon-32x32.png.meta b/Assets/Resources/icons/favicon-32x32.png.meta similarity index 100% rename from Assets/Resources/Materials/icons/favicon-32x32.png.meta rename to Assets/Resources/icons/favicon-32x32.png.meta diff --git a/Assets/Resources/Materials/icons/favicon.ico b/Assets/Resources/icons/favicon.ico similarity index 100% rename from Assets/Resources/Materials/icons/favicon.ico rename to Assets/Resources/icons/favicon.ico diff --git a/Assets/Resources/Materials/icons/favicon.ico.meta b/Assets/Resources/icons/favicon.ico.meta similarity index 100% rename from Assets/Resources/Materials/icons/favicon.ico.meta rename to Assets/Resources/icons/favicon.ico.meta From c0cbce94e97b8c92d204b7863f164410a516aa01 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 21 Feb 2024 17:38:15 +0900 Subject: [PATCH 08/47] remove requirecomponent in Main --- Assets/Scripts/Main.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Assets/Scripts/Main.cs b/Assets/Scripts/Main.cs index c27f7ce5..7ba5bea5 100644 --- a/Assets/Scripts/Main.cs +++ b/Assets/Scripts/Main.cs @@ -13,7 +13,6 @@ using UnityEngine.UI; [DefaultExecutionOrder(30)] -[RequireComponent(typeof(ObjectSpawning), typeof(SegmentationManager))] public class Main : MonoBehaviour { [Header("Block Loading SDF")] @@ -244,6 +243,9 @@ void Awake() DeviceHelper.SetGlobalSphericalCoordinates(sphericalCoordinates); SensorDevices.DepthCamera.LoadComputeShader(); + + gameObject.AddComponent(); + gameObject.AddComponent(); } void Start() @@ -502,7 +504,5 @@ void OnDestroy() { Assimp.Unmanaged.AssimpLibrary.Instance.FreeLibrary(); } - - } -} \ No newline at end of file +} From c3ce314fb081c3fc1666f216b9b26407e07c14d7 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 28 Feb 2024 17:27:50 +0900 Subject: [PATCH 09/47] ADdd segmentcamera type for CLOiSimPlugin Types --- Assets/Scripts/CLOiSimPlugins/Modules/Base/CLOiSimPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assets/Scripts/CLOiSimPlugins/Modules/Base/CLOiSimPlugin.cs b/Assets/Scripts/CLOiSimPlugins/Modules/Base/CLOiSimPlugin.cs index b03cd770..de5a959a 100644 --- a/Assets/Scripts/CLOiSimPlugins/Modules/Base/CLOiSimPlugin.cs +++ b/Assets/Scripts/CLOiSimPlugins/Modules/Base/CLOiSimPlugin.cs @@ -9,7 +9,7 @@ public interface ICLOiSimPlugin { - enum Type { WORLD, GROUNDTRUTH, ELEVATOR, ACTOR, MICOM, JOINTCONTROL, GPS, IMU, SONAR, LASER, CAMERA, DEPTHCAMERA, MULTICAMERA, REALSENSE }; + enum Type { WORLD, GROUNDTRUTH, ELEVATOR, ACTOR, MICOM, JOINTCONTROL, GPS, IMU, SONAR, LASER, CAMERA, DEPTHCAMERA, MULTICAMERA, REALSENSE, SEGMENTCAMERA}; void SetPluginParameters(in SDF.Plugin node); SDF.Plugin GetPluginParameters(); void Reset(); From ddc14c830e0cc39810af79bc95f9870e7b6fac61 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 28 Feb 2024 17:31:12 +0900 Subject: [PATCH 10/47] Modify indentation in ColorEncoding Module --- Assets/Scripts/Core/Modules/ColorEncoding.cs | 141 ++++++++++--------- 1 file changed, 72 insertions(+), 69 deletions(-) diff --git a/Assets/Scripts/Core/Modules/ColorEncoding.cs b/Assets/Scripts/Core/Modules/ColorEncoding.cs index 2b4dac53..82185e96 100644 --- a/Assets/Scripts/Core/Modules/ColorEncoding.cs +++ b/Assets/Scripts/Core/Modules/ColorEncoding.cs @@ -1,84 +1,87 @@ -using UnityEngine; +/* + * Copyright (c) 2024 LG Electronics Inc. + * + * SPDX-License-Identifier: MIT + */ -public class ColorEncoding -{ - // public static byte ReverseBits(byte value) - // { - // return (byte)((value * 0x0202020202 & 0x010884422010) % 1023); - // } - public static int SparsifyBits(byte value, in int sparse) - { - int retVal = 0; - for (int bits = 0; bits < 8; bits++, value >>= 1) - { - retVal |= (value & 1); - retVal <<= sparse; - } - return retVal >> sparse; - } +using UnityEngine; - public static Color EncodeIDAsColor(in int instanceId) - { - var uid = instanceId * 2; - if (uid < 0) - uid = -uid + 1; +public class ColorEncoding +{ + // public static byte ReverseBits(byte value) + // { + // return (byte)((value * 0x0202020202 & 0x010884422010) % 1023); + // } - var sid = - (SparsifyBits((byte)(uid >> 16), 3) << 2) | - (SparsifyBits((byte)(uid >> 8), 3) << 1) | - SparsifyBits((byte)(uid), 3); - //Debug.Log(uid + " >>> " + System.Convert.ToString(sid, 2).PadLeft(24, '0')); + public static int SparsifyBits(byte value, in int sparse) + { + int retVal = 0; + for (int bits = 0; bits < 8; bits++, value >>= 1) + { + retVal |= (value & 1); + retVal <<= sparse; + } + return retVal >> sparse; + } - var r = (byte)(sid >> 8); - var g = (byte)(sid >> 16); - var b = (byte)(sid); + public static Color EncodeIDAsColor(in int instanceId) + { + var uid = instanceId * 2; + if (uid < 0) + uid = -uid + 1; - //Debug.Log(r + " " + g + " " + b); - return new Color32(r, g, b, 255); - } + var sid = + (SparsifyBits((byte)(uid >> 16), 3) << 2) | + (SparsifyBits((byte)(uid >> 8), 3) << 1) | + SparsifyBits((byte)(uid), 3); + //Debug.Log(uid + " >>> " + System.Convert.ToString(sid, 2).PadLeft(24, '0')); - public static Color EncodeNameAsColor(in string name) - { - return EncodeTagAsColor(name); - } + var r = (byte)(sid >> 8); + var g = (byte)(sid >> 16); + var b = (byte)(sid); + // Debug.Log($"EncodeIDAsColor({instanceId}): {r} {g} {b}"); + return new Color32(r, g, b, 255); + } - public static Color EncodeTagAsColor(in string tag) - { - var hash = tag.GetHashCode(); - var a = (byte)(hash >> 24); - var r = (byte)(hash >> 16); - var g = (byte)(hash >> 8); - var b = (byte)(hash); - return new Color32(r, g, b, 255); - } + public static Color EncodeNameAsColor(in string name) + { + return EncodeTagAsColor(name); + } - public static Color EncodeLayerAsColor(in int layer) - { - // Following value must be in the range (0.5 .. 1.0) - // in order to avoid color overlaps when using 'divider' in this func - var z = .7f; + public static Color EncodeTagAsColor(in string tag) + { + var hash = tag.GetHashCode(); + var a = (byte)(hash >> 24); + var r = (byte)(hash >> 16); + var g = (byte)(hash >> 8); + var b = (byte)(hash); + // Debug.Log($"EncodeTagAsColor({tag}): {r} {g} {b}"); + return new Color32(r, g, b, 255); + } - // First 8 layers are Unity Builtin layers - // Unity supports up to 32 layers in total + public static Color EncodeLayerAsColor(in int layer) + { + // Following value must be in the range (0.5 .. 1.0) + // in order to avoid color overlaps when using 'divider' in this func + var z = .7f; - // Lets create palette of unique 16 colors - var uniqueColors = new Color[] { - new Color(1,1,1,1), new Color(z,z,z,1), // 0 - new Color(1,1,z,1), new Color(1,z,1,1), new Color(z,1,1,1), // - new Color(1,z,0,1), new Color(z,0,1,1), new Color(0,1,z,1), // 7 + var uniqueColors = new Color[] { + new Color(1,1,1,1), new Color(z,z,z,1), // 0 + new Color(1,1,z,1), new Color(1,z,1,1), new Color(z,1,1,1), // + new Color(1,z,0,1), new Color(z,0,1,1), new Color(0,1,z,1), // 7 - new Color(1,0,0,1), new Color(0,1,0,1), new Color(0,0,1,1), // 8 - new Color(1,1,0,1), new Color(1,0,1,1), new Color(0,1,1,1), // - new Color(1,z,z,1), new Color(z,1,z,1) // 15 - }; + new Color(1,0,0,1), new Color(0,1,0,1), new Color(0,0,1,1), // 8 + new Color(1,1,0,1), new Color(1,0,1,1), new Color(0,1,1,1), // + new Color(1,z,z,1), new Color(z,1,z,1) // 15 + }; - // Create as many colors as necessary by using base 16 color palette - // To create more than 16 - will simply adjust brightness with 'divider' - var color = uniqueColors[layer % uniqueColors.Length]; - var divider = 1.0f + Mathf.Floor(layer / uniqueColors.Length); - color /= divider; + // Create as many colors as necessary by using base 16 color palette + // To create more than 16 - will simply adjust brightness with 'divider' + var color = uniqueColors[layer % uniqueColors.Length]; + var divider = 1.0f + Mathf.Floor(layer / uniqueColors.Length); + color /= divider; - return color; - } + return color; + } } From 75f5ae5d66639fda6d123a2a0f51cbdea64e8d1e Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 28 Feb 2024 17:31:41 +0900 Subject: [PATCH 11/47] Modify Camera Plugin and Realsense Plugin --- Assets/Scripts/CLOiSimPlugins/CameraPlugin.cs | 40 +++++++++++++++---- .../Scripts/CLOiSimPlugins/RealSensePlugin.cs | 23 +---------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Assets/Scripts/CLOiSimPlugins/CameraPlugin.cs b/Assets/Scripts/CLOiSimPlugins/CameraPlugin.cs index ee0139e5..94febb7d 100644 --- a/Assets/Scripts/CLOiSimPlugins/CameraPlugin.cs +++ b/Assets/Scripts/CLOiSimPlugins/CameraPlugin.cs @@ -22,25 +22,49 @@ public SensorDevices.DepthCamera GetDepthCamera() protected override void OnAwake() { - var depthcam = gameObject.GetComponent(); - if (depthcam is null) + var depthCam = gameObject.GetComponent(); + var segCam = gameObject.GetComponent(); + + var deviceName = string.Empty; + if (depthCam is null && segCam is not null) { - ChangePluginType(ICLOiSimPlugin.Type.CAMERA); - cam = gameObject.GetComponent(); - attachedDevices.Add("Camera", cam); + ChangePluginType(ICLOiSimPlugin.Type.SEGMENTCAMERA); + cam = segCam; + deviceName = "SegmentationCamera"; } - else + else if (depthCam is not null && segCam is null) { ChangePluginType(ICLOiSimPlugin.Type.DEPTHCAMERA); - cam = depthcam; - attachedDevices.Add("DepthCamera", cam); + cam = depthCam; + deviceName = "DepthCamera"; } + else + { + ChangePluginType(ICLOiSimPlugin.Type.CAMERA); + cam = gameObject.GetComponent(); + deviceName = "Camera"; + } + + if (!string.IsNullOrEmpty(deviceName)) + attachedDevices.Add(deviceName, cam); partsName = DeviceHelper.GetPartName(gameObject); } protected override void OnStart() { + if (type == ICLOiSimPlugin.Type.DEPTHCAMERA) + { + if (GetPluginParameters() != null) + { + var depthScale = GetPluginParameters().GetValue("configuration/depth_scale", 1000); + if (cam != null) + { + ((SensorDevices.DepthCamera)cam).SetDepthScale(depthScale); + } + } + } + if (RegisterServiceDevice(out var portService, "Info")) { AddThread(portService, ServiceThread); diff --git a/Assets/Scripts/CLOiSimPlugins/RealSensePlugin.cs b/Assets/Scripts/CLOiSimPlugins/RealSensePlugin.cs index 38bb2968..190d625b 100644 --- a/Assets/Scripts/CLOiSimPlugins/RealSensePlugin.cs +++ b/Assets/Scripts/CLOiSimPlugins/RealSensePlugin.cs @@ -26,7 +26,6 @@ protected override void OnAwake() protected override void OnStart() { - var depthScale = GetPluginParameters().GetValue("configuration/depth_scale", 1000); var colorName = GetPluginParameters().GetValue("activate/module[@name='color']"); var leftImagerName = GetPluginParameters().GetValue("activate/module[@name='left_imager']"); var rightImagerName = GetPluginParameters().GetValue("activate/module[@name='right_imager']"); @@ -51,12 +50,12 @@ protected override void OnStart() if (!string.IsNullOrEmpty(depthName)) { - FindAndAddDepthCameraPlugin(depthName, depthScale); + FindAndAddCameraPlugin(depthName); } if (!string.IsNullOrEmpty(alignedDepthToColorName)) { - FindAndAddDepthCameraPlugin(alignedDepthToColorName, depthScale); + FindAndAddCameraPlugin(alignedDepthToColorName); } if (!string.IsNullOrEmpty(imuName)) @@ -85,24 +84,6 @@ private void AddImuPlugin(in string name) } } - private void FindAndAddDepthCameraPlugin(in string name, in uint depthScale) - { - var plugin = FindAndAddCameraPlugin(name); - if (plugin == null) - { - Debug.LogWarning(name + " plugin is not loaded."); - } - else - { - var depthCamera = plugin.GetDepthCamera(); - - if (depthCamera != null) - { - depthCamera.SetDepthScale(depthScale); - } - } - } - private CameraPlugin FindAndAddCameraPlugin(in string name) { foreach (var camera in cameras) From abd8bc50ffc59e15b6f090514c05d223de4900a2 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 28 Feb 2024 17:54:21 +0900 Subject: [PATCH 12/47] Change event in render objects for Segmentation material -> "AfterRenderingPostProcessing" it's important to make consistatnt color value in textures --- Assets/Resources/RenderPipelines/SegmentationRenderer.asset | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset index b4f27546..8872e5f8 100644 --- a/Assets/Resources/RenderPipelines/SegmentationRenderer.asset +++ b/Assets/Resources/RenderPipelines/SegmentationRenderer.asset @@ -15,7 +15,7 @@ MonoBehaviour: m_Active: 1 settings: passTag: RenderObjects - Event: 250 + Event: 600 filterSettings: RenderQueueType: 0 LayerMask: From 15064e3c432fc01261fb4943fca13206e906dee8 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Wed, 28 Feb 2024 17:57:38 +0900 Subject: [PATCH 13/47] change variable type in segmentation shader : float4 -> half4 for color --- Assets/Resources/Shader/Segmentation.shader | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Assets/Resources/Shader/Segmentation.shader b/Assets/Resources/Shader/Segmentation.shader index a048b315..912e0125 100644 --- a/Assets/Resources/Shader/Segmentation.shader +++ b/Assets/Resources/Shader/Segmentation.shader @@ -15,9 +15,6 @@ Shader "Sensor/Segmentation" "RenderType" = "Opaque" } - HLSLINCLUDE - ENDHLSL - Pass { Name "Segmentation" @@ -36,7 +33,6 @@ Shader "Sensor/Segmentation" int _OutputMode; CBUFFER_END - half4 replacement_output() { /* @@ -61,12 +57,12 @@ Shader "Sensor/Segmentation" struct Attributes { - half4 positionOS : POSITION; + float4 positionOS : POSITION; }; struct Varyings { - half4 positionCS : SV_POSITION; + float4 positionCS : SV_POSITION; half4 color : COLOR; }; @@ -75,13 +71,15 @@ Shader "Sensor/Segmentation" Varyings output = (Varyings)0; VertexPositionInputs vertexInput = GetVertexPositionInputs(input.positionOS.xyz); output.positionCS = vertexInput.positionCS; - output.color = replacement_output(); + half4 color =replacement_output(); + output.color = color; return output; } half4 frag(Varyings i) : SV_Target { - return replacement_output(); + half4 color =replacement_output(); + return color; } ENDHLSL } From 509cf3c7e7bc2e355224ea995d890f56536c6baf Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Fri, 1 Mar 2024 08:48:10 +0900 Subject: [PATCH 14/47] Modify Camer Sensor Devices - Camera Base - Depth Camera - Segmentation Camera --- Assets/Scripts/Devices/Camera.cs | 47 ++++--- Assets/Scripts/Devices/DepthCamera.cs | 123 ++++++++++--------- Assets/Scripts/Devices/Modules/DepthData.cs | 18 ++- Assets/Scripts/Devices/SegmentationCamera.cs | 43 ++++--- 4 files changed, 129 insertions(+), 102 deletions(-) diff --git a/Assets/Scripts/Devices/Camera.cs b/Assets/Scripts/Devices/Camera.cs index 0ff0427d..dee457a3 100644 --- a/Assets/Scripts/Devices/Camera.cs +++ b/Assets/Scripts/Devices/Camera.cs @@ -32,6 +32,11 @@ public class Camera : Device protected string _targetRTname; protected GraphicsFormat _targetColorFormat; + protected FilterMode _filterMode = FilterMode.Bilinear; + protected MSAASamples _msaaSample = MSAASamples.MSAA2x; + protected bool _useDynamicScale = true; + protected int _anisoLevel = 3; + protected GraphicsFormat _readbackDstFormat; protected CameraData.Image _camImageData; @@ -87,6 +92,7 @@ protected override void OnStart() if (camSensor) { SetupTexture(); + SetupDefaultCamera(); SetupCamera(); _startCameraWork = true; } @@ -94,15 +100,6 @@ protected override void OnStart() protected virtual void SetupTexture() { - camSensor.clearFlags = CameraClearFlags.Skybox; - camSensor.allowHDR = true; - camSensor.depthTextureMode = DepthTextureMode.None; - _universalCamData.requiresColorOption = CameraOverrideOption.On; - _universalCamData.requiresDepthOption = CameraOverrideOption.Off; - _universalCamData.requiresColorTexture = true; - _universalCamData.requiresDepthTexture = false; - _universalCamData.renderShadows = true; - // Debug.Log("This is not a Depth Camera!"); _targetRTname = "CameraColorTexture"; @@ -179,13 +176,16 @@ private void OnDisable() RenderPipelineManager.endCameraRendering -= OnEndCameraRendering; } - // protected virtual void SetupCamera() - private void SetupCamera() + private void SetupDefaultCamera() { camSensor.ResetWorldToCameraMatrix(); camSensor.ResetProjectionMatrix(); + camSensor.backgroundColor = Color.black; + camSensor.clearFlags = CameraClearFlags.Skybox; + camSensor.depthTextureMode = DepthTextureMode.None; camSensor.renderingPath = RenderingPath.Forward; + camSensor.allowHDR = true; camSensor.allowMSAA = true; camSensor.allowDynamicResolution = true; camSensor.useOcclusionCulling = true; @@ -202,17 +202,21 @@ private void SetupCamera() slices: 1, depthBufferBits: DepthBits.None, colorFormat: _targetColorFormat, + // filterMode: _filterMode, filterMode: FilterMode.Bilinear, wrapMode: TextureWrapMode.Clamp, dimension: TextureDimension.Tex2D, + // msaaSamples: _msaaSample, msaaSamples: MSAASamples.MSAA2x, enableRandomWrite: false, useMipMap: true, autoGenerateMips: true, isShadowMap: false, + // anisoLevel: _anisoLevel, anisoLevel: 3, mipMapBias: 0, bindTextureMS: false, + // useDynamicScale: _useDynamicScale, useDynamicScale: true, memoryless: RenderTextureMemoryless.None, name: _targetRTname); @@ -228,6 +232,11 @@ private void SetupCamera() var invertMatrix = Matrix4x4.Scale(new Vector3(1, -1, 1)); camSensor.projectionMatrix = projMatrix * invertMatrix; + _universalCamData.requiresColorOption = CameraOverrideOption.On; + _universalCamData.requiresDepthOption = CameraOverrideOption.Off; + _universalCamData.requiresColorTexture = true; + _universalCamData.requiresDepthTexture = false; + _universalCamData.renderShadows = true; _universalCamData.enabled = false; _universalCamData.stopNaN = true; _universalCamData.dithering = true; @@ -236,21 +245,20 @@ private void SetupCamera() _universalCamData.volumeLayerMask = LayerMask.GetMask("Nothing"); _universalCamData.renderType = CameraRenderType.Base; _universalCamData.cameraStack.Clear(); - camSensor.enabled = false; + camSensor.enabled = false; // camSensor.hideFlags |= HideFlags.NotEditable; } - protected new void OnDestroy() + protected virtual void SetupCamera() { - _startCameraWork = false; + } + protected new void OnDestroy() + { // Debug.Log("OnDestroy(Camera)"); - RenderPipelineManager.endCameraRendering -= OnEndCameraRendering; - RenderPipelineManager.beginCameraRendering -= OnBeginCameraRendering; - + _startCameraWork = false; _rtHandle?.Release(); - base.OnDestroy(); } @@ -311,7 +319,6 @@ protected void OnCompleteAsyncReadback(AsyncGPUReadbackRequest request) protected override void GenerateMessage() { - DeviceHelper.SetCurrentTime(imageStamped.Time); PushDeviceMessage(imageStamped); } @@ -335,6 +342,8 @@ protected virtual void ImageProcessing(ref NativeArray readbackData) { Debug.LogWarningFormat("{0}: Failed to get image Data", name); } + + DeviceHelper.SetCurrentTime(imageStamped.Time); } public messages.CameraSensor GetCameraInfo() diff --git a/Assets/Scripts/Devices/DepthCamera.cs b/Assets/Scripts/Devices/DepthCamera.cs index 051c8c12..5308afd6 100644 --- a/Assets/Scripts/Devices/DepthCamera.cs +++ b/Assets/Scripts/Devices/DepthCamera.cs @@ -11,6 +11,7 @@ using Unity.Collections; using Unity.Jobs; using System.Threading.Tasks; +using System; namespace SensorDevices { @@ -20,7 +21,7 @@ public class DepthCamera : Camera #region "For Compute Shader" private static ComputeShader ComputeShaderDepthBuffer = null; - private ComputeShader computeShader = null; + private ComputeShader _computeShader = null; private int _kernelIndex = -1; private const int ThreadGroupsX = 32; private const int ThreadGroupsY = 32; @@ -41,7 +42,6 @@ public class DepthCamera : Camera private int _computedBufferOutputUnitLength; private Texture2D _textureForCapture; - public static void LoadComputeShader() { if (ComputeShaderDepthBuffer == null) @@ -81,58 +81,23 @@ public void FlipXDepthData(in bool flip) public void SetDepthScale(in uint value) { depthScale = value; - - if (computeShader != null) - { - computeShader.SetFloat("_DepthScale", (float)depthScale); - } } new void OnDestroy() { // Debug.Log("OnDestroy(Depth Camera)"); - Destroy(computeShader); - computeShader = null; + Destroy(_computeShader); + _computeShader = null; base.OnDestroy(); } protected override void SetupTexture() { - computeShader = Instantiate(ComputeShaderDepthBuffer); - _kernelIndex = computeShader.FindKernel("CSScaleDepthBuffer"); - - var depthShader = Shader.Find("Sensor/Depth"); - depthMaterial = new Material(depthShader); - - if (camParameter.depth_camera_output.Equals("points")) - { - Debug.Log("Enable Point Cloud data mode - NOT SUPPORT YET!"); - camParameter.image.format = "RGB_FLOAT32"; - } - - camSensor.clearFlags = CameraClearFlags.Depth; - camSensor.allowHDR = false; - camSensor.depthTextureMode = DepthTextureMode.Depth; - _universalCamData.requiresColorOption = CameraOverrideOption.Off; - _universalCamData.requiresDepthOption = CameraOverrideOption.On; - _universalCamData.requiresColorTexture = false; - _universalCamData.requiresDepthTexture = true; - _universalCamData.renderShadows = false; - _targetRTname = "CameraDepthTexture"; _targetColorFormat = GraphicsFormat.R8G8B8A8_UNorm; _readbackDstFormat = GraphicsFormat.R8G8B8A8_UNorm; - var cb = new CommandBuffer(); - cb.name = "CommandBufferForDepthShading"; - var tempTextureId = Shader.PropertyToID("_RenderImageCameraDepthTexture"); - cb.GetTemporaryRT(tempTextureId, -1, -1); - cb.Blit(tempTextureId, BuiltinRenderTextureType.CameraTarget, depthMaterial); - cb.ReleaseTemporaryRT(tempTextureId); - camSensor.AddCommandBuffer(CameraEvent.AfterEverything, cb); - cb.Release(); - var width = camParameter.image.width; var height = camParameter.image.height; var format = CameraData.GetPixelFormat(camParameter.image.format); @@ -155,22 +120,58 @@ protected override void SetupTexture() _computedBufferOutputUnitLength = width * height; _computedBufferOutput = new byte[_computedBufferOutputUnitLength * OutputUnitSize]; - if (computeShader != null) + _threadGroupX = width / ThreadGroupsX; + _threadGroupY = height / ThreadGroupsY; + + _textureForCapture = new Texture2D(width, height, graphicFormat, 0, TextureCreationFlags.None); + _textureForCapture.filterMode = FilterMode.Point; + + _computeShader = Instantiate(ComputeShaderDepthBuffer); + + if (_computeShader != null) { + _kernelIndex = _computeShader.FindKernel("CSScaleDepthBuffer"); _imageDepth = CameraData.GetImageDepth(format); - computeShader.SetFloat("_DepthMax", (float)camParameter.clip.far); - computeShader.SetInt("_Width", width); - computeShader.SetInt("_UnitSize", _imageDepth); + + _computeShader.SetFloat("_DepthMax", (float)camParameter.clip.far); + _computeShader.SetInt("_Width", width); + _computeShader.SetInt("_UnitSize", _imageDepth); + _computeShader.SetFloat("_DepthScale", (float)depthScale); } + } - ReverseDepthData(false); - FlipXDepthData(false); + protected override void SetupCamera() + { + var depthShader = Shader.Find("Sensor/Depth"); + depthMaterial = new Material(depthShader); - _threadGroupX = width / ThreadGroupsX; - _threadGroupY = height / ThreadGroupsY; + if (camParameter.depth_camera_output.Equals("points")) + { + Debug.Log("Enable Point Cloud data mode - NOT SUPPORT YET!"); + camParameter.image.format = "RGB_FLOAT32"; + } - _textureForCapture = new Texture2D(width, height, graphicFormat, 0, TextureCreationFlags.None); - _textureForCapture.filterMode = FilterMode.Point; + camSensor.clearFlags = CameraClearFlags.Depth; + camSensor.allowHDR = false; + camSensor.depthTextureMode = DepthTextureMode.Depth; + + _universalCamData.requiresColorOption = CameraOverrideOption.Off; + _universalCamData.requiresDepthOption = CameraOverrideOption.On; + _universalCamData.requiresColorTexture = false; + _universalCamData.requiresDepthTexture = true; + _universalCamData.renderShadows = false; + + var cb = new CommandBuffer(); + cb.name = "CommandBufferForDepthShading"; + var tempTextureId = Shader.PropertyToID("_RenderImageCameraDepthTexture"); + cb.GetTemporaryRT(tempTextureId, -1, -1); + cb.Blit(tempTextureId, BuiltinRenderTextureType.CameraTarget, depthMaterial); + cb.ReleaseTemporaryRT(tempTextureId); + camSensor.AddCommandBuffer(CameraEvent.AfterEverything, cb); + cb.Release(); + + ReverseDepthData(false); + FlipXDepthData(false); } protected override void ImageProcessing(ref NativeArray readbackData) @@ -178,18 +179,18 @@ protected override void ImageProcessing(ref NativeArray readbackData) _depthCamBuffer.Allocate(); _depthCamBuffer.raw = readbackData; - if (_depthCamBuffer.depth.IsCreated && computeShader != null) + if (_depthCamBuffer.depth.IsCreated && _computeShader != null) { var jobHandleDepthCamBuffer = _depthCamBuffer.Schedule(_depthCamBuffer.Length(), BatchSize); jobHandleDepthCamBuffer.Complete(); var computeBufferSrc = new ComputeBuffer(_depthCamBuffer.depth.Length, sizeof(float)); - computeShader.SetBuffer(_kernelIndex, "_Input", computeBufferSrc); + _computeShader.SetBuffer(_kernelIndex, "_Input", computeBufferSrc); computeBufferSrc.SetData(_depthCamBuffer.depth); var computeBufferDst = new ComputeBuffer(_computedBufferOutput.Length, sizeof(byte)); - computeShader.SetBuffer(_kernelIndex, "_Output", computeBufferDst); - computeShader.Dispatch(_kernelIndex, _threadGroupX, _threadGroupY, 1); + _computeShader.SetBuffer(_kernelIndex, "_Output", computeBufferDst); + _computeShader.Dispatch(_kernelIndex, _threadGroupX, _threadGroupY, 1); computeBufferDst.GetData(_computedBufferOutput); computeBufferSrc.Release(); @@ -197,11 +198,19 @@ protected override void ImageProcessing(ref NativeArray readbackData) Parallel.For(0, _computedBufferOutputUnitLength, (int i) => { - for (int j = 0; j < _imageDepth; j++) - imageStamped.Image.Data[i * _imageDepth + j] = _computedBufferOutput[i * OutputUnitSize + j]; + Buffer.BlockCopy( + _computedBufferOutput, i * (int)OutputUnitSize, + imageStamped.Image.Data, i * _imageDepth, + _imageDepth); }); - // Debug.LogFormat("{0:X} {1:X} {2:X} {3:X} => {4}, {5}, {6}, {7}", image.Data[0], image.Data[1], image.Data[2], image.Data[3], System.BitConverter.ToInt16(image.Data, 0), System.BitConverter.ToUInt16(image.Data, 2), System.BitConverter.ToInt32(image.Data, 0), System.BitConverter.ToSingle(image.Data, 0)); + // Debug.LogFormat("{0:X} {1:X} {2:X} {3:X} => {4}, {5}, {6}, {7}", + // imageStamped.Image.Data[1000], imageStamped.Image.Data[2000], + // imageStamped.Image.Data[5000], imageStamped.Image.Data[8000], + // System.BitConverter.ToInt16(imageStamped.Image.Data, 0), + // System.BitConverter.ToUInt16(imageStamped.Image.Data, 2), + // System.BitConverter.ToInt32(imageStamped.Image.Data, 0), + // System.BitConverter.ToSingle(imageStamped.Image.Data, 0)); if (camParameter.save_enabled && _startCameraWork) { @@ -210,6 +219,8 @@ protected override void ImageProcessing(ref NativeArray readbackData) } } _depthCamBuffer.Deallocate(); + + DeviceHelper.SetCurrentTime(imageStamped.Time); } private void SaveRawImageData(in string path, in string name) diff --git a/Assets/Scripts/Devices/Modules/DepthData.cs b/Assets/Scripts/Devices/Modules/DepthData.cs index 9a80959c..9bd0bbf6 100644 --- a/Assets/Scripts/Devices/Modules/DepthData.cs +++ b/Assets/Scripts/Devices/Modules/DepthData.cs @@ -55,19 +55,17 @@ public void Execute(int i) private float GetDecodedData(in int index) { var imageOffset = index * ColorFormatUnitSize; - if (raw != null && imageOffset < raw.Length) - { - var r = raw[imageOffset]; - var g = raw[imageOffset + 1]; - var b = raw[imageOffset + 2]; - var a = raw[imageOffset + 3]; - - return DecodeFloatRGBA(r, g, b, a); - } - else + if (raw == null || imageOffset >= raw.Length) { return 0; } + + var r = raw[imageOffset]; + var g = raw[imageOffset + 1]; + var b = raw[imageOffset + 2]; + var a = raw[imageOffset + 3]; + + return DecodeFloatRGBA(r, g, b, a); } private float DecodeFloatRGBA(in byte r, in byte g, in byte b, in byte a) diff --git a/Assets/Scripts/Devices/SegmentationCamera.cs b/Assets/Scripts/Devices/SegmentationCamera.cs index 6e6e88ed..c97f9b44 100644 --- a/Assets/Scripts/Devices/SegmentationCamera.cs +++ b/Assets/Scripts/Devices/SegmentationCamera.cs @@ -15,6 +15,27 @@ namespace SensorDevices public class SegmentationCamera : Camera { protected override void SetupTexture() + { + _targetRTname = "SegmentationTexture"; + + _filterMode = FilterMode.Point; + _msaaSample = UnityEngine.Rendering.MSAASamples.None; + _useDynamicScale = false; + _anisoLevel = 0; + + var pixelFormat = CameraData.GetPixelFormat(camParameter.image.format); + if (pixelFormat != CameraData.PixelFormat.L_INT16) + { + Debug.Log("Only support INT16 format"); + } + + _targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; + _readbackDstFormat = GraphicsFormat.R8G8_UNorm; + + _camImageData = new CameraData.Image(camParameter.image.width, camParameter.image.height, pixelFormat); + } + + protected override void SetupCamera() { if (!camParameter.segmentation_type.Equals("semantic")) { @@ -23,9 +44,9 @@ protected override void SetupTexture() camSensor.backgroundColor = Color.black; camSensor.clearFlags = CameraClearFlags.SolidColor; - camSensor.depthTextureMode = DepthTextureMode.None; camSensor.allowHDR = false; - camSensor.allowMSAA = true; + camSensor.allowMSAA = false; + camSensor.allowDynamicResolution = false; // Refer to SegmentationRenderer (Universal Renderer Data) _universalCamData.SetRenderer(1); @@ -34,21 +55,7 @@ protected override void SetupTexture() _universalCamData.requiresDepthOption = CameraOverrideOption.Off; _universalCamData.requiresColorTexture = true; _universalCamData.requiresDepthTexture = false; - _universalCamData.renderShadows = true; - - _targetRTname = "SegmentationTexture"; - - var pixelFormat = CameraData.GetPixelFormat(camParameter.image.format); - if (pixelFormat != CameraData.PixelFormat.L_INT16) - { - Debug.Log("Only support INT16 format"); - } - - _targetColorFormat = GraphicsFormat.R8G8B8A8_SRGB; - _readbackDstFormat = GraphicsFormat.R16_UNorm; - // _readbackDstFormat = GraphicsFormat.R16_SFloat; - - _camImageData = new CameraData.Image(camParameter.image.width, camParameter.image.height, pixelFormat); + _universalCamData.renderShadows = false; } protected override void ImageProcessing(ref NativeArray readbackData) @@ -71,6 +78,8 @@ protected override void ImageProcessing(ref NativeArray readbackData) { Debug.LogWarningFormat("{0}: Failed to get image Data", name); } + + DeviceHelper.SetCurrentTime(imageStamped.Time); } } } \ No newline at end of file From 479310459dd0688fc04b584013ca992b725ed7c2 Mon Sep 17 00:00:00 2001 From: "Hyunseok Yang (YG)" Date: Fri, 1 Mar 2024 12:35:35 +0900 Subject: [PATCH 15/47] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 03409d8d..c83a418e 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Here are the list of items that is implemented(marked) or planned to be implemen - [X] Depth Camera - [X] Multi-camera - [X] RealSense (RGB + IR1 + IR2 + Depth) + - [X] Semantic Segmentation Camera - [X] GPS sensor - [ ] Sensor noise models - [X] Gaussian @@ -143,7 +144,7 @@ if `` element of `