From 2bf1942b5ed6dd805b1e6f4eb2661f4e4a86565a Mon Sep 17 00:00:00 2001 From: AlvaroHG Date: Thu, 19 Sep 2024 15:48:34 -0700 Subject: [PATCH] Distortion --- unity/Assets/Scripts/AgentManager.cs | 23 ++++++++++++++++++- unity/Assets/Scripts/DebugInputField.cs | 14 +++++++++++ .../Scripts/ImageSynthesis/ImageSynthesis.cs | 20 ++++++++++------ .../ImageSynthesis/Shaders/DepthBW.shader | 2 +- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/unity/Assets/Scripts/AgentManager.cs b/unity/Assets/Scripts/AgentManager.cs index b5f55887bc..5d2b7a13f8 100644 --- a/unity/Assets/Scripts/AgentManager.cs +++ b/unity/Assets/Scripts/AgentManager.cs @@ -51,6 +51,7 @@ public class AgentManager : MonoBehaviour, ActionInvokable { private bool initializedInstanceSeg; private bool renderNormalsImage; private bool renderFlowImage; + private bool renderDistortionImage; private Socket sock = null; [SerializeField] @@ -88,7 +89,8 @@ private enum serverTypes { "UpdateThirdPartyCamera", "ChangeResolution", "CoordinateFromRaycastThirdPartyCamera", - "ChangeQuality" + "ChangeQuality", + "SetDistortionShaderParams" }; public HashSet errorAllowedActions = new HashSet { "Reset" }; @@ -334,6 +336,7 @@ public void Initialize(ServerAction action) { this.renderInstanceSegmentation = this.initializedInstanceSeg = action.renderInstanceSegmentation; this.renderFlowImage = action.renderFlowImage; + this.renderDistortionImage = action.renderDistortionImage; this.fastActionEmit = action.fastActionEmit; PhysicsSceneManager.SetDefaultSimulationParams(action.defaultPhysicsSimulationParams); @@ -774,6 +777,7 @@ public void AddThirdPartyCamera( || renderInstanceSegmentation || renderNormalsImage || renderFlowImage + || renderDistortionImage ) { gameObject.AddComponent(typeof(ImageSynthesis)); } @@ -1837,6 +1841,22 @@ protected bool LoadBoolVariable(bool variable, string name) { public void SetCriticalErrorState() { this.agentManagerState = AgentState.Error; } + + public ActionFinished SetDistortionShaderParams(float zoomPercent, float k1, float k2, float k3, float k4, float strength = 1.0f) { + + if (this.primaryAgent.imageSynthesis == null) { + return new ActionFinished(success: false, errorMessage: "No imageSynthesis, make sure you pass 'renderDistortionImage = true' to the agent constructor."); + } + var material = this.primaryAgent.imageSynthesis.distortionMaterial; + material.SetFloat("_ZoomPercent", zoomPercent); + material.SetFloat("_k1", k1); + material.SetFloat("_k2", k2); + material.SetFloat("_k3", k3); + material.SetFloat("_k4", k4); + material.SetFloat("_LensDistortionStrength", strength); + return ActionFinished.Success; + + } } [Serializable] @@ -2589,6 +2609,7 @@ public class ServerAction { public bool renderInstanceSegmentation; public bool renderNormalsImage; public bool renderFlowImage; + public bool renderDistortionImage; public float cameraY = 0.675f; public bool placeStationary = true; // when placing/spawning an object, do we spawn it stationary (kinematic true) or spawn and let physics resolve final position diff --git a/unity/Assets/Scripts/DebugInputField.cs b/unity/Assets/Scripts/DebugInputField.cs index f82f7f48fc..411e6b0c52 100644 --- a/unity/Assets/Scripts/DebugInputField.cs +++ b/unity/Assets/Scripts/DebugInputField.cs @@ -2677,6 +2677,20 @@ IEnumerator executeBatch(JArray jActions) { break; } + case "sdp": { + Dictionary action = new Dictionary() { + ["action"] = "SetDistortionShaderParams", + ["zoomPercent"] = 0.84f, + ["k1"] = 0.1f, + ["k2"] = 0.01f, + ["k3"] = -0.2f, + ["k4"] = 0.0, + ["strength"] = 1.0 + }; + this.AManager.ProcessControlCommand(new DynamicServerAction(action)); + break; + } + case "gopro1": { Dictionary action = new Dictionary() { ["action"] = "UpdateMainCamera", diff --git a/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs b/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs index 3f1983e5f1..be5d2fc7da 100644 --- a/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs +++ b/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs @@ -204,6 +204,8 @@ public class ImageSynthesis : MonoBehaviour { needsRescale = true }, // (see issue with Motion Vectors in @KNOWN ISSUES) + new CapturePass() { name = "_distortion" } + // new CapturePass() { name = "_position" }, }; @@ -245,7 +247,7 @@ public void updateCameraStatuses(bool enabled) { private Shader opticalFlowShader; private Shader depthShader; - private Shader screenCopyShader; + private Shader distortionShader; // public Shader positionShader; @@ -259,7 +261,7 @@ public void updateCameraStatuses(bool enabled) { private Material opticalFlowMaterial; public Material depthMaterial; - private Material screenCopyMaterial; + public Material distortionMaterial; private RenderTexture renderTexture; System.Security.Cryptography.MD5 md5; @@ -299,14 +301,14 @@ public void OnEnable() { depthShader = Shader.Find("Hidden/Depth"); #endif - if (!screenCopyShader) { - screenCopyShader = Shader.Find("Hidden/ScreenCopy"); + if (!distortionShader) { + distortionShader = Shader.Find("Hidden/BarrelDistortion"); } // if (!positionShader) // positionShader = Shader.Find("Hidden/World"); - opticalFlowSensitivity = 40.0f; + opticalFlowSensitivity = 50.0f; // use real camera to capture final image capturePasses[0].camera = GetComponent(); @@ -484,8 +486,8 @@ public void OnCameraChange() { // screenCopyMaterial = new Material(screenCopyShader); - if (!screenCopyMaterial || screenCopyMaterial.shader != screenCopyShader) { - screenCopyMaterial = new Material(screenCopyShader); + if (!distortionMaterial || distortionMaterial.shader != distortionShader) { + distortionMaterial = new Material(distortionShader); } // capturePasses [1].camera.farClipPlane = 100; @@ -525,6 +527,10 @@ public void OnCameraChange() { DepthTextureMode.Depth | DepthTextureMode.MotionVectors ); + SetupCameraWithPostShader2(renderTexture, capturePasses[6].camera, distortionMaterial, + // screenCopyMaterial: screenCopyMaterial, + DepthTextureMode.Depth); + #if UNITY_EDITOR for (int i = 0; i < capturePasses.Length; i++) { // Debug.Log("Setting camera " + capturePasses[i].camera.gameObject.name + " to display " + i); diff --git a/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader b/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader index e55b98a5de..af10f41e96 100644 --- a/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader +++ b/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader @@ -92,7 +92,7 @@ // depth01 = pow(LinearEyeDepth(depth01), _DepthLevel); float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))); // (LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))) / (_ProjectionParams.z - _ProjectionParams.y); - // return fixed4(depth01, depth01, depth01, depth01); + return fixed4(depth01, depth01, depth01, depth01); // return fixed4(o.uv.x, 0, o.uv.y, 1.0); // _ScreenParams.x