From 668448067752a00505872308680e9c70cd7b5fbf Mon Sep 17 00:00:00 2001 From: AlvaroHG Date: Fri, 22 Nov 2024 01:27:09 -0800 Subject: [PATCH] Fixed depth --- ai2thor/interact.py | 1 - test_distortion.py | 17 +- unity/Assets/Scripts/AgentManager.cs | 5 +- unity/Assets/Scripts/CapturePass.cs | 6 +- .../ImageSynthesis/Shaders/DepthBW.shader | 226 +----------------- unity/Assets/Scripts/RenderingManager.cs | 4 +- 6 files changed, 26 insertions(+), 233 deletions(-) diff --git a/ai2thor/interact.py b/ai2thor/interact.py index 2adde955e5..f8c8e90653 100644 --- a/ai2thor/interact.py +++ b/ai2thor/interact.py @@ -328,7 +328,6 @@ def json_write(name, obj): if condition: frame = frame_func(event) if frame is not None: - frame = frame_func(event) frame = transform(frame) image_name = os.path.join( image_dir, diff --git a/test_distortion.py b/test_distortion.py index 12901cc6cb..bf91e9cc5d 100644 --- a/test_distortion.py +++ b/test_distortion.py @@ -109,15 +109,15 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, mainCamera=True, thidPartyCameraIndices=[0] ) - result = evt.metadata['actionReturn'] + distortionMaps = evt.metadata['actionReturn'] # keys = [key for (key, val) in result.items()] maps = [] - print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {result.keys()}") - print(f"[x,y] at (0,0) (bottom left corner) len {result['mainCamera'][0][0]}") - tex_height = len(result['mainCamera']) - tex_width = len(result['mainCamera'][0]) - print(f"[x,y] at (height, width) (top right corner) len {result['thirdPartyCameras'][0][tex_height-1][tex_width-1]}") + print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {distortionMaps.keys()}") + print(f"[x,y] at (0,0) (bottom left corner) len {distortionMaps['mainCamera'][0][0]}") + tex_height = len(distortionMaps['mainCamera']) + tex_width = len(distortionMaps['mainCamera'][0]) + print(f"[x,y] at (height, width) (top right corner) len {distortionMaps['thirdPartyCameras'][0][tex_height-1][tex_width-1]}") print(f'Error: {evt.metadata["errorMessage"]}') @@ -153,7 +153,10 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, print(f"Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']}") print(f'Error: {evt.metadata["errorMessage"]}') - + # controller.step(action="MoveAhead") + # controller.step(action="MoveAhead") + # controller.step(action="MoveAhead") + InteractiveControllerPrompt.write_image(controller.last_event, image_dir, "", semantic_segmentation_frame=True, depth_frame=True, color_frame=True, third_party_camera_frames=True, distortion_frame=distortion) # input() diff --git a/unity/Assets/Scripts/AgentManager.cs b/unity/Assets/Scripts/AgentManager.cs index ead0e94233..124754be4a 100644 --- a/unity/Assets/Scripts/AgentManager.cs +++ b/unity/Assets/Scripts/AgentManager.cs @@ -18,7 +18,6 @@ using Newtonsoft.Json.Serialization; using Thor.Procedural.Data; using Thor.Rendering; -using UnityEditor.AssetImporters; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Networking; @@ -1412,6 +1411,10 @@ string fieldName Debug.Log($"------- Gettin capture for {captureName} {flag}"); var renderingManager = camera.GetComponent(); byte[] bytes = renderingManager.GetCaptureBytes(captureName); + if (captureName == "_depth") { + var localBytesX = new byte[] {bytes[0], bytes[1], bytes[2], bytes[3] }; + Debug.Log($"-------- _depth bytes {string.Join(", ", bytes.Take(100))} convert {System.BitConverter.ToSingle(localBytesX, 0)}"); + } payload.Add(new KeyValuePair(fieldName, bytes)); } } diff --git a/unity/Assets/Scripts/CapturePass.cs b/unity/Assets/Scripts/CapturePass.cs index 6f05201ffe..3bf70bafd0 100644 --- a/unity/Assets/Scripts/CapturePass.cs +++ b/unity/Assets/Scripts/CapturePass.cs @@ -535,7 +535,11 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) { // commandBuffer.ReleaseTemporaryRT(screenCopyID); foreach (var pass in this.passDict.Values) { - commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, pass.GetRenderTexture(), pass.material); + if (pass.GetName() == "_depth") { + int depthID = Shader.PropertyToID("_DepthCopyTexture"); + } + commandBuffer.Blit(this.GetRenderTexture(), pass.GetRenderTexture(), pass.material); + } diff --git a/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader b/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader index af10f41e96..c12b2fe54f 100644 --- a/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader +++ b/unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader @@ -2,24 +2,6 @@ Properties { [MainTexture] _MainTex ("Base (RGB)", 2D) = "white" {} - _DepthLevel ("Depth Level", Range(1, 3)) = 1 - _LensDistortionStrength ("Lens Distortion Strength", Range (-20.0, 20.0)) = 1.0 - _LensDistortionTightness ("Lens Distortion Power", Range (-20.0, 20.0)) = 7.0 - _ZoomPercent ("Zoom Percent", Range (0.0, 5.0)) = 1.0 - - _DistortionIntensityX ("Distort X", Range (0.0, 6.0)) = 1.0 - _DistortionIntensityY ("Distort Y", Range (0.0, 6.0)) = 1.0 - - - _k1 ("K1 polynomial dist coeff", Range (-8.0, 8.0)) = -0.126 - _k2 ("K2 polynomial dist coeff", Range (-8.0, 8.0)) = 0.004 - _k3 ("K3 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0 - _k4 ("K4 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0 - _k5 ("K5 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0 - _k6 ("K6 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0 - - - _OutOfBoundColour ("Outline Color", Color) = (0.0, 0.0, 0.0, 1.0) } SubShader { @@ -33,24 +15,6 @@ uniform sampler2D _MainTex; uniform sampler2D _CameraDepthTexture; - uniform fixed _DepthLevel; - uniform half4 _MainTex_TexelSize; - - uniform float _LensDistortionStrength; - uniform float4 _OutOfBoundColour; - uniform float _LensDistortionTightness; - - uniform float _ZoomPercent; - - uniform float _DistortionIntensityX; - uniform float _DistortionIntensityY; - - uniform float _k1; - uniform float _k2; - uniform float _k3; - uniform float _k4; - uniform float _k5; - uniform float _k6; // uniform float4 _ScreenParams; @@ -90,195 +54,15 @@ fixed4 frag(output o) : COLOR { // depth01 = pow(LinearEyeDepth(depth01), _DepthLevel); - float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))); + float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))); // (LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))) / (_ProjectionParams.z - _ProjectionParams.y); - return fixed4(depth01, depth01, depth01, depth01); + // return fixed4(depth01, depth01, depth01, depth01); // return fixed4(o.uv.x, 0, o.uv.y, 1.0); - // _ScreenParams.x - // float zoom_percent_0 = 1.0; - // float translate_0 = (1.0 - zoom_percent_0) / 2.0; - - // screen centered frag coordinate - float effect = _LensDistortionStrength; - float zoom_offset = (1.0 - _ZoomPercent) / 2.0; - - // float2 offset = float2(0.5, 0.5); - // float2 centered_uv = o.uv*1.0 * zoom_percent_0 - float2(0.5, 0.5) + translate_0; - // float2 centered_uv = (o.uv*2.0 - float2(1.0, 1.0)); - // float2 centered_uv = (o.uv*2.0 * _ZoomPercent - float2(1.0, 1.0)) + float2(zoom_offset, zoom_offset); - float2 centered_uv = o.uv - float2(0.5, 0.5); - centered_uv = o.uv*2.0 - float2(1.0, 1.0); - - zoom_offset = (1.0 - _ZoomPercent); - centered_uv = (o.uv*2.0 * _ZoomPercent - float2(1.0, 1.0)) + float2(zoom_offset, zoom_offset); + // Make sure you read with RenderTextureType.RFloat, 32 bits at R component + return fixed4(depth01, 0, 0, 0); + //return fixed4(1.0, 0.0, 0.0 0.0); - // float2 centered_uv = (o.uv * _ZoomPercent - float2(0.5, 0.5)) + float2(zoom_offset, zoom_offset); - - // float2 centered_uv = (o.uv - float2(0.5, 0.5));// + float2(zoom_offset, zoom_offset); - - // centered_uv = centered_uv * _ZoomPercent + float2(zoom_offset, zoom_offset); - // float d=length(centered_uv); - // float z = sqrt(1.0 - d * d); - - float uv_dot = dot(centered_uv, centered_uv); - float r = sqrt(uv_dot); - float z = sqrt(1.0 - uv_dot * effect); - // float z = sqrt(1.0 - r * r); - float atan_distor = atan2(r, z) / 3.14159; - - float atan_distortX = atan2(r, sqrt(1.0 - uv_dot * _DistortionIntensityX)) / 3.14159; - float atan_distortY = atan2(r, sqrt(1.0 - uv_dot * _DistortionIntensityY)) / 3.14159; - - - // float theta = atan(r); - // float theta_d = theta * (1 + _k1 * pow(theta, 2.0) + _k2 * pow(theta, 4.0) + _k3 * pow(theta, 6.0) + _k4 * pow(theta, 8.0)); - // float focal = 1.0; - // float r_d = focal * theta_d; - - // atan_distor = r_d / r; - - - // atan_distor = - // float phi = atan(centered_uv.y / centered_uv.x); - - // float2 distort_uv = float2(atan_distor*cos(phi)+.5,atan_distor*sin(phi)+.5); - - // // float2 centered_uv = o.uv - offset; - // float4 dot_coord = dot(centered_uv, centered_uv); - // // radius - // float r = sqrt(dot_coord); - - // float z = sqrt(1.0 - dot_coord); - // float pi = 3.14159; - - // float distort = atan(r / z) / pi; - - - // float phi = atan(centered_uv.y / centered_uv.x); - - // // float2 distort_uv = float2(atan_distor * cos(phi), atan_distor * sin(phi)); - // float distort_uv = ( centered_uv / r ) * distort + offset; - // float _LensDistortionTightness = 7.0; - - float2 distort_uv = (centered_uv / r) * atan_distor + float2(0.5, 0.5); - float4 t = tex2D (_MainTex, distort_uv); - //return t; - - const float distortionMagnitude=abs(centered_uv.x*centered_uv.y); - // float smoothDistortionMagnitude = pow(r, _LensDistortionTightness);//use exponential function - float smoothDistortionMagnitude = pow(distortionMagnitude, _LensDistortionTightness); - smoothDistortionMagnitude = 1.0 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0); - - // smoothDistortionMagnitude = atan(r) / 3.14159; - // return float4(smoothDistortionMagnitude, smoothDistortionMagnitude, smoothDistortionMagnitude, 1.0); - - // smoothDistortionMagnitude = 1 + _k1 * pow(distortionMagnitude, 2.0) + _k2 * pow(distortionMagnitude, 4.0) + _k3 * pow(distortionMagnitude,6.0) + _k4 * pow(distortionMagnitude,8.0); - // smoothDistortionMagnitude = 1 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0); - //const float smoothDistortionMagnitude=1-sqrt(1-pow(distortionMagnitude,_LensDistortionTightness));//use circular function - //const float smoothDistortionMagnitude=pow(sin(UNITY_HALF_PI*distortionMagnitude),_LensDistortionTightness);// use sinusoidal - // r = distortionMagnitude; - // float smoothDistortionMagnitude = atan(length(centered_uv)); - // return smoothDistortionMagnitude; - - // Atan distortion - // smoothDistortionMagnitude = atan_distor; - - // r = distortionMagnitude; - - // smoothDistortionMagnitude = 1 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0); - - // float2 uvDistorted = o.uv + centered_uv * smoothDistortionMagnitude * _LensDistortionStrength ; //vector of distortion and add it to original uv - - float zoom_percent = 0.2; - float translate = (1.0 - zoom_percent) / 2.0; - float2 centered_uv_norm = normalize(centered_uv); - // float2 uvDistorted = centered_uv_norm * smoothDistortionMagnitude * _LensDistortionStrength; - // uvDistorted = 0.5 * (uvDistorted + float2(1.0, 1.0)); - - // float2 uvDistorted = (o.uv + centered_uv * smoothDistortionMagnitude * _LensDistortionStrength) ; //vector of distortion and add it to original uv - // float2 uvDistorted = centered_uv * smoothDistortionMagnitude * _LensDistortionStrength / 2.0 + float2(0.5, 0.5) ; - - // Works with zoom for poly distortion - float2 uvDistorted = centered_uv * smoothDistortionMagnitude * _LensDistortionStrength / 2.0 + float2(0.5, 0.5) ; - - // working for atan distortion - // float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5); - - // float2 distorted = ( centered_uv / r ) * - // float2 uvDistorted = ( centered_uv / r ) * float2(atan_distortX, atan_distortY) + float2(0.5, 0.5); - - // float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5); - // float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5); - - // float2 uvDistorted = (centered_uv / r) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5); - //uvDistorted = distort_uv; - float radius = 1.0; - // if (uvDistorted.x > radius) { - // return _OutOfBoundColour; - // } - // else { - // return tex2D(_MainTex, uvDistorted); - // } - // float pl = length(float2(o.pos.x, o.pos.y)); - // return float4(o.pos.x/pl, 0.0, o.pos.y/pl, 1.0); - - // uvDistorted = uvDistorted * 0.3 + float2(0.37, 0.37); - // uvDistorted = uvDistorted * _ZoomPercent + float2(zoom_offset, zoom_offset); - - // uvDistorted = uvDistorted * zoom_percent + float2(translate, translate); - // return float4(uvDistorted.x, 0.0, uvDistorted.y, 1.0); - //Handle out of bound uv - zoom_percent = _ZoomPercent; - translate = (1.0 - zoom_percent); - // centered_uv = (centered_uv / 2.0 + float2(0.5, 0.5)) * _ZoomPercent + translate; - - centered_uv = o.uv*2.0 *_ZoomPercent - float2(1.0, 1.0) + translate; - - float2 no_zoom_centered_uv = o.uv*2.0 - float2(1.0, 1.0); - - float no_zoom_uv_dot = dot(no_zoom_centered_uv, no_zoom_centered_uv); - float no_zoom_uv_len = sqrt(no_zoom_uv_dot); - - float centered_uv_len = length(centered_uv); - - r = centered_uv_len; - - float distort = _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0); - - float2 distorted_uv = ((centered_uv / r)) * distort; - - // distorted_uv = o.uv + distorted_uv; - // float2 distorted_uv = (no_zoom_centered_uv/centered_uv_len) * distort; - centered_uv = (distorted_uv / 2.0 + float2(0.5, 0.5)); - // centered_uv = distorted_uv;// + float2(0.5, 0.5); - - // centered_uv = (centered_uv / 2.0 + float2(0.5, 0.5)); - - // if (centered_uv.x < 0 || centered_uv.x > 1 || centered_uv.y < 0 || centered_uv.y > 1) { - // return _OutOfBoundColour;//uv out of bound so display out of bound color - // } else { - - // return tex2D(_MainTex, centered_uv); - // } - // return fixed4(centered_uv.x, 0, centered_uv.y, 1.0f); - if (uvDistorted.x < 0 || uvDistorted.x > 1 || uvDistorted.y < 0 || uvDistorted.y > 1) { - return _OutOfBoundColour;//uv out of bound so display out of bound color - } else { - // uvDistorted = uvDistorted * zoom_percent + translate; - return tex2D(_MainTex, uvDistorted); - } - // return tex2D(_MainTex, uvDistorted); - - - - - - // return fixed4(o.uv.x, 0, o.uv.y, t.z); - // return t; - // float2 tr = (centered_uv / r) * atan_distor + float2(0.5, 0.5); - // // float2 tr = float2(cos(phi), sin(phi)); - // return float4(tr.x, 0.0, tr.y, 1.0); } ENDCG diff --git a/unity/Assets/Scripts/RenderingManager.cs b/unity/Assets/Scripts/RenderingManager.cs index 1a51e2fe8f..520b3443af 100644 --- a/unity/Assets/Scripts/RenderingManager.cs +++ b/unity/Assets/Scripts/RenderingManager.cs @@ -169,7 +169,7 @@ void Awake() { var depthPass = new RenderToTexture( - new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/DepthBW", cloudRendering = cloudRenderingCapture }, + new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, depthBits = 32, shaderName = "Hidden/DepthBW", cloudRendering = cloudRenderingCapture, renderTextureFormat = RenderTextureFormat.RFloat }, camera: camera ); @@ -195,7 +195,7 @@ void Awake() { // make first _img capture created render to Display this.mainPass = new MultiCapture( - config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, depthBits = 24, cloudRendering = cloudRenderingCapture, toDisplay = IsMainCamera? 0 : null as int?}, + config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, depthBits = 32, cloudRendering = cloudRenderingCapture, toDisplay = IsMainCamera? 0 : null as int?}, camera: camera, passes: new List() { }