Skip to content

Commit

Permalink
Fixed depth
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlvaroHG
AlvaroHG committed Nov 22, 2024
1 parent 40c978c commit 6684480
Show file tree
Hide file tree
Showing 6 changed files with 26 additions and 233 deletions.
1 change: 0 additions & 1 deletion ai2thor/interact.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,6 @@ def json_write(name, obj):
if condition:
frame = frame_func(event)
if frame is not None:
frame = frame_func(event)
frame = transform(frame)
image_name = os.path.join(
image_dir,
Expand Down
17 changes: 10 additions & 7 deletions test_distortion.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,15 +109,15 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None,
mainCamera=True,
thidPartyCameraIndices=[0]
)
result = evt.metadata['actionReturn']
distortionMaps = evt.metadata['actionReturn']
# keys = [key for (key, val) in result.items()]
maps = []

print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {result.keys()}")
print(f"[x,y] at (0,0) (bottom left corner) len {result['mainCamera'][0][0]}")
tex_height = len(result['mainCamera'])
tex_width = len(result['mainCamera'][0])
print(f"[x,y] at (height, width) (top right corner) len {result['thirdPartyCameras'][0][tex_height-1][tex_width-1]}")
print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {distortionMaps.keys()}")
print(f"[x,y] at (0,0) (bottom left corner) len {distortionMaps['mainCamera'][0][0]}")
tex_height = len(distortionMaps['mainCamera'])
tex_width = len(distortionMaps['mainCamera'][0])
print(f"[x,y] at (height, width) (top right corner) len {distortionMaps['thirdPartyCameras'][0][tex_height-1][tex_width-1]}")

print(f'Error: {evt.metadata["errorMessage"]}')

Expand Down Expand Up @@ -153,7 +153,10 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None,

print(f"Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']}")
print(f'Error: {evt.metadata["errorMessage"]}')

# controller.step(action="MoveAhead")
# controller.step(action="MoveAhead")
# controller.step(action="MoveAhead")

InteractiveControllerPrompt.write_image(controller.last_event, image_dir, "", semantic_segmentation_frame=True, depth_frame=True, color_frame=True, third_party_camera_frames=True, distortion_frame=distortion)
# input()

Expand Down
5 changes: 4 additions & 1 deletion unity/Assets/Scripts/AgentManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
using Newtonsoft.Json.Serialization;
using Thor.Procedural.Data;
using Thor.Rendering;
using UnityEditor.AssetImporters;
using UnityEngine;
using UnityEngine.Experimental.Rendering;
using UnityEngine.Networking;
Expand Down Expand Up @@ -1412,6 +1411,10 @@ string fieldName
Debug.Log($"------- Gettin capture for {captureName} {flag}");
var renderingManager = camera.GetComponent<RenderingManager>();
byte[] bytes = renderingManager.GetCaptureBytes(captureName);
if (captureName == "_depth") {
var localBytesX = new byte[] {bytes[0], bytes[1], bytes[2], bytes[3] };
Debug.Log($"-------- _depth bytes {string.Join(", ", bytes.Take(100))} convert {System.BitConverter.ToSingle(localBytesX, 0)}");
}
payload.Add(new KeyValuePair<string, byte[]>(fieldName, bytes));
}
}
Expand Down
6 changes: 5 additions & 1 deletion unity/Assets/Scripts/CapturePass.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,11 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) {

// commandBuffer.ReleaseTemporaryRT(screenCopyID);
foreach (var pass in this.passDict.Values) {
commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, pass.GetRenderTexture(), pass.material);
if (pass.GetName() == "_depth") {
int depthID = Shader.PropertyToID("_DepthCopyTexture");
}
commandBuffer.Blit(this.GetRenderTexture(), pass.GetRenderTexture(), pass.material);

}


Expand Down
226 changes: 5 additions & 221 deletions unity/Assets/Scripts/ImageSynthesis/Shaders/DepthBW.shader
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,6 @@
Properties
{
[MainTexture] _MainTex ("Base (RGB)", 2D) = "white" {}
_DepthLevel ("Depth Level", Range(1, 3)) = 1
_LensDistortionStrength ("Lens Distortion Strength", Range (-20.0, 20.0)) = 1.0
_LensDistortionTightness ("Lens Distortion Power", Range (-20.0, 20.0)) = 7.0
_ZoomPercent ("Zoom Percent", Range (0.0, 5.0)) = 1.0

_DistortionIntensityX ("Distort X", Range (0.0, 6.0)) = 1.0
_DistortionIntensityY ("Distort Y", Range (0.0, 6.0)) = 1.0


_k1 ("K1 polynomial dist coeff", Range (-8.0, 8.0)) = -0.126
_k2 ("K2 polynomial dist coeff", Range (-8.0, 8.0)) = 0.004
_k3 ("K3 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0
_k4 ("K4 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0
_k5 ("K5 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0
_k6 ("K6 polynomial dist coeff", Range (-8.0, 8.0)) = 0.0


_OutOfBoundColour ("Outline Color", Color) = (0.0, 0.0, 0.0, 1.0)
}
SubShader
{
Expand All @@ -33,24 +15,6 @@

uniform sampler2D _MainTex;
uniform sampler2D _CameraDepthTexture;
uniform fixed _DepthLevel;
uniform half4 _MainTex_TexelSize;

uniform float _LensDistortionStrength;
uniform float4 _OutOfBoundColour;
uniform float _LensDistortionTightness;

uniform float _ZoomPercent;

uniform float _DistortionIntensityX;
uniform float _DistortionIntensityY;

uniform float _k1;
uniform float _k2;
uniform float _k3;
uniform float _k4;
uniform float _k5;
uniform float _k6;


// uniform float4 _ScreenParams;
Expand Down Expand Up @@ -90,195 +54,15 @@
fixed4 frag(output o) : COLOR
{
// depth01 = pow(LinearEyeDepth(depth01), _DepthLevel);
float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv))));
float depth01 = (Linear01Depth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv))));
// (LinearEyeDepth(UNITY_SAMPLE_DEPTH(tex2D(_CameraDepthTexture, o.uv)))) / (_ProjectionParams.z - _ProjectionParams.y);
return fixed4(depth01, depth01, depth01, depth01);
// return fixed4(depth01, depth01, depth01, depth01);
// return fixed4(o.uv.x, 0, o.uv.y, 1.0);
// _ScreenParams.x

// float zoom_percent_0 = 1.0;
// float translate_0 = (1.0 - zoom_percent_0) / 2.0;

// screen centered frag coordinate
float effect = _LensDistortionStrength;
float zoom_offset = (1.0 - _ZoomPercent) / 2.0;

// float2 offset = float2(0.5, 0.5);
// float2 centered_uv = o.uv*1.0 * zoom_percent_0 - float2(0.5, 0.5) + translate_0;
// float2 centered_uv = (o.uv*2.0 - float2(1.0, 1.0));
// float2 centered_uv = (o.uv*2.0 * _ZoomPercent - float2(1.0, 1.0)) + float2(zoom_offset, zoom_offset);
float2 centered_uv = o.uv - float2(0.5, 0.5);
centered_uv = o.uv*2.0 - float2(1.0, 1.0);

zoom_offset = (1.0 - _ZoomPercent);
centered_uv = (o.uv*2.0 * _ZoomPercent - float2(1.0, 1.0)) + float2(zoom_offset, zoom_offset);
// Make sure you read with RenderTextureType.RFloat, 32 bits at R component
return fixed4(depth01, 0, 0, 0);
//return fixed4(1.0, 0.0, 0.0 0.0);

// float2 centered_uv = (o.uv * _ZoomPercent - float2(0.5, 0.5)) + float2(zoom_offset, zoom_offset);

// float2 centered_uv = (o.uv - float2(0.5, 0.5));// + float2(zoom_offset, zoom_offset);

// centered_uv = centered_uv * _ZoomPercent + float2(zoom_offset, zoom_offset);
// float d=length(centered_uv);
// float z = sqrt(1.0 - d * d);

float uv_dot = dot(centered_uv, centered_uv);
float r = sqrt(uv_dot);
float z = sqrt(1.0 - uv_dot * effect);
// float z = sqrt(1.0 - r * r);
float atan_distor = atan2(r, z) / 3.14159;

float atan_distortX = atan2(r, sqrt(1.0 - uv_dot * _DistortionIntensityX)) / 3.14159;
float atan_distortY = atan2(r, sqrt(1.0 - uv_dot * _DistortionIntensityY)) / 3.14159;


// float theta = atan(r);
// float theta_d = theta * (1 + _k1 * pow(theta, 2.0) + _k2 * pow(theta, 4.0) + _k3 * pow(theta, 6.0) + _k4 * pow(theta, 8.0));
// float focal = 1.0;
// float r_d = focal * theta_d;

// atan_distor = r_d / r;


// atan_distor =
// float phi = atan(centered_uv.y / centered_uv.x);

// float2 distort_uv = float2(atan_distor*cos(phi)+.5,atan_distor*sin(phi)+.5);

// // float2 centered_uv = o.uv - offset;
// float4 dot_coord = dot(centered_uv, centered_uv);
// // radius
// float r = sqrt(dot_coord);

// float z = sqrt(1.0 - dot_coord);
// float pi = 3.14159;

// float distort = atan(r / z) / pi;


// float phi = atan(centered_uv.y / centered_uv.x);

// // float2 distort_uv = float2(atan_distor * cos(phi), atan_distor * sin(phi));
// float distort_uv = ( centered_uv / r ) * distort + offset;
// float _LensDistortionTightness = 7.0;

float2 distort_uv = (centered_uv / r) * atan_distor + float2(0.5, 0.5);
float4 t = tex2D (_MainTex, distort_uv);
//return t;

const float distortionMagnitude=abs(centered_uv.x*centered_uv.y);
// float smoothDistortionMagnitude = pow(r, _LensDistortionTightness);//use exponential function
float smoothDistortionMagnitude = pow(distortionMagnitude, _LensDistortionTightness);
smoothDistortionMagnitude = 1.0 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0);

// smoothDistortionMagnitude = atan(r) / 3.14159;
// return float4(smoothDistortionMagnitude, smoothDistortionMagnitude, smoothDistortionMagnitude, 1.0);

// smoothDistortionMagnitude = 1 + _k1 * pow(distortionMagnitude, 2.0) + _k2 * pow(distortionMagnitude, 4.0) + _k3 * pow(distortionMagnitude,6.0) + _k4 * pow(distortionMagnitude,8.0);
// smoothDistortionMagnitude = 1 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0);
//const float smoothDistortionMagnitude=1-sqrt(1-pow(distortionMagnitude,_LensDistortionTightness));//use circular function
//const float smoothDistortionMagnitude=pow(sin(UNITY_HALF_PI*distortionMagnitude),_LensDistortionTightness);// use sinusoidal
// r = distortionMagnitude;
// float smoothDistortionMagnitude = atan(length(centered_uv));
// return smoothDistortionMagnitude;

// Atan distortion
// smoothDistortionMagnitude = atan_distor;

// r = distortionMagnitude;

// smoothDistortionMagnitude = 1 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0);

// float2 uvDistorted = o.uv + centered_uv * smoothDistortionMagnitude * _LensDistortionStrength ; //vector of distortion and add it to original uv

float zoom_percent = 0.2;
float translate = (1.0 - zoom_percent) / 2.0;
float2 centered_uv_norm = normalize(centered_uv);
// float2 uvDistorted = centered_uv_norm * smoothDistortionMagnitude * _LensDistortionStrength;
// uvDistorted = 0.5 * (uvDistorted + float2(1.0, 1.0));

// float2 uvDistorted = (o.uv + centered_uv * smoothDistortionMagnitude * _LensDistortionStrength) ; //vector of distortion and add it to original uv
// float2 uvDistorted = centered_uv * smoothDistortionMagnitude * _LensDistortionStrength / 2.0 + float2(0.5, 0.5) ;

// Works with zoom for poly distortion
float2 uvDistorted = centered_uv * smoothDistortionMagnitude * _LensDistortionStrength / 2.0 + float2(0.5, 0.5) ;

// working for atan distortion
// float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5);

// float2 distorted = ( centered_uv / r ) *
// float2 uvDistorted = ( centered_uv / r ) * float2(atan_distortX, atan_distortY) + float2(0.5, 0.5);

// float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5);
// float2 uvDistorted = ( centered_uv / r ) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5);

// float2 uvDistorted = (centered_uv / r) * smoothDistortionMagnitude * _LensDistortionStrength + float2(0.5, 0.5);
//uvDistorted = distort_uv;
float radius = 1.0;
// if (uvDistorted.x > radius) {
// return _OutOfBoundColour;
// }
// else {
// return tex2D(_MainTex, uvDistorted);
// }
// float pl = length(float2(o.pos.x, o.pos.y));
// return float4(o.pos.x/pl, 0.0, o.pos.y/pl, 1.0);

// uvDistorted = uvDistorted * 0.3 + float2(0.37, 0.37);
// uvDistorted = uvDistorted * _ZoomPercent + float2(zoom_offset, zoom_offset);

// uvDistorted = uvDistorted * zoom_percent + float2(translate, translate);
// return float4(uvDistorted.x, 0.0, uvDistorted.y, 1.0);
//Handle out of bound uv
zoom_percent = _ZoomPercent;
translate = (1.0 - zoom_percent);
// centered_uv = (centered_uv / 2.0 + float2(0.5, 0.5)) * _ZoomPercent + translate;

centered_uv = o.uv*2.0 *_ZoomPercent - float2(1.0, 1.0) + translate;

float2 no_zoom_centered_uv = o.uv*2.0 - float2(1.0, 1.0);

float no_zoom_uv_dot = dot(no_zoom_centered_uv, no_zoom_centered_uv);
float no_zoom_uv_len = sqrt(no_zoom_uv_dot);

float centered_uv_len = length(centered_uv);

r = centered_uv_len;

float distort = _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0);

float2 distorted_uv = ((centered_uv / r)) * distort;

// distorted_uv = o.uv + distorted_uv;
// float2 distorted_uv = (no_zoom_centered_uv/centered_uv_len) * distort;
centered_uv = (distorted_uv / 2.0 + float2(0.5, 0.5));
// centered_uv = distorted_uv;// + float2(0.5, 0.5);

// centered_uv = (centered_uv / 2.0 + float2(0.5, 0.5));

// if (centered_uv.x < 0 || centered_uv.x > 1 || centered_uv.y < 0 || centered_uv.y > 1) {
// return _OutOfBoundColour;//uv out of bound so display out of bound color
// } else {

// return tex2D(_MainTex, centered_uv);
// }
// return fixed4(centered_uv.x, 0, centered_uv.y, 1.0f);
if (uvDistorted.x < 0 || uvDistorted.x > 1 || uvDistorted.y < 0 || uvDistorted.y > 1) {
return _OutOfBoundColour;//uv out of bound so display out of bound color
} else {
// uvDistorted = uvDistorted * zoom_percent + translate;
return tex2D(_MainTex, uvDistorted);
}
// return tex2D(_MainTex, uvDistorted);





// return fixed4(o.uv.x, 0, o.uv.y, t.z);
// return t;
// float2 tr = (centered_uv / r) * atan_distor + float2(0.5, 0.5);
// // float2 tr = float2(cos(phi), sin(phi));
// return float4(tr.x, 0.0, tr.y, 1.0);
}

ENDCG
Expand Down
4 changes: 2 additions & 2 deletions unity/Assets/Scripts/RenderingManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ void Awake() {


var depthPass = new RenderToTexture(
new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/DepthBW", cloudRendering = cloudRenderingCapture },
new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, depthBits = 32, shaderName = "Hidden/DepthBW", cloudRendering = cloudRenderingCapture, renderTextureFormat = RenderTextureFormat.RFloat },
camera: camera
);

Expand All @@ -195,7 +195,7 @@ void Awake() {

// make first _img capture created render to Display
this.mainPass = new MultiCapture(
config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, depthBits = 24, cloudRendering = cloudRenderingCapture, toDisplay = IsMainCamera? 0 : null as int?},
config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, depthBits = 32, cloudRendering = cloudRenderingCapture, toDisplay = IsMainCamera? 0 : null as int?},
camera: camera,
passes: new List<RenderToTexture>() {
}
Expand Down

0 comments on commit 6684480

Please sign in to comment.