Cloud rendering textures hoping to avoid SIGSEV

allenai · Nov 22, 2024 · d7a40c0 · d7a40c0
1 parent ce40508
commit d7a40c0
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 12 deletions.
diff --git a/test_distortion.py b/test_distortion.py
@@ -114,6 +114,8 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None,
         maps = []
 
         print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {distortionMaps.keys()}")
+        # result is dict with 3d arrays in the following:
+        # {'mainCamera': float[height][width][2], 'thirdPartyCameras': float[thirdPartyCameraCount][height][width][2] }
         print(f"[x,y] at (0,0) (bottom left corner) len {distortionMaps['mainCamera'][0][0]}")
         tex_height = len(distortionMaps['mainCamera'])
         tex_width = len(distortionMaps['mainCamera'][0])
@@ -153,10 +155,7 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None,
 
     print(f"Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']}")
     print(f'Error: {evt.metadata["errorMessage"]}')
-    # controller.step(action="MoveAhead")
-    # controller.step(action="MoveAhead")
-    # controller.step(action="MoveAhead")
-
+
     InteractiveControllerPrompt.write_image(controller.last_event, image_dir, "", semantic_segmentation_frame=True, depth_frame=True, color_frame=True, third_party_camera_frames=True, distortion_frame=distortion)
     # input()
 

diff --git a/unity/Assets/Scripts/AgentManager.cs b/unity/Assets/Scripts/AgentManager.cs
@@ -2163,8 +2163,8 @@ public ActionFinished SetDistortionShaderParams(bool mainCamera = true, IEnumera
     }
 
     public class DistortionMapReturn {
-        public float[][,] mainCamera;
-        public List<float[][,]> thirdPartyCameras;
+        public float[][][] mainCamera;
+        public List<float[][][]> thirdPartyCameras;
 
     }
 
@@ -2177,7 +2177,7 @@ public ActionFinished GetDistortionMaps(bool mainCamera = true, IEnumerable<int>
         renderingManagers = renderingManagers.ToList();
 
         var result = new DistortionMapReturn() {
-                thirdPartyCameras = new List<float[][,]>()
+                thirdPartyCameras = new List<float[][][]>()
             };
         foreach (var (index, renderingManager) in renderingManagers) {
             var rt = renderingManager.distortionMap.GetRenderTexture();
@@ -2204,11 +2204,12 @@ public ActionFinished GetDistortionMaps(bool mainCamera = true, IEnumerable<int>
             var y_f = Enumerable.Range(0, maxIndex).Select(i => System.BitConverter.ToSingle(new byte[]{ bytes[i*8 + 4], bytes[ i*8 + 5], bytes[i*8 + 6], bytes[i*8 + 7] }, 0) );
 
             var map = Enumerable.Range(0, rt.height).Select(i => {
-                var row = new float[rt.width, 2];
+                var row = new float[rt.width][];
                 for (int j = 0; j < rt.width; j++) {
                     var val = x_f[i*rt.width + j];
-                    row[j, 0] = val.x;
-                    row[j, 1] = val.y;
+                    row[j] = new float[2];
+                    row[j][0] = val.x;
+                    row[j][1] = val.y;
                 }
                 return row;
             }).ToArray();

diff --git a/unity/Assets/Scripts/CapturePass.cs b/unity/Assets/Scripts/CapturePass.cs
@@ -220,11 +220,25 @@ protected RenderTexture CreateRenderTexture(int width, int height) {
         RenderTexture rt = null;
 
         if (cloudRendering) {
+
+            GraphicsFormat cloudRenderingRTFormat;
+             if (this.renderTextureFormat == RenderTextureFormat.RGFloat) {
+                readTextureFormat = TextureFormat.RGFloat;
+                cloudRenderingRTFormat = GraphicsFormat.R32G32_SFloat;
+            } 
+            else if (this.renderTextureFormat == RenderTextureFormat.RFloat) {
+                readTextureFormat = TextureFormat.RFloat;
+                cloudRenderingRTFormat = GraphicsFormat.R32_SFloat;
+            } 
+            else {
+                this.renderTextureFormat = RenderTextureFormat.ARGB32;
+                cloudRenderingRTFormat = GraphicsFormat.R8G8B8A8_UNorm;
+            }
 
             // Why 0 for depth here ?
-            rt = new RenderTexture(Screen.width, Screen.height, 0, GraphicsFormat.R8G8B8A8_UNorm);
+            rt = new RenderTexture(Screen.width, Screen.height, 0, cloudRenderingRTFormat);
             // TODO: if 0 then RGB24? if not RGB32?
-            readTextureFormat = TextureFormat.RGBA32;
+
 
         //     RenderTexture rt = new RenderTexture(
         //     width: width,