From 9e5aca5a50112caa18967305639e498b827f656f Mon Sep 17 00:00:00 2001 From: AlvaroHG Date: Wed, 20 Nov 2024 18:57:43 -0800 Subject: [PATCH] All the refactor changes, imagesynthesis semi-deprecateed --- unity/Assets/Editor/Build.cs | 4 +- .../Resources/ThirdPartyCameraTemplate.prefab | 13 + unity/Assets/Scripts/AgentManager.cs | 388 ++++++++++++++---- unity/Assets/Scripts/ArmAgentController.cs | 2 +- .../Assets/Scripts/BaseFPSAgentController.cs | 11 +- unity/Assets/Scripts/CapturePass.cs | 247 +++++++++-- .../Scripts/ImageSynthesis/ImageSynthesis.cs | 231 ++++++----- unity/Assets/Scripts/RenderingManager.cs | 268 ++++++++++++ .../Assets/Scripts/StretchAgentController.cs | 2 +- .../Prefabs/FPSController.prefab | 17 +- unity/ProjectSettings/ProjectSettings.asset | 2 +- 11 files changed, 951 insertions(+), 234 deletions(-) create mode 100644 unity/Assets/Scripts/RenderingManager.cs diff --git a/unity/Assets/Editor/Build.cs b/unity/Assets/Editor/Build.cs index d4ca3a8068..ef15544189 100644 --- a/unity/Assets/Editor/Build.cs +++ b/unity/Assets/Editor/Build.cs @@ -181,8 +181,8 @@ private static List GetAllScenePaths() } } - return scenes; - // return scenes.Where(x => x.Contains("FloorPlan1_") || x.Contains("FloorPlan28_") || x.Contains("Procedural") || x.Contains("ProceduralAB") || x.Contains("Calibration_Room")).ToList(); + // return scenes; + return scenes.Where(x => x.Contains("FloorPlan1_") || x.Contains("FloorPlan28_") || x.Contains("Procedural") || x.Contains("ProceduralAB") || x.Contains("Calibration_Room")).ToList(); // uncomment for faster builds for testing return scenes; //.Where(x => x.Contains("Procedural.unity") || x.Contains("Procedural.unity")).ToList(); //.Where(x => x.Contains("FloorPlan1_") || x.Contains("Procedural")).ToList(); } diff --git a/unity/Assets/Resources/ThirdPartyCameraTemplate.prefab b/unity/Assets/Resources/ThirdPartyCameraTemplate.prefab index 72f09e6084..eaa63241b9 100644 --- a/unity/Assets/Resources/ThirdPartyCameraTemplate.prefab +++ b/unity/Assets/Resources/ThirdPartyCameraTemplate.prefab @@ -12,6 +12,7 @@ GameObject: - component: {fileID: 5990649868303501632} - component: {fileID: 3402215512074657615} - component: {fileID: 5576228602605313083} + - component: {fileID: 5929461731447045472} m_Layer: 0 m_Name: ThirdPartyCameraTemplate m_TagString: Untagged @@ -148,3 +149,15 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: dd09293d18a1af44d95b8ed5c5418f95, type: 3} m_Name: m_EditorClassIdentifier: +--- !u!114 &5929461731447045472 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7692911009727723218} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 68871f0bc5fb742138947ad4c5e73f7d, type: 3} + m_Name: + m_EditorClassIdentifier: diff --git a/unity/Assets/Scripts/AgentManager.cs b/unity/Assets/Scripts/AgentManager.cs index 5525782476..2a8f6f97b6 100644 --- a/unity/Assets/Scripts/AgentManager.cs +++ b/unity/Assets/Scripts/AgentManager.cs @@ -17,6 +17,7 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; using Thor.Procedural.Data; +using Thor.Rendering; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Networking; @@ -52,6 +53,8 @@ public class AgentManager : MonoBehaviour, ActionInvokable { private bool renderNormalsImage; private bool renderFlowImage; private bool renderDistortionImage; + + private IEnumerable activeCapturePassList; private Socket sock = null; [SerializeField] @@ -184,8 +187,9 @@ void Start() { // needed to ensure that the com.unity.simulation.capture package // gets initialized var instance = Manager.Instance; - Camera camera = this.primaryAgent.gameObject.GetComponentInChildren(); - camera.targetTexture = createRenderTexture(Screen.width, Screen.height); + + // Camera camera = this.primaryAgent.gameObject.GetComponentInChildren(); + // camera.targetTexture = createRenderTexture(Screen.width, Screen.height); #endif primaryAgent.actionDuration = this.actionDuration; @@ -339,6 +343,17 @@ public void Initialize(ServerAction action) { this.renderDistortionImage = action.renderDistortionImage; this.fastActionEmit = action.fastActionEmit; + // TODO Refactor so that we know where these strings come from + activeCapturePassList = new Dictionary() { + {"_img", true}, + {"_depth", this.renderDepthImage}, + {"_id", this.renderInstanceSegmentation}, + {"_class", this.renderSemanticSegmentation}, + {"_normals", this.renderNormalsImage}, + {"_flow", this.renderFlowImage}, + {"_distortion", this.renderDistortionImage} + }.Where(x => x.Value).Select(x => x.Key); + PhysicsSceneManager.SetDefaultSimulationParams(action.defaultPhysicsSimulationParams); Time.fixedDeltaTime = ( action.defaultPhysicsSimulationParams?.fixedDeltaTime @@ -370,6 +385,8 @@ public void Initialize(ServerAction action) { } else { ResetSceneBounds(); } + // this.updateImageSynthesis(true, activeCapturePassList); + this.updateRenderingManagers(activeCapturePassList, true); this.agentManagerState = AgentState.ActionComplete; } @@ -539,13 +556,15 @@ GameObject go in UnityEngine ResetSceneBounds(); } + + // TODO: deprecate this make people call AddThirdPartyCamera for unified tpcam code public void registerAsThirdPartyCamera(Camera camera) { this.thirdPartyCameras.Add(camera); #if PLATFORM_CLOUD_RENDERING - camera.targetTexture = createRenderTexture( - this.primaryAgent.m_Camera.pixelWidth, - this.primaryAgent.m_Camera.targetTexture.height - ); + // camera.targetTexture = createRenderTexture( + // this.primaryAgent.m_Camera.pixelWidth, + // this.primaryAgent.m_Camera.targetTexture.height + // ); #endif } @@ -559,9 +578,19 @@ private float ClampFieldOfView( return (fov <= min || fov > max) ? defaultVal : fov; } - public void updateImageSynthesis(bool status) { + public void updateRenderingManagers(IEnumerable activePassList, bool cameraChange = false) { + var renderingManagers = agents.Select(agent => agent.m_Camera.GetComponent()) + .Concat(thirdPartyCameras.Select(cam => cam.GetComponent())); + foreach (var renderingManager in renderingManagers) { + Debug.Log($"------ updateRenderingManagers enablepasses for {renderingManager.transform.parent.name}"); + renderingManager.EnablePasses(activePassList, cameraChange); + } + } + + public void updateImageSynthesis(bool status, IEnumerable activePassList, bool cameraChange = false) { foreach (var agent in this.agents) { - agent.updateImageSynthesis(status); + agent.updateImageSynthesis(status, activePassList); + // updateThirdPartyCameraImageSynthesis(status, activePassList, cameraChange); } } @@ -574,8 +603,16 @@ public void updateThirdPartyCameraImageSynthesis(bool status) { if (imageSynthesis == null) { gameObject.AddComponent(typeof(ImageSynthesis)); } + + // gameObject.AddComponent(typeof(RenderCapture)) imageSynthesis = gameObject.GetComponentInChildren() as ImageSynthesis; + + // var renderingManager = gameObject.GetComponentInChildren(); + + // if (cameraChange) { + // renderingManager.OnCameraChange() + // renderingManager.a imageSynthesis.enabled = status; } } @@ -704,6 +741,20 @@ private void updateCameraProperties( if (imageSynthesis != null && imageSynthesis.enabled) { imageSynthesis.OnCameraChange(); } + var renderingManager = camera.GetComponent(); + // renderingManager.OnCameraChange(); + + activeCapturePassList = new Dictionary() { + {"_img", true}, + {"_depth", this.renderDepthImage}, + {"_id", this.renderInstanceSegmentation}, + {"_class", this.renderSemanticSegmentation}, + {"_normals", this.renderNormalsImage}, + {"_flow", this.renderFlowImage}, + {"_distortion", this.renderDistortionImage} + }.Where(x => x.Value).Select(x => x.Key); + + this.updateRenderingManagers(activeCapturePassList, true); this.activeAgent().actionFinished(success: true); } @@ -781,12 +832,29 @@ public void AddThirdPartyCamera( ) { gameObject.AddComponent(typeof(ImageSynthesis)); } + // activeCapturePassList = new Dictionary() { + // {"_img", true}, + // {"_depth", this.renderDepthImage}, + // {"_id", this.renderInstanceSegmentation}, + // {"_class", this.renderSemanticSegmentation}, + // {"_normals", this.renderNormalsImage}, + // {"_flow", this.renderFlowImage}, + // {"_distortion", this.renderDistortionImage} + // }.Where(x => x.Value).Select(x => x.Key); + + // // no on camera change because camera has not been changed, oncamerachange gets called on updateCameraProperties + // // consider unifying? + // this.updateRenderingManagers(activeCapturePassList); #if PLATFORM_CLOUD_RENDERING - camera.targetTexture = createRenderTexture( - this.primaryAgent.m_Camera.pixelWidth, - this.primaryAgent.m_Camera.targetTexture.height - ); + + // this now should happen in call updateCameraProperties which calls updateRenderingManagers + // which calls RenderingManager.OnCameraChange and recreates renderTexture + + // camera.targetTexture = createRenderTexture( + // this.primaryAgent.m_Camera.pixelWidth, + // this.primaryAgent.m_Camera.targetTexture.height + // ); #endif //default to no post processing needed on third party cameras @@ -968,12 +1036,19 @@ private void addAgent(ServerAction action) { UpdateAgentColor(agent, agentColors[this.agents.Count]); #if PLATFORM_CLOUD_RENDERING - agent.m_Camera.targetTexture = createRenderTexture( - this.primaryAgent.m_Camera.targetTexture.width, - this.primaryAgent.m_Camera.targetTexture.height - ); + + // this should happen after ProcessControlCommand of AgentManager which is the only thing that calls + // this function via addAgents + // agent.m_Camera.targetTexture = createRenderTexture( + // this.primaryAgent.m_Camera.targetTexture.width, + // this.primaryAgent.m_Camera.targetTexture.height + // ); #endif agent.ProcessControlCommand(action.dynamicServerAction); + // TODO: instead of calling agent.ProcessControlCommand call + // agentmanager ProcessControlCommand which calls RenderingManager updates automatically + // var rm = agent.m_Camera.GetComponent(); + // rm.EnablePasses() } private Vector3 agentStartPosition(BaseFPSAgentController agent) { @@ -1078,6 +1153,15 @@ private void Update() { physicsSceneManager.isSceneAtRest = true; // assume the scene is at rest by default } + private void captureScreenAsyncNew( + List> payload, + string key, + Camera camera + ) { + var rm = camera.GetComponent(); + rm.GetCaptureAsync("_img", payload, key); + } + private void captureScreenAsync( List> payload, string key, @@ -1085,7 +1169,7 @@ Camera camera ) { RenderTexture tt = camera.targetTexture; RenderTexture.active = tt; - camera.Render(); + // camera.Render(); AsyncGPUReadback.Request( tt, 0, @@ -1110,38 +1194,74 @@ private byte[] captureScreen() { ); readPixelsRect = new Rect(0, 0, UnityEngine.Screen.width, UnityEngine.Screen.height); } + tex.ReadPixels(readPixelsRect, 0, 0); // tex.Apply(); return tex.GetRawTextureData(); } + private byte[] captureCamera(Camera camera) { + if (tex.height != UnityEngine.Screen.height || tex.width != UnityEngine.Screen.width) { + tex = new Texture2D( + UnityEngine.Screen.width, + UnityEngine.Screen.height, + TextureFormat.RGB24, + false + ); + readPixelsRect = new Rect(0, 0, UnityEngine.Screen.width, UnityEngine.Screen.height); + } + var renderingManager = camera.GetComponent(); + + var prevActiveTex = RenderTexture.active; + + var renderTexture = renderingManager.GetPassRenderTexture("_img"); + RenderTexture.active = renderTexture; + + tex.ReadPixels(readPixelsRect, 0, 0); + // camera.Render(); + + RenderTexture.active = prevActiveTex; + // tex.Apply(); + var bytes = tex.GetRawTextureData(); + + Debug.Log($"-------- captureCamera {bytes.Length}"); + return bytes; + } + private void addThirdPartyCameraImage(List> payload, Camera camera) { #if PLATFORM_CLOUD_RENDERING - captureScreenAsync(payload, "image-thirdParty-camera", camera); + // captureScreenAsync(payload, "image-thirdParty-camera", camera); + captureScreenAsyncNew(payload, "image-thirdParty-camera", camera); #else - RenderTexture.active = camera.activeTexture; - camera.Render(); - payload.Add(new KeyValuePair("image-thirdParty-camera", captureScreen())); + // RenderTexture.active = camera.activeTexture; + // camera.Render(); + // payload.Add(new KeyValuePair("image-thirdParty-camera", captureScreen())); + + payload.Add(new KeyValuePair("image-thirdParty-camera", captureCamera(camera))); #endif } private void addImage(List> payload, BaseFPSAgentController agent) { if (this.renderImage) { #if PLATFORM_CLOUD_RENDERING - captureScreenAsync(payload, "image", agent.m_Camera); + // captureScreenAsync(payload, "image", agent.m_Camera); + captureScreenAsyncNew(payload, "image", agent.m_Camera); #else - // XXX may not need this since we call render in captureScreenAsync - if (this.agents.Count > 1 || this.thirdPartyCameras.Count > 0) { - RenderTexture.active = agent.m_Camera.activeTexture; - agent.m_Camera.Render(); - } - payload.Add(new KeyValuePair("image", captureScreen())); + // // XXX may not need this since we call render in captureScreenAsync + // if (this.agents.Count > 1 || this.thirdPartyCameras.Count > 0) { + // RenderTexture.active = agent.m_Camera.activeTexture; + // agent.m_Camera.Render(); + // } + // payload.Add(new KeyValuePair("image", captureScreen())); + payload.Add(new KeyValuePair("image", captureCamera(agent.m_Camera))); #endif } } private void resetImageSynthesis(Camera camera) { ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren(); + var renderingManager = camera.GetComponent(); + renderingManager.OnCameraChange(); if (imageSynthesis != null && imageSynthesis.enabled) { imageSynthesis.OnCameraChange(); imageSynthesis.OnSceneChange(); @@ -1175,7 +1295,7 @@ public IEnumerator WaitOnResolutionChange(int width, int height) { + $" This is likely due to Unity not supporting the requested resolution and instead using the closest possible resolution." ); } - + // TODO once image synthesis is deprecated change to updateRenderingManagers this.resetAllImageSynthesis(); this.primaryAgent.actionFinished(success); } @@ -1198,21 +1318,24 @@ public void ChangeResolution(int x, int y) { "current screen resolution pre change: " + Screen.width + " height" + Screen.height ); #if PLATFORM_CLOUD_RENDERING - foreach (var agent in this.agents) - { - var rt = agent.m_Camera.targetTexture; - rt.Release(); - Destroy(rt); - agent.m_Camera.targetTexture = createRenderTexture(x, y); - } - foreach (var camera in this.thirdPartyCameras) - { - var rt = camera.targetTexture; - rt.Release(); - Destroy(rt); - camera.targetTexture = createRenderTexture(x, y); - } + // This should happen automatically with each RenderingManager for it's capture passes + // in resetImageSynthesis in coroutine for screen as it detects new screen parameters + // foreach (var agent in this.agents) + // { + // var rt = agent.m_Camera.targetTexture; + // rt.Release(); + // Destroy(rt); + // agent.m_Camera.targetTexture = createRenderTexture(x, y); + // } + + // foreach (var camera in this.thirdPartyCameras) + // { + // var rt = camera.targetTexture; + // rt.Release(); + // Destroy(rt); + // camera.targetTexture = createRenderTexture(x, y); + // } #endif StartCoroutine(WaitOnResolutionChange(width: x, height: y)); } @@ -1229,10 +1352,16 @@ ref MetadataWrapper metadata "Object Image not available in imagesynthesis - returning empty image" ); } - byte[] bytes = agent.ImageSynthesis.Encode("_id"); + var renderingManager = agent.m_Camera.GetComponent(); + + byte[] bytes = renderingManager.GetCaptureBytes("_id"); + // byte[] bytes = agent.ImageSynthesis.Encode("_id"); + payload.Add(new KeyValuePair("image_ids", bytes)); List colors = new List(); + + Debug.Log($"----- imageSynt null? {agent.ImageSynthesis == null} { agent.ImageSynthesis} colorids null {agent.ImageSynthesis.colorIds == null}"); foreach (Color key in agent.ImageSynthesis.colorIds.Keys) { ColorId cid = new ColorId(); cid.color = new ushort[] @@ -1265,6 +1394,21 @@ string fieldName } } + private void addCapture( + List> payload, + Camera camera, + bool flag, + string captureName, + string fieldName + ) { + if (flag) { + Debug.Log($"------- Gettin capture for {captureName} {flag}"); + var renderingManager = camera.GetComponent(); + byte[] bytes = renderingManager.GetCaptureBytes(captureName); + payload.Add(new KeyValuePair(fieldName, bytes)); + } + } + // Used for benchmarking only the server-side // no call is made to the Python side private IEnumerator EmitFrameNoClient() { @@ -1334,53 +1478,98 @@ bool shouldRenderImageSynthesis cMetadata.fieldOfView = camera.fieldOfView; cameraMetadata[i] = cMetadata; - addThirdPartyCameraImage(renderPayload, camera); + // addThirdPartyCameraImage(renderPayload, camera); + renderPayload.Add(new KeyValuePair("image-thirdParty-camera", captureCamera(camera))); + if (shouldRenderImageSynthesis) { ImageSynthesis imageSynthesis = camera.gameObject.GetComponentInChildren() as ImageSynthesis; - addImageSynthesisImage( + + addCapture( renderPayload, - imageSynthesis, + camera, this.renderDepthImage, "_depth", "image_thirdParty_depth" ); - addImageSynthesisImage( + addCapture( renderPayload, - imageSynthesis, + camera, this.renderNormalsImage, "_normals", "image_thirdParty_normals" ); - addImageSynthesisImage( + addCapture( renderPayload, - imageSynthesis, + camera, this.renderInstanceSegmentation, "_id", "image_thirdParty_image_ids" ); - addImageSynthesisImage( + addCapture( renderPayload, - imageSynthesis, + camera, this.renderSemanticSegmentation, "_class", "image_thirdParty_classes" ); - addImageSynthesisImage( + addCapture( renderPayload, - imageSynthesis, - this.renderSemanticSegmentation, + camera, + this.renderFlowImage, "_flow", "image_thirdParty_flow" ); // XXX fix this in a bit - addImageSynthesisImage( + addCapture( renderPayload, - imageSynthesis, + camera, this.renderDistortionImage, "_distortion", "image_thirdParty_distortion" ); + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderDepthImage, + // "_depth", + // "image_thirdParty_depth" + // ); + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderNormalsImage, + // "_normals", + // "image_thirdParty_normals" + // ); + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderInstanceSegmentation, + // "_id", + // "image_thirdParty_image_ids" + // ); + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderSemanticSegmentation, + // "_class", + // "image_thirdParty_classes" + // ); + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderSemanticSegmentation, + // "_flow", + // "image_thirdParty_flow" + // ); // XXX fix this in a bit + // addImageSynthesisImage( + // renderPayload, + // imageSynthesis, + // this.renderDistortionImage, + // "_distortion", + // "image_thirdParty_distortion" + // ); } } } @@ -1398,42 +1587,78 @@ bool shouldRenderImageSynthesis if (shouldRender) { addImage(renderPayload, agent); if (shouldRenderImageSynthesis) { - addImageSynthesisImage( + addCapture( renderPayload, - agent.imageSynthesis, + agent.m_Camera, this.renderDepthImage, "_depth", "image_depth" ); - addImageSynthesisImage( + addCapture( renderPayload, - agent.imageSynthesis, + agent.m_Camera, this.renderNormalsImage, "_normals", "image_normals" ); addObjectImage(renderPayload, agent, ref metadata); - addImageSynthesisImage( + addCapture( renderPayload, - agent.imageSynthesis, + agent.m_Camera, this.renderSemanticSegmentation, "_class", "image_classes" ); - addImageSynthesisImage( + addCapture( renderPayload, - agent.imageSynthesis, + agent.m_Camera, this.renderFlowImage, "_flow", "image_flow" ); - addImageSynthesisImage( + addCapture( renderPayload, - agent.imageSynthesis, + agent.m_Camera, this.renderDistortionImage, "_distortion", "image_distortion" ); + // addImageSynthesisImage( + // renderPayload, + // agent.imageSynthesis, + // this.renderDepthImage, + // "_depth", + // "image_depth" + // ); + // addImageSynthesisImage( + // renderPayload, + // agent.imageSynthesis, + // this.renderNormalsImage, + // "_normals", + // "image_normals" + // ); + // addObjectImage(renderPayload, agent, ref metadata); + // addImageSynthesisImage( + // renderPayload, + // agent.imageSynthesis, + // this.renderSemanticSegmentation, + // "_class", + // "image_classes" + // ); + // addImageSynthesisImage( + // renderPayload, + // agent.imageSynthesis, + // this.renderFlowImage, + // "_flow", + // "image_flow" + // ); + // addImageSynthesisImage( + // renderPayload, + // agent.imageSynthesis, + // this.renderDistortionImage, + // "_distortion", + // "image_distortion" + // ); } metadata.thirdPartyCameras = cameraMetadata; } @@ -1778,16 +2003,22 @@ public void ProcessControlCommand(DynamicServerAction controlCommand) { this.renderInstanceSegmentation = true; } + + /// TODOremove if? if ( this.renderDepthImage || this.renderSemanticSegmentation || this.renderInstanceSegmentation || this.renderNormalsImage + || this.renderDistortionImage ) { - updateImageSynthesis(true); + + + updateImageSynthesis(true, activeCapturePassList); // Call with flags updateThirdPartyCameraImageSynthesis(true); } + updateRenderingManagers(activeCapturePassList, true); // let's look in the agent's set of actions for the action this.activeAgent().ProcessControlCommand(controlCommand: controlCommand); @@ -1859,13 +2090,18 @@ public void SetCriticalErrorState() { public ActionFinished SetDistortionShaderParams(bool mainCamera = true, IEnumerable thidPartyCameraIndices = null, float zoomPercent = 1.0f, float k1 = 0.0f, float k2 = 0.0f, float k3 = 0.0f, float k4 = 0.0f, float strength = 1.0f, float intensityX = 1.0f, float intensityY = 1.0f) { - IEnumerable imageSynths = mainCamera ? new List() {this.primaryAgent.imageSynthesis} : new List(); - imageSynths = thidPartyCameraIndices != null ? imageSynths.Concat(this.thirdPartyCameras.Where((cam, i) => thidPartyCameraIndices.Contains(i)).Select(cam => cam.GetComponent())) : imageSynths; - if (imageSynths.Any(x => x == null)) { - return new ActionFinished(success: false, errorMessage: "No imageSynthesis, make sure you pass 'renderDistortionImage = true' to the agent constructor."); - } - foreach (var imageSynthesis in imageSynths) { - var material = imageSynthesis.distortionMaterial; + IEnumerable renderingManagers = mainCamera ? new List() {this.primaryAgent.m_Camera.GetComponent()} : new List(); + renderingManagers = thidPartyCameraIndices != null ? renderingManagers.Concat(this.thirdPartyCameras.Where((cam, i) => thidPartyCameraIndices.Contains(i)).Select(cam => cam.GetComponent())) : renderingManagers; + // TODO remove as this should never happen with new flow + // if (renderingManagers.Any(x => x == null)) { + // return new ActionFinished(success: false, errorMessage: "No RenderingManager, make sure you pass 'renderDistortionImage = true' to the agent constructor."); + // } + foreach (var renderingManager in renderingManagers) { + var distortion = renderingManager.GetCapturePass("_distortion"); + if (distortion == null) { + return new ActionFinished(success: false, errorMessage: "No Distortion pass, make sure you pass 'renderDistortionImage = true' to the agent constructor."); + } + var material = distortion.material; material.SetFloat("_ZoomPercent", zoomPercent); material.SetFloat("_k1", k1); material.SetFloat("_k2", k2); diff --git a/unity/Assets/Scripts/ArmAgentController.cs b/unity/Assets/Scripts/ArmAgentController.cs index 5e3aea7317..ef2a893846 100644 --- a/unity/Assets/Scripts/ArmAgentController.cs +++ b/unity/Assets/Scripts/ArmAgentController.cs @@ -29,7 +29,7 @@ public ActionFinished ToggleMagnetVisibility(bool? visible = null) { return ActionFinished.Success; } - public override void updateImageSynthesis(bool status) { + public override void updateImageSynthesis(bool status, IEnumerable activePassList = null) { base.updateImageSynthesis(status); // updateImageSynthesis is run in BaseFPSController's Initialize method after the diff --git a/unity/Assets/Scripts/BaseFPSAgentController.cs b/unity/Assets/Scripts/BaseFPSAgentController.cs index a2ef43c4fc..9ba6126910 100644 --- a/unity/Assets/Scripts/BaseFPSAgentController.cs +++ b/unity/Assets/Scripts/BaseFPSAgentController.cs @@ -2725,11 +2725,18 @@ public virtual MetadataWrapper generateMetadataWrapper() { return metaMessage; } - public virtual void updateImageSynthesis(bool status) { + public virtual void updateImageSynthesis(bool status, IEnumerable activePassList = null) { if (this.imageSynthesis == null) { - imageSynthesis = + this.imageSynthesis = this.m_Camera.gameObject.GetComponent() as ImageSynthesis; } + // this.imageSynthesis.EnablePasses(activePassList); + + // var renderingManager = this.m_Camera.gameObject.GetComponent(); + // renderingManager.EnablePasses(activePassList); + // if (cameraChange) { + // renderingManager.OnCameraChange(); + // } imageSynthesis.enabled = status; } diff --git a/unity/Assets/Scripts/CapturePass.cs b/unity/Assets/Scripts/CapturePass.cs index da9c3012ec..734f38af52 100644 --- a/unity/Assets/Scripts/CapturePass.cs +++ b/unity/Assets/Scripts/CapturePass.cs @@ -4,8 +4,12 @@ using System.Collections.Specialized; using System.IO; using System.Linq; +using System.Runtime.InteropServices; +using UnityEditor; using UnityEngine; using UnityEngine.Rendering; +using UnityEngine.Experimental.Rendering; +using System.Security.Cryptography; // // @TODO: // // . support custom color wheels in optical flow via lookup textures @@ -35,11 +39,36 @@ public enum ReplacelementMode { Normals = 4, Flow = 5, }; + + // public class PassName { + // const string Image = "_img", + // Depth = "_depth" + // } + + public class CaptureConfig { + public string name; + + public int antiAliasLevel = 1; + + public string shaderName; + + public ReplacelementMode replacementMode; + + public int? toDisplay = 0; + + public bool cloudRendering; + + + } public interface ICapturePass { - public RenderTargetIdentifier GetRenderTarget(); + public RenderTexture GetRenderTexture(); public void AddToCommandBuffer(CommandBuffer commandBuffer); public string GetName(); + public void OnInitialize(Camera mainCamera); + public void OnCameraChange(Camera mainCamera); + public bool IsInitialized(); + public byte[] GetBytes(bool jpg = false); } @@ -76,20 +105,41 @@ public static void SetupCameraWithPostShader( private Texture2D tex; private RenderTexture renderTexture; + public Material material; protected Shader shader; protected CommandBuffer cb; + protected bool cloudRendering; + + protected int? toDisplayId; + + private bool initialized; + + private TextureFormat readTextureFormat; + + + + // private Texture2D readTexture; + + public bool IsInitialized() { + return initialized; + } - public RenderToTexture(string name, Camera camera, int antiAliasLevel = 1, string shaderName = null) { + public RenderToTexture(CaptureConfig config, Camera camera) { this.camera = camera; - this.antiAliasLevel = antiAliasLevel; - this.shaderName = shaderName; - this.name = name; + this.antiAliasLevel = config.antiAliasLevel; + this.shaderName = config.shaderName; + this.name = config.name; + this.cloudRendering = config.cloudRendering; + this.toDisplayId = config.toDisplay; + + // TODO. if config.toDisplay is present then render to display buffer and copy to render texture + // for debugging purposes } - public virtual RenderTargetIdentifier GetRenderTarget() { - return new RenderTargetIdentifier(renderTexture); + public virtual RenderTexture GetRenderTexture() { + return renderTexture; } public virtual string GetName() { @@ -97,6 +147,7 @@ public virtual string GetName() { } public virtual void AddToCommandBuffer(CommandBuffer commandBuffer) { + Debug.Log("-------- AddToCommandBuffer " + name); // int screenCopyID = Shader.PropertyToID("_MainTex"); // cb.GetTemporaryRT(screenCopyID, -1, -1, 0, FilterMode.Bilinear); @@ -107,6 +158,14 @@ public virtual void AddToCommandBuffer(CommandBuffer commandBuffer) { // else { // cb.Blit(screenCopyID, BuiltinRenderTextureType.CameraTarget); // } + // cb.SetRenderTarget(RenderTargetIdentifier); + // cb.Blit(this.GetRenderTarget(), RenderTexture.active); + + Debug.Log("------- command buffer set for " + this.name); + // cb.SetRenderTarget(new RenderTargetIdentifier(null as Texture)); + //this.camera.targetTexture = null; + + // cb.Blit(this.GetRenderTarget(), BuiltinRenderTextureType.CurrentActive); // cb.ReleaseTemporaryRT(screenCopyID); @@ -127,6 +186,8 @@ public virtual void AddToCommandBuffer(CommandBuffer commandBuffer) { // this.camera.AddCommandBuffer(CameraEvent.BeforeImageEffects, cb); // this.camera.depthTextureMode = DepthTextureMode.Depth; + + // this.camera.AddCommandBuffer(CameraEvent.AfterImageEffects, cb); } @@ -142,7 +203,38 @@ private RenderTexture CreateRenderTexture(int width, int height) { // RenderTextureFormat.ARGB32 // ); - var rt = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); + // TODO: add rescaling here + if (this.renderTexture != null && this.renderTexture.IsCreated()) + { + this.renderTexture.Release(); + } + + + RenderTexture rt = null; + + + if (cloudRendering) { + // Why 0 for depth here ? + rt = new RenderTexture(Screen.width, Screen.height, 0, GraphicsFormat.R8G8B8A8_UNorm); + // TODO: if 0 then RGB24? if not RGB32? + readTextureFormat = TextureFormat.RGBA32; + + // RenderTexture rt = new RenderTexture( + // width: width, + // height: height, + // depth: 0, + // GraphicsFormat.R8G8B8A8_UNorm + // ); + } + else { + rt = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); + readTextureFormat = TextureFormat.RGBA32; + } + + if (this.tex == null) { + Debug.Log($"------------ texture format {Enum.GetName(typeof(TextureFormat), readTextureFormat)} val {readTextureFormat}"); + tex = new Texture2D(Screen.width, Screen.height, readTextureFormat, false); + } rt.antiAliasing = antiAliasLevel; if (rt.Create()) { Debug.Log("Created Render Texture with width= " + width + " height=" + height); @@ -155,38 +247,48 @@ private RenderTexture CreateRenderTexture(int width, int height) { } public virtual void OnInitialize(Camera mainCamera) { - // if (!shader && !string.IsNullOrEmpty(shaderName)) { - // shader = Shader.Find(shaderName); - // } - - // if (!material || material.shader != shader) { - // material = new Material(material); - // } - // OnInitialize(); - // OnCameraChange(camera); - } - public virtual void Initialize(Camera mainCamera) { + Debug.Log("++++ initialize called for " + this.name); if (!shader && !string.IsNullOrEmpty(shaderName)) { + + Debug.Log("---- loading shader " + shaderName); shader = Shader.Find(shaderName); } - if (!material || material.shader != shader) { - material = new Material(material); + if (shader && (!material || material.shader != shader)) { + material = new Material(shader); } - OnInitialize(mainCamera); - OnCameraChange(mainCamera); + //OnInitialize(mainCamera); + // OnCameraChange(mainCamera); + initialized = true; } + // public virtual void Initialize(Camera mainCamera) { + // if (!shader && !string.IsNullOrEmpty(shaderName)) { + // shader = Shader.Find(shaderName); + // } + + // if (!material || material.shader != shader) { + // material = new Material(material); + // } + + // OnInitialize(mainCamera); + // OnCameraChange(mainCamera); + // initialized = true; + // } + public virtual void OnCameraChange(Camera mainCamera) { + Debug.Log("-------OnCameraChange " + name); if (tex != null) { UnityEngine.Object.Destroy(tex); tex = null; } - + // if (cb != null) { + // this.camera.RemoveCommandBuffer(CameraEvent.AfterImageEffects, cb); + // } this.camera.RemoveAllCommandBuffers(); // copy all "main" camera parameters into capturing camera @@ -212,17 +314,21 @@ public virtual void OnCameraChange(Camera mainCamera) { // this.camera.depth = 0; // This ensures the new camera does not get rendered on screen - // TODO: add rescaling here - if (renderTexture != null && renderTexture.IsCreated()) - { - renderTexture.Release(); - } + // renderTexture = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); - renderTexture = CreateRenderTexture(Screen.width, Screen.height); + this.renderTexture = CreateRenderTexture(Screen.width, Screen.height); + // if (this.name != "_img") { + // this.camera.targetTexture = this.renderTexture; + // } + this.camera.targetDisplay = this.toDisplayId.GetValueOrDefault(); - this.camera.targetTexture = renderTexture; + // If set to render to display don't set render texture because display buffer is only written to if targetTexture is null + if (!this.toDisplayId.HasValue) { + this.camera.targetTexture = this.renderTexture; + } + if (cb != null) { cb.Clear(); @@ -230,8 +336,9 @@ public virtual void OnCameraChange(Camera mainCamera) { else { cb = new CommandBuffer(); } - + Debug.Log("------- Before command buffer " + name); this.AddToCommandBuffer(cb); + // TODO: debug code for editor @@ -278,7 +385,32 @@ public virtual void OnCameraChange(Camera mainCamera) { // // } - + public virtual byte[] GetBytes( + bool jpg = false + ) { + + var renderTexture = this.GetRenderTexture(); + var prevActiveRT = RenderTexture.active; + RenderTexture.active = renderTexture; + + // float startTime = Time.realtimeSinceStartup; + + tex.ReadPixels(new Rect(0, 0, tex.width, tex.height), 0, 0); + + + // startTime = Time.realtimeSinceStartup; + + // encode texture into PNG/JPG + byte[] bytes; + if (jpg) { + bytes = tex.EncodeToJPG(); + } else { + bytes = tex.GetRawTextureData(); + } + RenderTexture.active = prevActiveRT; + return bytes; + } + } @@ -287,13 +419,14 @@ public class ReplacementShaderCapture: RenderToTexture { private readonly Transform cameraParent; private ReplacelementMode mode; - public ReplacementShaderCapture(string name, Transform cameraParent, ReplacelementMode replacementMode, int antiAliasLevel = 1, string shaderName = null) : base(name, null, antiAliasLevel, shaderName) { + public ReplacementShaderCapture(CaptureConfig config, Transform cameraParent) : base(config, null) { this.cameraParent = cameraParent; - this.mode = replacementMode; + this.mode = config.replacementMode; } public override void OnInitialize(Camera mainCamera) { this.camera = CreateHiddenCamera(cameraParent, this.name); + base.OnInitialize(mainCamera); } @@ -326,9 +459,11 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) { this.camera.renderingPath = RenderingPath.Forward; this.camera.AddCommandBuffer(CameraEvent.BeforeForwardOpaque, cb); this.camera.AddCommandBuffer(CameraEvent.BeforeFinalPass, cb); + Debug.Log($"------ AddToCommandBuffer for {this.name}"); this.camera.SetReplacementShader(shader, ""); this.camera.backgroundColor = Color.blue; this.camera.clearFlags = CameraClearFlags.SolidColor; + this.camera.targetTexture = this.GetRenderTexture(); } @@ -338,8 +473,8 @@ public class MultiCapture : RenderToTexture { IEnumerable passes; Dictionary passDict; - public MultiCapture(string name, Camera camera, IEnumerable passes, int antiAliasLevel = 1, string shaderName = null) : base(name, camera, antiAliasLevel, shaderName) { - this.passDict = passes.ToDictionary(p => p.GetName()); + public MultiCapture(CaptureConfig config, Camera camera, IEnumerable passes) : base(config, camera) { + this.passDict = passes.ToDictionary(p => p.GetName(), p => p); } public override void AddToCommandBuffer(CommandBuffer commandBuffer) { @@ -357,25 +492,43 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) { // foreach (var pass in passes) { // commandBuffer.Blit(screenCopyID, pass.GetRenderTarget(), material); // } + // cb.SetRenderTarget(new RenderTargetIdentifier(null as Texture)); + //this.camera.targetTexture = null; + + // cb.Blit(this.GetRenderTarget(), BuiltinRenderTextureType.CurrentActive); + Debug.Log($"----------- Blit for multipass"); + + // If rendering to display + if (this.toDisplayId.HasValue) { + // if it's not cloudrendering camera.targetTexture is null which means it's rendering to the display buffer + // so then we need to copy the display buffer into render texture + + // for cloudrendering rbb is rendered directly into our render texture so no need to do this + cb.Blit(BuiltinRenderTextureType.CurrentActive, this.GetRenderTexture()); + } + + // cb.SetRenderTarget(this.GetRenderTarget()); // commandBuffer.ReleaseTemporaryRT(screenCopyID); foreach (var pass in this.passDict.Values) { - commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, pass.GetRenderTarget(), pass.material); + commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, pass.GetRenderTexture(), pass.material); } this.camera.AddCommandBuffer(CameraEvent.BeforeImageEffects, commandBuffer); this.camera.depthTextureMode = DepthTextureMode.Depth; + } - public override void Initialize(Camera mainCamera) { + public override void OnInitialize(Camera mainCamera) { foreach (var pair in this.passDict) { - pair.Value.Initialize(mainCamera); + pair.Value.OnInitialize(mainCamera); } - base.Initialize(mainCamera); + base.OnInitialize(mainCamera); } public override void OnCameraChange(Camera mainCamera) { + Debug.Log($"----------- Multipass OnCameraChange for {this.passDict.Values.Select(x => x.GetName())}"); foreach (var pair in this.passDict) { pair.Value.OnCameraChange(mainCamera); } @@ -388,7 +541,7 @@ public override void OnCameraChange(Camera mainCamera) { - public void AddCapturePass(RenderToTexture pass) { + public void AddUpdateCapturePass(RenderToTexture pass) { this.passDict[pass.GetName()] = pass; // if (!this.passDict.ContainsKey(pass.name)) // this.passDict.Add(pass) @@ -396,10 +549,20 @@ public void AddCapturePass(RenderToTexture pass) { } + + // public class DistortionCapture : RenderToTexture { + // public DistortionCapture(CaptureConfig config, Camera camera) : base(config, camera) { + // } + + + // } } + + + // public class ImageD : MonoBehaviour { // } \ No newline at end of file diff --git a/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs b/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs index 6152525b6f..ed5224d934 100644 --- a/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs +++ b/unity/Assets/Scripts/ImageSynthesis/ImageSynthesis.cs @@ -279,8 +279,26 @@ public void updateCameraStatuses(bool enabled) { private Dictionary availablePasses; + private ICapturePass mainPass; + private Dictionary activePasses; + private bool useImageSynthesis = true; + + public void EnablePasses(IEnumerable activePassesNames) { + if (activePassesNames != null) { + var newActive = activePassesNames.Select(name => { + ICapturePass capturePass; + var exists = availablePasses.TryGetValue(name, out capturePass); + return capturePass; + }); + if (newActive.Any(x => x == null)) { + throw new InvalidOperationException($"Invalid capture passes `{string.Join(", ", newActive.Where(x => x == null))}`"); + } + this.activePasses = newActive.ToDictionary(x => x.GetName(), x => x); + } + } + public void OnEnable() { @@ -304,24 +322,7 @@ public void OnEnable() { // new CapturePass() { name = "_distortion", noCamera=true } - // var camera = GetComponent(); - // var antiAliasLevel = 1; - // var mainPass = new MultiCapture("_img", camera, new List() {}); - // var dPass = new RenderToTexture( - // name: "_depth", camera: camera, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/DepthBW" - // ); - - // var distPass = new RenderToTexture( - // name: "_distortion", camera: camera, antiAliasLevel: antiAliasLevel, shaderName: "Custom/BarrelDistortion" - // ); - - // var idPass = new ReplacementShaderCapture( - // name: "_id", cameraParent: this.transform, replacementMode: ReplacelementMode.ObjectId, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/UberReplacement" - // ); - - // var classPass = new ReplacementShaderCapture( - // name: "_class", cameraParent: this.transform, replacementMode: ReplacelementMode.CatergoryId, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/UberReplacement" - // ); + // private static void SetupCameraWithReplacementShader( // Camera cam, @@ -346,6 +347,10 @@ public void OnEnable() { // programatically in other functions and need them to be initialized immediately. Debug.Log("OnEnable image synth"); if (!initialized) { + + + + // XXXXXXXXXXX************ // Remember, adding any new Shaders requires them to be included in Project Settings->Graphics->Always Included Shaders // otherwise the standlone will build without the shaders and you will be sad @@ -381,19 +386,22 @@ public void OnEnable() { opticalFlowSensitivity = 50.0f; // use real camera to capture final image + if (useImageSynthesis) { var mainCamera = GetComponent(); capturePasses[0].camera = GetComponent(); for (int q = 1; q < capturePasses.Length; q++) { - if (!capturePasses[q].noCamera) { - capturePasses[q].camera = CreateHiddenCamera(capturePasses[q].name); - } - else { - capturePasses[q].camera = mainCamera; + // if (!capturePasses[q].noCamera) { + // capturePasses[q].camera = CreateHiddenCamera(capturePasses[q].name); + // } + // else { + //capturePasses[q].camera = mainCamera; - } + // } } md5 = System.Security.Cryptography.MD5.Create(); + } + OnCameraChange(); Debug.Log("OnEnable image synth scenechange"); OnSceneChange(); @@ -561,6 +569,7 @@ enum ReplacelementModes { // Call this if the settings on the main camera ever change? But the main camera now uses slightly different layer masks and deffered/forward render settings than these image synth cameras // do, so maybe it's fine for now I dunno public void OnCameraChange() { + if (useImageSynthesis) { if (tex != null) { Destroy(tex); tex = null; @@ -570,101 +579,103 @@ public void OnCameraChange() { // TODO: add tests, not needed when target display is different // mainCamera.depth = 9999; // This ensures the main camera is rendered on screen - foreach (var pass in capturePasses) { - pass.camera.RemoveAllCommandBuffers(); - if (pass.camera == mainCamera) { - continue; - } + // foreach (var pass in capturePasses) { + // pass.camera.RemoveAllCommandBuffers(); + // if (pass.camera == mainCamera) { + // continue; + // } - // cleanup capturing camera - // pass.camera.RemoveAllCommandBuffers(); + // // cleanup capturing camera + // // pass.camera.RemoveAllCommandBuffers(); - // copy all "main" camera parameters into capturing camera - pass.camera.CopyFrom(mainCamera); + // // copy all "main" camera parameters into capturing camera + // pass.camera.CopyFrom(mainCamera); - // make sure the capturing camera is set to Forward rendering (main camera uses Deffered now) - pass.camera.renderingPath = RenderingPath.Forward; - // make sure capturing camera renders all layers (value copied from Main camera excludes PlaceableSurfaces layer, which needs to be rendered on this camera) - pass.camera.cullingMask = -1; + // // make sure the capturing camera is set to Forward rendering (main camera uses Deffered now) + // pass.camera.renderingPath = RenderingPath.Forward; + // // make sure capturing camera renders all layers (value copied from Main camera excludes PlaceableSurfaces layer, which needs to be rendered on this camera) + // pass.camera.cullingMask = -1; - pass.camera.depth = 0; // This ensures the new camera does not get rendered on screen - } + // pass.camera.depth = 0; // This ensures the new camera does not get rendered on screen + // } // cache materials and setup material properties - if (!opticalFlowMaterial || opticalFlowMaterial.shader != opticalFlowShader) { - opticalFlowMaterial = new Material(opticalFlowShader); - } - opticalFlowMaterial.SetFloat("_Sensitivity", opticalFlowSensitivity); + // if (!opticalFlowMaterial || opticalFlowMaterial.shader != opticalFlowShader) { + // opticalFlowMaterial = new Material(opticalFlowShader); + // } + // opticalFlowMaterial.SetFloat("_Sensitivity", opticalFlowSensitivity); - if (!depthMaterial || depthMaterial.shader != depthShader) { - depthMaterial = new Material(depthShader); - } + // if (!depthMaterial || depthMaterial.shader != depthShader) { + // depthMaterial = new Material(depthShader); + // } - // screenCopyMaterial = new Material(screenCopyShader); + // // screenCopyMaterial = new Material(screenCopyShader); - if (!distortionMaterial || distortionMaterial.shader != distortionShader) { - distortionMaterial = new Material(distortionShader); - } - var distortionPass = capturePasses.First(x => x.name == "_distortion"); + // if (!distortionMaterial || distortionMaterial.shader != distortionShader) { + // distortionMaterial = new Material(distortionShader); + // } + // var distortionPass = capturePasses.First(x => x.name == "_distortion"); - distortionMaterial.SetFloat("_fov_y", distortionPass.camera.fieldOfView); + // distortionMaterial.SetFloat("_fov_y", distortionPass.camera.fieldOfView); - Texture2D realTex = null; - byte[] fileData; - var filePath = Application.dataPath + "/real_camera/" + "frame_1.png"; + // Texture2D realTex = null; + // byte[] fileData; + // var filePath = Application.dataPath + "/real_camera/" + "frame_1.png"; - if (File.Exists(filePath)) { - fileData = File.ReadAllBytes(filePath); - realTex = new Texture2D(2, 2); - realTex.LoadImage(fileData); //..this will auto-resize the texture dimensions. + // Move to a shader Uniform Set Callback + // if (File.Exists(filePath)) { + // fileData = File.ReadAllBytes(filePath); + // realTex = new Texture2D(2, 2); + // realTex.LoadImage(fileData); //..this will auto-resize the texture dimensions. - distortionMaterial.SetTexture("_RealImage", realTex); - } + // distortionMaterial.SetTexture("_RealImage", realTex); + // } // capturePasses [1].camera.farClipPlane = 100; // SetupCameraWithReplacementShader(capturePasses[1].camera, uberReplacementShader, ReplacelementModes.DepthMultichannel); - if (renderTexture != null && renderTexture.IsCreated()) - { - renderTexture.Release(); - } + // if (renderTexture != null && renderTexture.IsCreated()) + // { + // renderTexture.Release(); + // } - for (int i = 0; i < capturePasses.Length; i++) { - if (capturePasses[i].noCamera) { - if (capturePasses[i].renderTexture != null && capturePasses[i].renderTexture.IsCreated()) { - capturePasses[i].renderTexture.Release(); + // for (int i = 0; i < capturePasses.Length; i++) { + // if (capturePasses[i].noCamera) { + // if (capturePasses[i].renderTexture != null && capturePasses[i].renderTexture.IsCreated()) { + // capturePasses[i].renderTexture.Release(); - } - var format = RenderTextureFormat.ARGB32; - // if (capturePasses[i].name == "_depth") { - // format = RenderTextureFormat.Depth; - // } - capturePasses[i].renderTexture = new RenderTexture(Screen.width, Screen.height, 24, format, RenderTextureReadWrite.Default); - } - } + // } + // var format = RenderTextureFormat.ARGB32; + // // if (capturePasses[i].name == "_depth") { + // // format = RenderTextureFormat.Depth; + // // } + // capturePasses[i].renderTexture = new RenderTexture(Screen.width, Screen.height, 24, format, RenderTextureReadWrite.Default); + // } + // } - renderTexture = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); + // renderTexture = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); // SetupCameraWithPostShader2(renderTexture, capturePasses[1].camera, depthMaterial, // // screenCopyMaterial: screenCopyMaterial, // DepthTextureMode.Depth); - SetupCameraWithReplacementShader( - capturePasses[2].camera, - uberReplacementShader, - ReplacelementModes.ObjectId - ); - SetupCameraWithReplacementShader( - capturePasses[3].camera, - uberReplacementShader, - ReplacelementModes.CatergoryId - ); - SetupCameraWithReplacementShader( - capturePasses[4].camera, - uberReplacementShader, - ReplacelementModes.Normals - ); + // SetupCameraWithReplacementShader( + // capturePasses[2].camera, + // uberReplacementShader, + // ReplacelementModes.ObjectId + // ); + // SetupCameraWithReplacementShader( + // capturePasses[3].camera, + // uberReplacementShader, + // ReplacelementModes.CatergoryId + // ); + // SetupCameraWithReplacementShader( + // capturePasses[4].camera, + // uberReplacementShader, + // ReplacelementModes.Normals + // ); + // SetupCameraWithPostShader2( // renderTexture, // capturePasses[5].camera, @@ -677,25 +688,29 @@ public void OnCameraChange() { // // screenCopyMaterial: screenCopyMaterial, // DepthTextureMode.Depth); - SetupCameraWithPostShaders( - renderTexture, - capturePasses[0].camera, // main camera - new List<(Material, CapturePass)>() { - (depthMaterial, capturePasses[1]), - // opticalFlowMaterial // unused so disabling - (distortionMaterial, capturePasses[6]) - }, - DepthTextureMode.Depth | DepthTextureMode.MotionVectors - ); + // SetupCameraWithPostShaders( + // renderTexture, + // capturePasses[0].camera, // main camera + // new List<(Material, CapturePass)>() { + // (depthMaterial, capturePasses[1]), + // // opticalFlowMaterial // unused so disabling + // (distortionMaterial, capturePasses[6]) + // }, + // DepthTextureMode.Depth | DepthTextureMode.MotionVectors + // ); + #if UNITY_EDITOR - for (int i = 0; i < capturePasses.Length; i++) { - // Debug.Log("Setting camera " + capturePasses[i].camera.gameObject.name + " to display " + i); - capturePasses[i].camera.targetDisplay = i; - } + // for (int i = 0; i < capturePasses.Length; i++) { + // // Debug.Log("Setting camera " + capturePasses[i].camera.gameObject.name + " to display " + i); + // if (capturePasses[i].camera != mainCamera) { + // capturePasses[i].camera.targetDisplay = i; + // } + // } #endif + } /* SetupCameraWithReplacementShader(capturePasses[6].camera, positionShader); @@ -764,7 +779,7 @@ public void OnSceneChange() { Color classColor = ColorEncoding.EncodeTagAsColor(classTag); Color objColor = ColorEncoding.EncodeTagAsColor(objTag); - capturePasses[0].camera.WorldToScreenPoint(r.bounds.center); + // capturePasses[0].camera.WorldToScreenPoint(r.bounds.center); if (so != null || sop != null) { colorIds[objColor] = objTag; diff --git a/unity/Assets/Scripts/RenderingManager.cs b/unity/Assets/Scripts/RenderingManager.cs new file mode 100644 index 0000000000..78c83b8c2e --- /dev/null +++ b/unity/Assets/Scripts/RenderingManager.cs @@ -0,0 +1,268 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using Thor.Rendering; +using System.Linq; +using System; +using Unity.Rendering; +using MessagePack.Resolvers; +using UnityEngine.Rendering; +using System.Runtime.Remoting.Messaging; + +public class RenderingManager : MonoBehaviour { + + + private Dictionary availablePasses; + + private MultiCapture mainPass; + + private Dictionary activePasses; + // Start is called before the first frame update + + // Captures that can go with the main render _img pass, optimization + private HashSet WithMainMultiPass = new HashSet { + "_depth", + "_distortion" + }; + + public Material distortionMat; + + private Texture2D readTex; + + private static bool isMainCameraPassCreated = false; + + void Initialize(Camera camera) { + + // var camera = GetComponent(); + // var antiAliasLevel = 1; + + + // var depthPass = new RenderToTexture( + // name: "_depth", camera: camera, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/DepthBW" + // ); + + // var distPass = new RenderToTexture( + // name: "_distortion", camera: camera, antiAliasLevel: antiAliasLevel, shaderName: "Custom/BarrelDistortion" + // ); + + // var idPass = new ReplacementShaderCapture( + // name: "_id", cameraParent: this.transform, replacementMode: ReplacelementMode.ObjectId, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/UberReplacement" + // ); + + // var classPass = new ReplacementShaderCapture( + // name: "_class", cameraParent: this.transform, replacementMode: ReplacelementMode.CatergoryId, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/UberReplacement" + // ); + + // var normalsPass = new ReplacementShaderCapture( + // name: "_normals", cameraParent: this.transform, replacementMode: ReplacelementMode.Normals, antiAliasLevel: antiAliasLevel, shaderName: "Hidden/UberReplacement" + // ); + + // this.mainPass = new MultiCapture("_img", camera, new List() { + + // }); + + // availablePasses = new List() { + // this.mainPass, + // depthPass, + // distPass, + // idPass, + // classPass + // }.ToDictionary(x => x.GetName(), x => x); + + // this.activePasses = new List() { + // this.mainPass + // }.ToDictionary(x => x.GetName(), x => x); + // mainPass.OnInitialize(camera); + + } + + public void EnablePasses(IEnumerable activePassesNames, bool cameraChange) { + var mainCamera = GetComponent(); + + if (activePassesNames != null) { + Debug.Log($"--------- Enabling passes 0 {string.Join(", ", activePassesNames)}"); + var newActive = activePassesNames.Select(name => { + ICapturePass capturePass; + var exists = availablePasses.TryGetValue(name, out capturePass); + return capturePass; + }); + // var newActive = + if (newActive.Any(x => x == null)) { + throw new InvalidOperationException($"Invalid capture passes `{string.Join(", ", newActive.Where(x => x == null))}`"); + } + Debug.Log($"--------- Enabling passes 2 {string.Join(", ", newActive.Select(x => x.GetName()))}"); + var toInitialize = newActive.Where( x => ! x.IsInitialized()); + // if this is one of the passes that is part of a MultiPass + var mainMultiPassUpdate = toInitialize.Where(x => this.WithMainMultiPass.Contains(x.GetName())); + foreach (var pass in mainMultiPassUpdate) { + // TODO bad typecast, rework types + mainPass.AddUpdateCapturePass(pass as RenderToTexture); + } + + Debug.Log($"--------- Enabling passes 3 toinitialize {string.Join(", ", toInitialize.Select(x => x.GetName()))}"); + //Sort by + // Weird that multiPasCapture does not get Initialized? or already was + + // Don't initialize or onCamerachange passes that belong to other passes + // toInitialize = toInitialize.Where(x => !this.WithMainMultiPass.Contains(x.GetName())); + var initialized = new HashSet(); + foreach (var newPass in toInitialize) { + newPass.OnInitialize(mainCamera); + initialized.Add(newPass.GetName()); + } + // this.activePasses = newActive.Where(x => !this.WithMainMultiPass.Contains(x.GetName())).ToDictionary(x => x.GetName(), x => x); + this.activePasses = newActive.ToDictionary(x => x.GetName(), x => x); + if (cameraChange) { + + // TODO order important? + // Initialize calls OnCameraChange + var onCameraChange = activePasses.Values.Where(x => !this.WithMainMultiPass.Contains(x.GetName())); + + Debug.Log($"--------- OnCameraChange passes 3 {string.Join(", ", onCameraChange)}"); + foreach (var pass in onCameraChange) { // && !initialized.Contains(x.GetName()))) { + pass.OnCameraChange(mainCamera); + } + } + + Debug.Log($"--------- Enabling passes 4 activePasses {string.Join(", ", this.activePasses)}"); + } + } + + + public void OnCameraChange() { + Debug.Log($"===== OnCameraChange multipass for {string.Join(", ", this.activePasses.Values.Select(x => x.GetName()))}"); + var mainCamera = GetComponent(); + foreach (var pass in this.activePasses.Values) { + pass.OnCameraChange(mainCamera); + } + } + + + void Awake() { + + Debug.Log("=-------- Rendering Manager Awake"); + var camera = GetComponent(); + bool supportsAntialiasing = false; + var antiAliasLevel = supportsAntialiasing ? Mathf.Max(1, QualitySettings.antiAliasing) : 1; + + var cloudRenderingCapture = false; + #if PLATFORM_CLOUD_RENDERING + cloudRenderingCapture = true; + #endif + + + var depthPass = new RenderToTexture( + new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/DepthBW" }, + camera: camera + ); + + var distPass = new RenderToTexture( + new CaptureConfig() { name = "_distortion", antiAliasLevel = antiAliasLevel, shaderName = "Custom/BarrelDistortion" }, + camera: camera + ); + + var idPass = new ReplacementShaderCapture( + new CaptureConfig() { name = "_id", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.ObjectId }, + cameraParent: camera.transform + ); + + var classPass = new ReplacementShaderCapture( + new CaptureConfig() { name = "_class", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.CatergoryId }, + cameraParent: camera.transform + ); + + var normalsPass = new ReplacementShaderCapture( + new CaptureConfig() { name = "_normals", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.Normals }, + cameraParent: camera.transform + ); + + // make first _img capture created render to Display + int? toDisplay = null; + this.mainPass = new MultiCapture( + config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, cloudRendering = cloudRenderingCapture, toDisplay = isMainCameraPassCreated ? toDisplay : 0}, + camera: camera, + passes: new List() { + } + ); + + availablePasses = new List() { + this.mainPass, + depthPass, + distPass, + idPass, + classPass + }.ToDictionary(x => x.GetName(), x => x); + + this.activePasses = new List() { + this.mainPass + }.ToDictionary(x => x.GetName(), x => x); + mainPass.OnInitialize(camera); + // this.enabled = true; + } + + public RenderToTexture GetCapturePass(string passName) { + ICapturePass pass; + if (!this.activePasses.TryGetValue(passName, out pass)) { + Debug.LogError($"No active pass at GetPassRenderTexture {passName}"); + return null; + } + return pass as RenderToTexture; + } + + public RenderTexture GetPassRenderTexture(string passName) { + ICapturePass pass; + if (!this.activePasses.TryGetValue(passName, out pass)) { + throw new InvalidOperationException($"No active pass at GetPassRenderTexture {passName}"); + } + + + return pass.GetRenderTexture(); + } + + + public byte[] GetCaptureBytes(string passName, bool jpeg = false) { + ICapturePass pass; + if (!this.activePasses.TryGetValue(passName, out pass)) { + Debug.LogError($"No active pass at GetPassRenderTexture {passName}"); + return (new byte[0]); + } + else { + Debug.Log($"--- call GetBytes on pass {passName}"); + var bytes = pass.GetBytes(jpeg); + Debug.Log($"-------- bytes size {bytes.Length}"); + return bytes; + } + } + + + + public void GetCaptureAsync( + string passName, + List> payload, + string key + ) { + ICapturePass pass; + if (!this.activePasses.TryGetValue(passName, out pass)) { + Debug.LogError($"No active pass at GetPassRenderTexture {passName}"); + } + RenderTexture tt = pass.GetRenderTexture(); + var prevActiveTex = RenderTexture.active; + RenderTexture.active = tt; + // camera.Render(); + AsyncGPUReadback.Request( + tt, + 0, + (request) => { + if (!request.hasError) { + var data = request.GetData().ToArray(); + payload.Add(new KeyValuePair(key, data)); + } else { + Debug.Log("Request error: " + request.hasError); + } + } + ); + } + + // Update is called once per frame + void Update() { } +} diff --git a/unity/Assets/Scripts/StretchAgentController.cs b/unity/Assets/Scripts/StretchAgentController.cs index 22da7e5c7e..a69c58a221 100644 --- a/unity/Assets/Scripts/StretchAgentController.cs +++ b/unity/Assets/Scripts/StretchAgentController.cs @@ -62,7 +62,7 @@ AgentManager agentManager ) : base(baseAgentComponent, agentManager) { } - public override void updateImageSynthesis(bool status) { + public override void updateImageSynthesis(bool status, IEnumerable activePassList = null) { base.updateImageSynthesis(status); // updateImageSynthesis is run in BaseFPSController's Initialize method after the diff --git a/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab b/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab index d91b00c49f..83f0d93dd0 100644 --- a/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab +++ b/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab @@ -258,6 +258,7 @@ GameObject: - component: {fileID: 114176532703538896} - component: {fileID: 992261166} - component: {fileID: 2007864990} + - component: {fileID: 8452980845336696311} m_Layer: 10 m_Name: FirstPersonCharacter m_TagString: MainCamera @@ -366,6 +367,8 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: opticalFlowSensitivity: 0 + depthMaterial: {fileID: 0} + distortionMaterial: {fileID: 0} sentColorCorrespondence: 0 tex: {fileID: 0} --- !u!114 &114176532703538896 @@ -375,7 +378,7 @@ MonoBehaviour: m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 100002} - m_Enabled: 0 + m_Enabled: 1 m_EditorHideFlags: 0 m_Script: {fileID: 11500000, guid: 76b5ec6153a1d55438228df10fe66844, type: 3} m_Name: @@ -462,6 +465,18 @@ MonoBehaviour: m_BeforeTransparentBundles: [] m_BeforeStackBundles: [] m_AfterStackBundles: [] +--- !u!114 &8452980845336696311 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 100002} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 68871f0bc5fb742138947ad4c5e73f7d, type: 3} + m_Name: + m_EditorClassIdentifier: --- !u!1 &1340499340313870 GameObject: m_ObjectHideFlags: 0 diff --git a/unity/ProjectSettings/ProjectSettings.asset b/unity/ProjectSettings/ProjectSettings.asset index 29c345dbd9..6a9ed65b4e 100644 --- a/unity/ProjectSettings/ProjectSettings.asset +++ b/unity/ProjectSettings/ProjectSettings.asset @@ -644,7 +644,7 @@ PlayerSettings: webGLThreadsSupport: 0 webGLDecompressionFallback: 0 scriptingDefineSymbols: - 1: UNITY_POST_PROCESSING_STACK_V2 + 1: 4: CROSS_PLATFORM_INPUT;MOBILE_INPUT;UNITY_POST_PROCESSING_STACK_V2 7: CROSS_PLATFORM_INPUT;MOBILE_INPUT;UNITY_POST_PROCESSING_STACK_V2 13: UNITY_POST_PROCESSING_STACK_V2