diff --git a/Assets/SEE/Tools/Videochat.meta b/Assets/SEE/Tools/Livekit.meta similarity index 100% rename from Assets/SEE/Tools/Videochat.meta rename to Assets/SEE/Tools/Livekit.meta diff --git a/Assets/SEE/Tools/Livekit/LivekitVideo.cs b/Assets/SEE/Tools/Livekit/LivekitVideo.cs new file mode 100644 index 0000000000..e75bb77358 --- /dev/null +++ b/Assets/SEE/Tools/Livekit/LivekitVideo.cs @@ -0,0 +1,91 @@ +using UnityEngine; +using Unity.Netcode; +using SEE.Controls; +using SEE.Utils; + +namespace SEE.Tools.Livekit +{ + /// + /// Handles the positioning of the Livekit video stream relative to the player's head. + /// + public class LivekitVideo : NetworkBehaviour + { + /// + /// The Transform representing the player's head. + /// + private Transform playerHead; + + /// + /// The bone path leading to the player's head. Used to position the video. + /// + private const string FaceCamOrientationBone = "CC_Base_BoneRoot/CC_Base_Hip/CC_Base_Waist/CC_Base_Spine01/CC_Base_Spine02/CC_Base_NeckTwist01/CC_Base_NeckTwist02/CC_Base_Head"; + + /// + /// Determines whether the video is in front of the face or above the head. + /// True if positioned in front of the face. + /// + private bool faceCamOnFront = true; + + /// + /// Offset for positioning the video in front of the player's face. + /// + private readonly Vector3 offsetInFrontOfFace = new Vector3(0, 0.065f, 0.15f); + + /// + /// Offset for positioning the video above the player's head. + /// + private readonly Vector3 offsetAboveHead = new Vector3(0, 0.35f, 0); + + /// + /// Initializes the player head reference and names the object according to the owner ID. + /// Logs an error and disables the component if the player head cannot be found. + /// + private void Start() + { + gameObject.name = "LivekitVideo_" + OwnerClientId; + // Localizes the player's head bone for the positioning of the video. + playerHead = transform.parent.Find(FaceCamOrientationBone); + + if (playerHead == null) + { + Debug.LogError($"Player head not found for client ID {OwnerClientId}. Disabling LivekitVideo component."); + enabled = false; + } + } + + /// + /// Updates the position of the video every frame. Toggles the video position based on input. + /// + private void Update() + { + UpdatePosition(); + + if (SEEInput.ToggleFaceCamPosition()) + { + faceCamOnFront = !faceCamOnFront; + } + } + + /// + /// Updates the position and rotation of the video based on the player's head. + /// + private void UpdatePosition() + { + if (faceCamOnFront) + { + // Position the video in front of the player's face. + transform.SetPositionAndRotation(playerHead.TransformPoint(offsetInFrontOfFace), playerHead.rotation); + } + else + { + // Position the video above the player's head. + transform.position = playerHead.TransformPoint(offsetAboveHead); + // If this object is not owned by the local client, make it face the main camera. + if (!IsOwner && MainCamera.Camera != null) + { + transform.LookAt(MainCamera.Camera.transform); + } + } + } + } +} diff --git a/Assets/SEE/Tools/Videochat/LivekitVideo.cs.meta b/Assets/SEE/Tools/Livekit/LivekitVideo.cs.meta similarity index 100% rename from Assets/SEE/Tools/Videochat/LivekitVideo.cs.meta rename to Assets/SEE/Tools/Livekit/LivekitVideo.cs.meta diff --git a/Assets/SEE/Tools/Livekit/LivekitVideoManager.cs b/Assets/SEE/Tools/Livekit/LivekitVideoManager.cs new file mode 100644 index 0000000000..7c83784d29 --- /dev/null +++ b/Assets/SEE/Tools/Livekit/LivekitVideoManager.cs @@ -0,0 +1,474 @@ +// Code inspired by https://github.com/livekit-examples/unity-example/blob/main/LivekitUnitySampleApp/Assets/LivekitSamples.cs +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using LiveKit; +using LiveKit.Proto; +using RoomOptions = LiveKit.RoomOptions; +using UnityEngine.UI; +using TMPro; +using Unity.Netcode; +using SEE.Controls; + +namespace SEE.Tools.Livekit +{ + /// + /// Manages LiveKit video streams and plays them via the LivekitVideo object in the player objects. + /// Handles publishing/unpublishing local video, subscribing/unsubscribing to remote video, + /// and switching between available camera devices. + /// + public class LivekitVideoManager : NetworkBehaviour + { + /// + /// The URL of the LiveKit server to connect to. + /// + public string LivekitUrl = "ws://localhost:7880"; + + /// + /// The URL used to fetch the access token required for authentication. + /// + public string TokenUrl = "http://localhost:3000"; + + /// + /// The room name to join in LiveKit. + /// + public string RoomName = "development"; + + /// + /// The LiveKit room object that manages connection and tracks. + /// + private Room room = null; + + /// + /// The local video track being published to the LiveKit server. + /// + private LocalVideoTrack publishedTrack = null; + + /// + /// The WebCamTexture used to capture the video stream from the selected camera. + /// + private WebCamTexture webCamTexture = null; + + /// + /// The index of the currently selected camera. + /// + private int currentCameraIndex = 0; + + /// + /// An array containing the available webcam devices. + /// + private WebCamDevice[] devices; + + /// + /// The dropdown UI component used to select between different available cameras. + /// + public TMP_Dropdown CameraDropdown; + + /// + /// A dictionary that maps participant identities to the GameObjects that represent their video streams. + /// + private Dictionary videoObjects = new(); + + /// + /// A list of video sources created from the local webcam that are currently being published to the room. + /// + private List rtcVideoSources = new(); + + /// + /// A list of video streams from remote participants in the LiveKit room. + /// + private List videoStreams = new(); + + /// + /// Initializes the video manager by obtaining a token and setting up the camera dropdown. + /// + private void Start() + { + SetupCameraDropdown(); + StartCoroutine(GetToken()); + } + + /// + /// Cleans up resources when the object is destroyed, including stopping the webcam + /// and disconnecting from the LiveKit room. + /// + private void OnDestroy() + { + if (webCamTexture != null) + { + webCamTexture.Stop(); + } + room.Disconnect(); + CleanUp(); + room = null; + } + + /// + /// Toggles video publishing on and off. + /// It can be toggled with . + /// + private void Update() + { + if (SEEInput.ToggleFaceCam()) + { + if (publishedTrack == null) + { + StartCoroutine(PublishVideo()); + } + else + { + StartCoroutine(UnpublishVideo()); + } + } + } + + /// + /// Pauses or resumes the webcam video when the application is paused or resumed. + /// + /// Whether the application is paused. + private void OnApplicationPause(bool pause) + { + if (webCamTexture != null) + { + if (pause) + { + webCamTexture.Pause(); + } + else + { + webCamTexture.Play(); + } + } + } + + #region Camera Methods + /// + /// Initializes the camera dropdown menu with the list of available camera devices. + /// This method populates the dropdown with the names of all connected camera devices + /// and sets up a listener to handle camera selection changes. + /// + private void SetupCameraDropdown() + { + // Retrieve the list of available camera devices. + devices = WebCamTexture.devices; + + if (devices.Length > 0) + { + List cameraNames = new(); + + // Iterate through each device and add its name to the list. + foreach (WebCamDevice device in devices) + { + cameraNames.Add(string.IsNullOrEmpty(device.name) ? "Unnamed Camera" : device.name); + } + + CameraDropdown.ClearOptions(); + CameraDropdown.AddOptions(cameraNames); + CameraDropdown.onValueChanged.AddListener(OpenSelectedCamera); + + // Open the first camera by default. + OpenSelectedCamera(0); + } + else + { + Debug.LogError("[Livekit] No camera devices available"); + } + } + + /// + /// Opens and starts the selected camera device based on the provided index. + /// This method stops any currently active camera, initializes a new WebCamTexture with the selected + /// camera device, and starts capturing video from it. It also republishes the video stream. + /// + /// The index of the selected camera device in the dropdown list. + private void OpenSelectedCamera(int index) + { + if (index >= 0 && index < devices.Length) + { + webCamTexture?.Stop(); + + string selectedDeviceName = devices[index].name; + + // Initialize a new WebCamTexture with the selected camera device. + webCamTexture = new WebCamTexture(selectedDeviceName); + webCamTexture.Play(); + + StartCoroutine(RepublishVideo()); + Debug.Log($"[Livekit] Switched to camera: {selectedDeviceName}"); + } + } + #endregion + + #region Connection Methods + /// + /// Fetches an authentication token from the token server and connects to the LiveKit room. + /// Makes a GET request to the token server with the room name and participant name. + /// The name of the participant is the local client ID. + /// + /// Coroutine to handle the asynchronous token request process. + private IEnumerator GetToken() + { + // Send a GET request to the token server to retrieve the token for this client. + using (UnityEngine.Networking.UnityWebRequest www = UnityEngine.Networking.UnityWebRequest.Get( + $"{TokenUrl}/getToken?roomName={RoomName}&participantName={NetworkManager.Singleton.LocalClientId.ToString()}")) + { + // Wait for the request to complete. + yield return www.SendWebRequest(); + + // Check if the request was successful. + if (www.result == UnityEngine.Networking.UnityWebRequest.Result.Success) + { + // Token received, proceed to join the room using the received token. + StartCoroutine(JoinRoom(www.downloadHandler.text)); + } + } + } + + /// + /// Connects to the LiveKit room using the previously fetched token. + /// Initializes the room, subscribes to events, and connects with provided room options. + /// + /// The authentication token received from the token server. + /// Coroutine that handles the connection to the room. + private IEnumerator JoinRoom(string token) + { + // Initialize a new room instance. + room = new(); + + // Subscribe to events related to track management. + room.TrackSubscribed += TrackSubscribed; + room.TrackUnsubscribed += UnTrackSubscribed; + + RoomOptions options = new(); + + // Attempt to connect to the room using the LiveKit server URL and the provided token. + ConnectInstruction connect = room.Connect(LivekitUrl, token, options); + yield return connect; + + // Check if the connection was successful. + if (!connect.IsError) + { + Debug.Log("[Livekit] Connected to " + room.Name); + } + } + #endregion + + #region Publish Methods + /// + /// Publishes the local video track to the LiveKit room. + /// Creates a video track from the webcamtexture and publishes it to the room. + /// Updates the mesh object for the local client with the video. + /// The Mesh object is provided by LivekitVideo.prefab, + /// which is instantiated as an immediate child of the Player object. + /// + /// Coroutine to handle the asynchronous publishing process. + private IEnumerator PublishVideo() + { + // Check if the room is initialized. + if (room != null) + { + // Create a video source from the current webcam texture. + TextureVideoSource source = new TextureVideoSource(webCamTexture); + + // Create a local video track with the video source. + LocalVideoTrack track = LocalVideoTrack.CreateVideoTrack("my-video-track", source, room); + + // Define options for publishing the video track. + TrackPublishOptions options = new TrackPublishOptions + { + VideoCodec = VideoCodec.H264, // Codec to be used for video. + VideoEncoding = new VideoEncoding + { + MaxBitrate = 512000, // Maximum bitrate in bits per second. + // Higher values improve the quality, but require more bandwidth. + MaxFramerate = 30 // Maximum frames per second. + }, + Simulcast = true, // Enable simulcast for better scalability. + // Allows participants different quality levels, but increases the server load. + Source = TrackSource.SourceCamera // Specify the source as the camera. + }; + + // Publish the video track to the room. + PublishTrackInstruction publish = room.LocalParticipant.PublishTrack(track, options); + yield return publish; + + // Check if the publishing was successful. + if (!publish.IsError) + { + Debug.Log("[Livekit] Video track published!"); + publishedTrack = track; + + // Find and update the mesh object for the local client with the video. + string localClientId = NetworkManager.Singleton.LocalClientId.ToString(); + GameObject meshObject = GameObject.Find("LivekitVideo_" + localClientId); + + if (meshObject != null) + { + MeshRenderer renderer = meshObject.GetComponent(); + if (renderer != null) + { + // Enable the renderer and set the texture. + renderer.material.mainTexture = webCamTexture; + renderer.enabled = true; + } + } + + // Store the mesh object in the dictionary. + videoObjects[localClientId] = meshObject; + } + + // Start capturing and updating the video source. + source.Start(); + StartCoroutine(source.Update()); + rtcVideoSources.Add(source); + } + } + + /// + /// Unpublishes the local video track from the LiveKit room. + /// Stops the video, disables the mesh renderer, and removes the video track. + /// + /// Coroutine to handle the asynchronous unpublishing process. + private IEnumerator UnpublishVideo() + { + // Check if the room is initialized. + if (room != null) + { + // Unpublish the video track from the room. + UnpublishTrackInstruction unpublish = room.LocalParticipant.UnpublishTrack(publishedTrack, true); + yield return unpublish; + + // Check if the unpublishing was successful. + if (!unpublish.IsError) + { + Debug.Log("[Livekit] Video track unpublished."); + publishedTrack = null; + + // Find and update the mesh object for the local client. + string localClientId = NetworkManager.Singleton.LocalClientId.ToString(); + if (videoObjects.TryGetValue(localClientId, out GameObject meshObject) && meshObject != null) + { + MeshRenderer renderer = meshObject.GetComponent(); + if (renderer != null) + { + // Disable the renderer and clear the texture. + renderer.enabled = false; + renderer.material.mainTexture = null; + } + + // Remove the mesh object from the dictionary. + videoObjects.Remove(localClientId); + } + } + } + } + + /// + /// Republishes the video track, typically used after switching the camera. + /// + /// Coroutine that handles the republishing of the video. + private IEnumerator RepublishVideo() + { + if (publishedTrack != null) + { + yield return StartCoroutine(UnpublishVideo()); + StartCoroutine(PublishVideo()); + } + } + #endregion + + #region Track Methods + /// + /// Callback method that is invoked when a remote track is subscribed. + /// Handles the display of the remote video stream on a mesh object. + /// The Mesh object is provided by LivekitVideo.prefab, + /// which is instantiated as an immediate child of the Player object. + /// + /// The remote track being subscribed to. + /// The publication details of the track. + /// The remote participant owning the track. + private void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant) + { + if (track is RemoteVideoTrack videoTrack) + { + Debug.Log("[Livekit] TrackSubscribed for " + participant.Identity); + + // Find the LivekitVideo object to display the video stream. + GameObject meshObject = GameObject.Find("LivekitVideo_" + participant.Identity); + if (meshObject != null) + { + MeshRenderer renderer = meshObject.GetComponent(); + + // Create a new VideoStream instance for the subscribed track. + VideoStream stream = new VideoStream(videoTrack); + stream.TextureReceived += texture => + { + if (renderer != null) + { + // Enable the renderer and set the texture. + renderer.material.mainTexture = texture; + renderer.enabled = true; + } + }; + + videoObjects[participant.Identity] = meshObject; // Add the VideoStream to the list of video streams. + stream.Start(); // Start the video stream. + StartCoroutine(stream.Update()); // Continuously update the video stream. + videoStreams.Add(stream); // Add the stream to the list of active video streams. + } + } + } + + /// + /// Callback method that is invoked when a remote track is unsubscribed. + /// Cleans up the mesh object associated with the remote video stream. + /// + /// The remote track being unsubscribed. + /// The publication details of the track. + /// The remote participant owning the track. + private void UnTrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant) + { + if (track is RemoteVideoTrack videoTrack) + { + if (videoObjects.TryGetValue(participant.Identity, out GameObject meshObject) && meshObject != null) + { + MeshRenderer renderer = meshObject.GetComponent(); + if (renderer != null) + { + // Disable the renderer and clear the texture. + renderer.enabled = false; + renderer.material.mainTexture = null; + } + } + + // Remove the stream from the list of active video streams. + videoObjects.Remove(participant.Identity); + } + } + #endregion + + #region Cleanup Methods + /// + /// Cleans up all video-related objects and stops all video streams and RTC sources. + /// + private void CleanUp() + { + foreach (KeyValuePair videoObject in videoObjects) + { + foreach (RtcVideoSource rtcVideoSource in rtcVideoSources) + { + rtcVideoSource.Stop(); + } + + foreach (VideoStream videoStream in videoStreams) + { + videoStream.Stop(); + } + + rtcVideoSources.Clear(); + videoStreams.Clear(); + } + } + } +} +#endregion diff --git a/Assets/SEE/Tools/Videochat/LivekitVideoManager.cs.meta b/Assets/SEE/Tools/Livekit/LivekitVideoManager.cs.meta similarity index 100% rename from Assets/SEE/Tools/Videochat/LivekitVideoManager.cs.meta rename to Assets/SEE/Tools/Livekit/LivekitVideoManager.cs.meta diff --git a/Assets/SEE/Tools/Videochat/LivekitVideo.cs b/Assets/SEE/Tools/Videochat/LivekitVideo.cs deleted file mode 100644 index 61108e0b0c..0000000000 --- a/Assets/SEE/Tools/Videochat/LivekitVideo.cs +++ /dev/null @@ -1,97 +0,0 @@ -using UnityEngine; -using Unity.Netcode; -using SEE.Controls; -using SEE.Utils; - -public class LivekitVideo : NetworkBehaviour -{ - /// - /// The object with the position where the player's head is located. - /// - private Transform playerHead; - - /// - /// The relative path to the bone of the player's head. - /// This is used to position the Livekit video. - /// - private const string faceCamOrientationBone = "CC_Base_BoneRoot/CC_Base_Hip/CC_Base_Waist/CC_Base_Spine01/CC_Base_Spine02/CC_Base_NeckTwist01/CC_Base_NeckTwist02/CC_Base_Head"; - - /// - /// The status of the position of the Livekit video. - /// Can be positioned in front of the face or above the face, tilted towards the observer. - /// When faceCamOnFront is true, the video is positioned in front of the face using offsetInFrontOfFace. - /// When faceCamOnFront is false, the video is positioned above the head using offsetAboveHead. - /// - private bool faceCamOnFront = true; - - /// - /// Offset for positioning the video in front of the player's face. - /// Used when faceCamOnFront is true. - /// - private Vector3 offsetInFrontOfFace = new Vector3(0, 0.065f, 0.15f); - - /// - /// Offset for positioning the video above the player's head. - /// Used when faceCamOnFront is false. - /// - private Vector3 offsetAboveHead = new Vector3(0, 0.35f, 0); - - /// - /// Called when the script instance is being loaded. - /// Initializes the player head reference and sets the object name so that it contains the owner ID of the client. - /// - private void Start() - { - // Set the name of the object to "LivekitVideo_" followed by the owner client ID. - gameObject.name = "LivekitVideo_" + OwnerClientId; - - // Localizes the player's head bone for the positioning of the video. - playerHead = transform.parent.Find(faceCamOrientationBone); - } - - /// - /// Called once per frame to update the position and rotation of the video. - /// Toggles the position of the video between above the player's head or in front of it. - /// The position can be toggled with . - /// - private void Update() - { - // Update the video position when the player's head is found. - if (playerHead != null) - { - UpdatePosition(); - } - - // Check for input to toggle the video position. - if (SEEInput.ToggleFaceCamPosition()) - { - faceCamOnFront = !faceCamOnFront; - } - } - - /// - /// Updates the position and orientation of the video based on the player's head. - /// - /// If the video is positioned in front of the player's face, it follows - /// the head's position and rotation. If positioned above the head, it faces the camera - /// for remote clients. - private void UpdatePosition() - { - if (faceCamOnFront) - { - // Position the video in front of the player's face. - transform.SetPositionAndRotation(playerHead.TransformPoint(offsetInFrontOfFace), playerHead.rotation); - } - else - { - // Position the video above the player's head. - transform.position = playerHead.TransformPoint(offsetAboveHead); - - // If this object is not owned by the local client, make it face the main camera. - if (!IsOwner && MainCamera.Camera != null) - { - transform.LookAt(MainCamera.Camera.transform); - } - } - } -} diff --git a/Assets/SEE/Tools/Videochat/LivekitVideoManager.cs b/Assets/SEE/Tools/Videochat/LivekitVideoManager.cs deleted file mode 100644 index 5e577c6798..0000000000 --- a/Assets/SEE/Tools/Videochat/LivekitVideoManager.cs +++ /dev/null @@ -1,466 +0,0 @@ -// Code inspired by https://github.com/livekit-examples/unity-example/blob/main/LivekitUnitySampleApp/Assets/LivekitSamples.cs -using System.Collections; -using System.Collections.Generic; -using UnityEngine; -using LiveKit; -using LiveKit.Proto; -using RoomOptions = LiveKit.RoomOptions; -using UnityEngine.UI; -using TMPro; -using Unity.Netcode; -using SEE.Controls; - -/// -/// Manages LiveKit video streams and plays them via the LivekitVideo object in the player objects. -/// Handles publishing/unpublishing local video, subscribing/unsubscribing to remote video, -/// and switching between available camera devices. -/// -public class LivekitVideoManager : NetworkBehaviour -{ - /// - /// The URL of the LiveKit server to connect to. - /// - public string livekitUrl = "ws://localhost:7880"; - - /// - /// The URL used to fetch the access token required for authentication. - /// - public string tokenUrl = "http://localhost:3000"; - - /// - /// The room name to join in LiveKit. - /// - public string roomName = "development"; - - /// - /// The LiveKit room object that manages connection and tracks. - /// - private Room room = null; - - /// - /// The local video track being published to the LiveKit server. - /// - private LocalVideoTrack publishedTrack = null; - - /// - /// The WebCamTexture used to capture the video stream from the selected camera. - /// - private WebCamTexture webCamTexture = null; - - /// - /// The index of the currently selected camera. - /// - private int currentCameraIndex = 0; - - /// - /// An array containing the available webcam devices. - /// - private WebCamDevice[] devices; - - /// - /// The dropdown UI component used to select between different available cameras. - /// - public TMP_Dropdown cameraDropdown; - - /// - /// A dictionary that maps participant identities to the GameObjects that represent their video streams. - /// - private Dictionary _videoObjects = new(); - - /// - /// A list of video sources created from the local webcam that are currently being published to the room. - /// - private List _rtcVideoSources = new(); - - /// - /// A list of video streams from remote participants in the LiveKit room. - /// - private List _videoStreams = new(); - - /// - /// Initializes the video manager by obtaining a token and setting up the camera dropdown. - /// - private void Start() - { - SetupCameraDropdown(); - StartCoroutine(GetToken()); - } - - /// - /// Cleans up resources when the object is destroyed, including stopping the webcam - /// and disconnecting from the LiveKit room. - /// - private void OnDestroy() - { - if (webCamTexture != null) - { - webCamTexture.Stop(); - } - room.Disconnect(); - CleanUp(); - room = null; - } - - /// - /// Toggles video publishing on and off. - /// It can be toggled with . - /// - private void Update() - { - if (SEEInput.ToggleFaceCam()) - { - if (publishedTrack == null) - { - StartCoroutine(PublishVideo()); - } - else - { - StartCoroutine(UnpublishVideo()); - } - } - } - - /// - /// Pauses or resumes the webcam video when the application is paused or resumed. - /// - /// Whether the application is paused. - private void OnApplicationPause(bool pause) - { - if (webCamTexture != null) - { - if (pause) - { - webCamTexture.Pause(); - } - else - { - webCamTexture.Play(); - } - } - } - - // Camera Methods - /// - /// Initializes the camera dropdown menu with the list of available camera devices. - /// This method populates the dropdown with the names of all connected camera devices - /// and sets up a listener to handle camera selection changes. - /// - private void SetupCameraDropdown() - { - // Retrieve the list of available camera devices - devices = WebCamTexture.devices; - - if (devices.Length > 0) - { - List cameraNames = new List(); - - // Iterate through each device and add its name to the list - foreach (var device in devices) - { - cameraNames.Add(string.IsNullOrEmpty(device.name) ? "Unnamed Camera" : device.name); - } - - cameraDropdown.ClearOptions(); - cameraDropdown.AddOptions(cameraNames); - cameraDropdown.onValueChanged.AddListener(OpenSelectedCamera); - - // Open the first camera by default - OpenSelectedCamera(0); - } - else - { - Debug.LogError("[Livekit] No camera devices available"); - } - } - - /// - /// Opens and starts the selected camera device based on the provided index. - /// This method stops any currently active camera, initializes a new WebCamTexture with the selected - /// camera device, and starts capturing video from it. It also republishes the video stream. - /// - /// The index of the selected camera device in the dropdown list. - private void OpenSelectedCamera(int index) - { - if (index >= 0 && index < devices.Length) - { - webCamTexture?.Stop(); - - string selectedDeviceName = devices[index].name; - - // Initialize a new WebCamTexture with the selected camera device - webCamTexture = new WebCamTexture(selectedDeviceName); - webCamTexture.Play(); - - StartCoroutine(RepublishVideo()); - Debug.Log($"[Livekit] Switched to camera: {selectedDeviceName}"); - } - } - - // Connection Methods - /// - /// Fetches an authentication token from the token server and connects to the LiveKit room. - /// Makes a GET request to the token server with the room name and participant name. - /// The name of the participant is the local client ID. - /// - /// Coroutine to handle the asynchronous token request process. - private IEnumerator GetToken() - { - // Send a GET request to the token server to retrieve the token for this client - using (UnityEngine.Networking.UnityWebRequest www = UnityEngine.Networking.UnityWebRequest.Get( - $"{tokenUrl}/getToken?roomName={roomName}&participantName={NetworkManager.Singleton.LocalClientId.ToString()}")) - { - // Wait for the request to complete. - yield return www.SendWebRequest(); - - // Check if the request was successful. - if (www.result == UnityEngine.Networking.UnityWebRequest.Result.Success) - { - // Token received, proceed to join the room using the received token. - StartCoroutine(JoinRoom(www.downloadHandler.text)); - } - } - } - - /// - /// Connects to the LiveKit room using the previously fetched token. - /// Initializes the room, subscribes to events, and connects with provided room options. - /// - /// The authentication token received from the token server. - /// Coroutine that handles the connection to the room. - private IEnumerator JoinRoom(string token) - { - // Initialize a new room instance. - room = new Room(); - - // Subscribe to events related to track management. - room.TrackSubscribed += TrackSubscribed; - room.TrackUnsubscribed += UnTrackSubscribed; - - var options = new RoomOptions(); - - // Attempt to connect to the room using the LiveKit server URL and the provided token. - var connect = room.Connect(livekitUrl, token, options); - yield return connect; - - // Check if the connection was successful. - if (!connect.IsError) - { - Debug.Log("[Livekit] Connected to " + room.Name); - } - } - - // Publish Methods - /// - /// Publishes the local video track to the LiveKit room. - /// Creates a video track from the webcamtexture and publishes it to the room. - /// Updates the mesh object for the local client with the video. - /// The Mesh object is provided by LivekitVideo.prefab, - /// which is instantiated as a immediate child of the Player object. - /// - /// Coroutine to handle the asynchronous publishing process. - private IEnumerator PublishVideo() - { - // Check if the room is initialized. - if (room != null) - { - // Create a video source from the current webcam texture. - var source = new TextureVideoSource(webCamTexture); - - // Create a local video track with the video source. - var track = LocalVideoTrack.CreateVideoTrack("my-video-track", source, room); - - // Define options for publishing the video track. - var options = new TrackPublishOptions - { - VideoCodec = VideoCodec.H264, // Codec to be used for video. - VideoEncoding = new VideoEncoding - { - MaxBitrate = 512000, // Maximum bitrate in bits per second. - // Higher values improve the quality, but require more bandwidth. - MaxFramerate = 30 // Maximum frames per second. - }, - Simulcast = true, // Enable simulcast for better scalability. - // Allows participants different quality levels, but increases the server load. - Source = TrackSource.SourceCamera // Specify the source as the camera. - }; - - // Publish the video track to the room. - var publish = room.LocalParticipant.PublishTrack(track, options); - yield return publish; - - // Check if the publishing was successful. - if (!publish.IsError) - { - Debug.Log("[Livekit] Video track published!"); - publishedTrack = track; - - // Find and update the mesh object for the local client with the video. - var localClientId = NetworkManager.Singleton.LocalClientId.ToString(); - var meshObject = GameObject.Find("LivekitVideo_" + localClientId); - - if (meshObject != null) - { - MeshRenderer renderer = meshObject.GetComponent(); - if (renderer != null) - { - // Enable the renderer and set the texture. - renderer.material.mainTexture = webCamTexture; - renderer.enabled = true; - } - } - - // Store the mesh object in the dictionary. - _videoObjects[localClientId] = meshObject; - } - - // Start capturing and updating the video source. - source.Start(); - StartCoroutine(source.Update()); - _rtcVideoSources.Add(source); - } - } - - /// - /// Unpublishes the local video track from the LiveKit room. - /// Stops the video, disables the mesh renderer, and removes the video track. - /// - /// Coroutine to handle the asynchronous unpublishing process. - private IEnumerator UnpublishVideo() - { - // Check if the room is initialized. - if (room != null) - { - // Unpublish the video track from the room. - var unpublish = room.LocalParticipant.UnpublishTrack(publishedTrack, true); - yield return unpublish; - - // Check if the unpublishing was successful. - if (!unpublish.IsError) - { - Debug.Log("[Livekit] Video track unpublished."); - publishedTrack = null; - - // Find and update the mesh object for the local client. - var localClientId = NetworkManager.Singleton.LocalClientId.ToString(); - if (_videoObjects.TryGetValue(localClientId, out GameObject meshObject) && meshObject != null) - { - MeshRenderer renderer = meshObject.GetComponent(); - if (renderer != null) - { - // Disable the renderer and clear the texture. - renderer.enabled = false; - renderer.material.mainTexture = null; - } - - // Remove the mesh object from the dictionary. - _videoObjects.Remove(localClientId); - } - } - } - } - - /// - /// Republishes the video track, typically used after switching the camera. - /// - /// Coroutine that handles the republishing of the video. - private IEnumerator RepublishVideo() - { - if (publishedTrack != null) - { - yield return StartCoroutine(UnpublishVideo()); - StartCoroutine(PublishVideo()); - } - } - - // Track Methods - /// - /// Callback method that is invoked when a remote track is subscribed. - /// Handles the display of the remote video stream on a mesh object. - /// The Mesh object is provided by LivekitVideo.prefab, - /// which is instantiated as a immediate child of the Player object. - /// - /// The remote track being subscribed to. - /// The publication details of the track. - /// The remote participant owning the track. - private void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant) - { - if (track is RemoteVideoTrack videoTrack) - { - Debug.Log("[Livekit] TrackSubscribed for " + participant.Identity); - - // Find the LivekitVideo object to display the video stream. - var meshObject = GameObject.Find("LivekitVideo_" + participant.Identity); - if (meshObject != null) - { - MeshRenderer renderer = meshObject.GetComponent(); - - // Create a new VideoStream instance for the subscribed track. - var stream = new VideoStream(videoTrack); - stream.TextureReceived += texture => - { - if (renderer != null) - { - // Enable the renderer and set the texture. - renderer.material.mainTexture = texture; - renderer.enabled = true; - } - }; - - _videoObjects[participant.Identity] = meshObject; // Add the VideoStream to the list of video streams. - stream.Start(); // Start the video stream. - StartCoroutine(stream.Update()); // Continuously update the video stream. - _videoStreams.Add(stream); // Add the stream to the list of active video streams. - } - } - } - - /// - /// Callback method that is invoked when a remote track is unsubscribed. - /// Cleans up the mesh object associated with the remote video stream. - /// - /// The remote track being unsubscribed. - /// The publication details of the track. - /// The remote participant owning the track. - private void UnTrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, RemoteParticipant participant) - { - if (track is RemoteVideoTrack videoTrack) - { - if (_videoObjects.TryGetValue(participant.Identity, out GameObject meshObject) && meshObject != null) - { - MeshRenderer renderer = meshObject.GetComponent(); - if (renderer != null) - { - // Disable the renderer and clear the texture. - renderer.enabled = false; - renderer.material.mainTexture = null; - } - } - - // Remove the stream from the list of active video streams. - _videoObjects.Remove(participant.Identity); - } - } - - // Cleanup Methods - /// - /// Cleans up all video-related objects and stops all video streams and RTC sources. - /// - private void CleanUp() - { - foreach (var item in _videoObjects) - { - foreach (var rtcVideoSource in _rtcVideoSources) - { - rtcVideoSource.Stop(); - } - - foreach (var videoStream in _videoStreams) - { - videoStream.Stop(); - } - - _rtcVideoSources.Clear(); - _videoStreams.Clear(); - } - } -}