Add support for MediaStreams (#117)

examples/echo/lib/echo/peer_handler.ex

@@ -48,8 +48,9 @@ defmodule Echo.PeerHandler do
         audio_codecs: @audio_codecs
       )
 
-    video_track = MediaStreamTrack.new(:video)
-    audio_track = MediaStreamTrack.new(:audio)
+    stream_id = MediaStreamTrack.generate_stream_id()
+    video_track = MediaStreamTrack.new(:video, [stream_id])
+    audio_track = MediaStreamTrack.new(:audio, [stream_id])
 
     {:ok, _sender} = PeerConnection.add_track(pc, video_track)
     {:ok, _sender} = PeerConnection.add_track(pc, audio_track)

examples/echo/priv/static/script.js

@@ -1,18 +1,16 @@
 const pcConfig = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' },] };
 // we set the resolution manually in order to give simulcast enough bitrate to create 3 encodings
 const mediaConstraints = {video: {width: {ideal: 1280}, height: {ideal: 720}, frameRate: {ideal: 24}}, audio: true}
+const videoPlayer = document.getElementById("videoPlayer");
 
 const proto = window.location.protocol === "https:" ? "wss:" : "ws:"
 const ws = new WebSocket(`${proto}//${window.location.host}/ws`);
 ws.onopen = _ => start_connection(ws);
 ws.onclose = event => console.log("WebSocket connection was terminated:", event);
 
 const start_connection = async (ws) => {
-  const videoPlayer = document.getElementById("videoPlayer");
-  videoPlayer.srcObject = new MediaStream();
-
   const pc = new RTCPeerConnection(pcConfig);
-  pc.ontrack = event => videoPlayer.srcObject.addTrack(event.track);
+  pc.ontrack = event => videoPlayer.srcObject = event.streams[0];
   pc.onicecandidate = event => {
     if (event.candidate === null) return;
 

examples/send_from_file/lib/send_from_file/peer_handler.ex

@@ -59,8 +59,9 @@ defmodule SendFromFile.PeerHandler do
         audio_codecs: @audio_codecs
       )
 
-    video_track = MediaStreamTrack.new(:video)
-    audio_track = MediaStreamTrack.new(:audio)
+    stream_id = MediaStreamTrack.generate_stream_id()
+    video_track = MediaStreamTrack.new(:video, [stream_id])
+    audio_track = MediaStreamTrack.new(:audio, [stream_id])
 
     {:ok, _sender} = PeerConnection.add_track(pc, video_track)
     {:ok, _sender} = PeerConnection.add_track(pc, audio_track)

examples/send_from_file/priv/static/script.js

@@ -1,4 +1,5 @@
 const pcConfig = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' },] };
+const videoPlayer = document.getElementById("videoPlayer");
 
 
 const proto = window.location.protocol === "https:" ? "wss:" : "ws:"
@@ -7,11 +8,8 @@ ws.onopen = _ => start_connection(ws);
 ws.onclose = event => console.log("WebSocket connection was terminated:", event);
 
 const start_connection = async (ws) => {
-  const videoPlayer = document.getElementById("videoPlayer");
-  videoPlayer.srcObject = new MediaStream();
-
   const pc = new RTCPeerConnection(pcConfig);
-  pc.ontrack = event => videoPlayer.srcObject.addTrack(event.track);
+  pc.ontrack = event => videoPlayer.srcObject = event.streams[0];
   pc.onicecandidate = event => {
     if (event.candidate === null) return;
 

examples/whip_whep/lib/whip_whep/peer_supervisor.ex

@@ -85,8 +85,9 @@ defmodule WhipWhep.PeerSupervisor do
 
   defp setup_transceivers(pc, direction) do
     if direction == :sendonly do
-      {:ok, _sender} = PeerConnection.add_track(pc, MediaStreamTrack.new(:audio))
-      {:ok, _sender} = PeerConnection.add_track(pc, MediaStreamTrack.new(:video))
+      stream_id = MediaStreamTrack.generate_stream_id()
+      {:ok, _sender} = PeerConnection.add_track(pc, MediaStreamTrack.new(:audio, [stream_id]))
+      {:ok, _sender} = PeerConnection.add_track(pc, MediaStreamTrack.new(:video, [stream_id]))
     end
 
     transceivers = PeerConnection.get_transceivers(pc)

lib/ex_webrtc/media_stream_track.ex

@@ -6,18 +6,27 @@ defmodule ExWebRTC.MediaStreamTrack do
   alias ExWebRTC.Utils
 
   @type id() :: integer()
+  @type stream_id() :: String.t()
   @type kind() :: :audio | :video
 
   @type t() :: %__MODULE__{
           kind: kind(),
-          id: id()
+          id: id(),
+          streams: [stream_id()]
         }
 
   @enforce_keys [:id, :kind]
-  defstruct @enforce_keys
+  defstruct @enforce_keys ++ [streams: []]
 
   @spec new(kind()) :: t()
-  def new(kind) when kind in [:audio, :video] do
-    %__MODULE__{kind: kind, id: Utils.generate_id()}
+  def new(kind, streams \\ []) when kind in [:audio, :video] do
+    %__MODULE__{kind: kind, id: Utils.generate_id(), streams: streams}
+  end
+
+  @spec generate_stream_id() :: stream_id()
+  def generate_stream_id() do
+    20
+    |> :crypto.strong_rand_bytes()
+    |> Base.encode32()
   end
 end

lib/ex_webrtc/peer_connection.ex

@@ -529,7 +529,8 @@ defmodule ExWebRTC.PeerConnection do
       offer
       |> ExSDP.add_attributes([
         %ExSDP.Attribute.Group{semantics: "BUNDLE", mids: mids},
-        "extmap-allow-mixed"
+        "extmap-allow-mixed",
+        {"msid-semantic", "WMS *"}
       ])
       |> ExSDP.add_media(mlines)
 
@@ -586,7 +587,8 @@ defmodule ExWebRTC.PeerConnection do
         %ExSDP.Attribute.Group{semantics: "BUNDLE", mids: mids},
         # always allow for mixing one- and two-byte RTP header extensions
         # TODO ensure this was also offered
-        "extmap-allow-mixed"
+        "extmap-allow-mixed",
+        {"msid-semantic", "WMS *"}
       ])
       |> ExSDP.add_media(mlines)
 

lib/ex_webrtc/rtp_receiver.ex

@@ -70,10 +70,13 @@ defmodule ExWebRTC.RTPReceiver do
   end
 
   @doc false
-  @spec update(receiver(), RTPCodecParameters.t() | nil, [ExSDP.Attribute.Extmap.t()]) ::
+  @spec update(receiver(), RTPCodecParameters.t() | nil, [ExSDP.Attribute.Extmap.t()], [
+          String.t()
+        ]) ::
           receiver()
-  def update(receiver, codec, rtp_hdr_exts) do
+  def update(receiver, codec, rtp_hdr_exts, stream_ids) do
     simulcast_demuxer = SimulcastDemuxer.update(receiver.simulcast_demuxer, rtp_hdr_exts)
+    track = %MediaStreamTrack{receiver.track | streams: stream_ids}
 
     layers =
       Map.new(receiver.layers, fn {rid, layer} ->
@@ -85,7 +88,8 @@ defmodule ExWebRTC.RTPReceiver do
       receiver
       | codec: codec,
         simulcast_demuxer: simulcast_demuxer,
-        layers: layers
+        layers: layers,
+        track: track
     }
   end
 

lib/ex_webrtc/rtp_transceiver.ex

@@ -182,7 +182,8 @@ defmodule ExWebRTC.RTPTransceiver do
     rtp_hdr_exts = get_rtp_hdr_extensions(mline, config)
     {:mid, mid} = ExSDP.get_attribute(mline, :mid)
 
-    track = MediaStreamTrack.new(mline.type)
+    stream_ids = SDPUtils.get_stream_ids(mline)
+    track = MediaStreamTrack.new(mline.type, stream_ids)
     codec = get_codec(codecs)
     rtx_codec = get_rtx(codecs, codec)
 
@@ -238,8 +239,9 @@ defmodule ExWebRTC.RTPTransceiver do
     rtp_hdr_exts = get_rtp_hdr_extensions(mline, config)
     codec = get_codec(codecs)
     rtx_codec = get_rtx(codecs, codec)
+    stream_ids = SDPUtils.get_stream_ids(mline)
 
-    receiver = RTPReceiver.update(transceiver.receiver, codec, rtp_hdr_exts)
+    receiver = RTPReceiver.update(transceiver.receiver, codec, rtp_hdr_exts, stream_ids)
     sender = RTPSender.update(transceiver.sender, mid, codec, rtx_codec, rtp_hdr_exts)
 
     %{
@@ -457,6 +459,18 @@ defmodule ExWebRTC.RTPTransceiver do
         [rtp_mapping, codec.sdp_fmtp_line, codec.rtcp_fbs]
       end)
 
+    msids =
+      case transceiver.sender.track do
+        nil ->
+          []
+
+        %MediaStreamTrack{id: id, streams: streams} ->
+          case Enum.map(streams, &ExSDP.Attribute.MSID.new(&1, id)) do
+            [] -> [ExSDP.Attribute.MSID.new("-", id)]
+            other -> other
+          end
+      end
+
     attributes =
       if(Keyword.get(opts, :rtcp, false), do: [{"rtcp", "9 IN IP4 0.0.0.0"}], else: []) ++
         Keyword.get(opts, :simulcast, []) ++
@@ -469,7 +483,7 @@ defmodule ExWebRTC.RTPTransceiver do
           {:fingerprint, Keyword.fetch!(opts, :fingerprint)},
           {:setup, Keyword.fetch!(opts, :setup)},
           :rtcp_mux
-        ] ++ transceiver.rtp_hdr_exts
+        ] ++ transceiver.rtp_hdr_exts ++ msids
 
     %ExSDP.Media{
       ExSDP.Media.new(transceiver.kind, 9, "UDP/TLS/RTP/SAVPF", pt)

lib/ex_webrtc/sdp_utils.ex

@@ -96,6 +96,13 @@ defmodule ExWebRTC.SDPUtils do
     |> Enum.reject(&(&1 == nil))
   end
 
+  @spec get_stream_ids(ExSDP.Media.t()) :: [String.t()]
+  def get_stream_ids(media) do
+    ExSDP.get_attributes(media, :msid)
+    |> Enum.reject(fn msid -> msid.id == "-" end)
+    |> Enum.map(fn msid -> msid.id end)
+  end
+
   @spec get_ice_credentials(ExSDP.t()) ::
           {:ok, {binary(), binary()}}
           | {:error,

test/ex_webrtc/peer_connection_test.exs

@@ -736,6 +736,24 @@ defmodule ExWebRTC.PeerConnectionTest do
       assert_receive {:ex_webrtc, ^pc2, {:track, %MediaStreamTrack{kind: :video}}}
       refute_receive {:ex_webrtc, ^pc2, {:track, %MediaStreamTrack{}}}
     end
+
+    test "with media stream" do
+      {:ok, pc1} = PeerConnection.start_link()
+      {:ok, pc2} = PeerConnection.start_link()
+
+      # only track2 is assigned a stream
+      stream_id = MediaStreamTrack.generate_stream_id()
+      track1 = MediaStreamTrack.new(:audio)
+      track2 = MediaStreamTrack.new(:audio, [stream_id])
+
+      {:ok, _sender} = PeerConnection.add_track(pc1, track1)
+      {:ok, _sender} = PeerConnection.add_track(pc1, track2)
+
+      negotiate(pc1, pc2)
+
+      assert_receive {:ex_webrtc, ^pc2, {:track, %MediaStreamTrack{streams: []}}}
+      assert_receive {:ex_webrtc, ^pc2, {:track, %MediaStreamTrack{streams: [^stream_id]}}}
+    end
   end
 
   describe "replace_track/3" do