From eb41e01a04cfcb87b74194b43b0d47c6c56992f7 Mon Sep 17 00:00:00 2001 From: Jakub Pisarek <99591440+sgfn@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:44:34 +0200 Subject: [PATCH 1/4] Add behaviour for (de)payloaders --- .../lib/save_to_file/peer_handler.ex | 19 +++++++---- .../lib/send_from_file/peer_handler.ex | 5 ++- lib/ex_webrtc/rtp/depayloader.ex | 23 +++++++++++++ lib/ex_webrtc/rtp/opus/depayloader.ex | 26 +++++++++++++-- lib/ex_webrtc/rtp/opus/payloader.ex | 26 +++++++++++++-- lib/ex_webrtc/rtp/payloader.ex | 21 ++++++++++++ lib/ex_webrtc/rtp/vp8/depayloader.ex | 32 ++++++++++++++----- lib/ex_webrtc/rtp/vp8/payloader.ex | 15 +++++++-- test/ex_webrtc/rtp/vp8/depayloader_test.exs | 20 ++++++------ 9 files changed, 155 insertions(+), 32 deletions(-) create mode 100644 lib/ex_webrtc/rtp/depayloader.ex create mode 100644 lib/ex_webrtc/rtp/payloader.ex diff --git a/examples/save_to_file/lib/save_to_file/peer_handler.ex b/examples/save_to_file/lib/save_to_file/peer_handler.ex index b28373b..0ecc520 100644 --- a/examples/save_to_file/lib/save_to_file/peer_handler.ex +++ b/examples/save_to_file/lib/save_to_file/peer_handler.ex @@ -54,6 +54,7 @@ defmodule SaveToFile.PeerHandler do video_writer: nil, video_depayloader: nil, audio_writer: nil, + audio_payloader: nil, frames_cnt: 0 } @@ -154,7 +155,13 @@ defmodule SaveToFile.PeerHandler do # by default uses 1 mono channel and 48k clock rate {:ok, audio_writer} = Ogg.Writer.open(@audio_file) - state = %{state | audio_writer: audio_writer, audio_track_id: id} + state = %{ + state + | audio_depayloader: Opus.Depayloader.new(), + audio_writer: audio_writer, + audio_track_id: id + } + {:ok, state} end @@ -166,11 +173,11 @@ defmodule SaveToFile.PeerHandler do defp handle_webrtc_msg({:rtp, id, nil, packet}, %{video_track_id: id} = state) do state = - case VP8.Depayloader.write(state.video_depayloader, packet) do - {:ok, video_depayloader} -> + case VP8.Depayloader.depayload(state.video_depayloader, packet) do + {nil, video_depayloader} -> %{state | video_depayloader: video_depayloader} - {:ok, vp8_frame, video_depayloader} -> + {vp8_frame, video_depayloader} -> frame = %IVF.Frame{timestamp: state.frames_cnt, data: vp8_frame} {:ok, video_writer} = IVF.Writer.write_frame(state.video_writer, frame) @@ -186,10 +193,10 @@ defmodule SaveToFile.PeerHandler do end defp handle_webrtc_msg({:rtp, id, nil, packet}, %{audio_track_id: id} = state) do - opus_packet = Opus.Depayloader.depayload(packet) + {opus_packet, depayloader} = Opus.Depayloader.depayload(state.audio_depayloader, packet) {:ok, audio_writer} = Ogg.Writer.write_packet(state.audio_writer, opus_packet) - {:ok, %{state | audio_writer: audio_writer}} + {:ok, %{state | audio_depayloader: depayloader, audio_writer: audio_writer}} end defp handle_webrtc_msg(_msg, state), do: {:ok, state} diff --git a/examples/send_from_file/lib/send_from_file/peer_handler.ex b/examples/send_from_file/lib/send_from_file/peer_handler.ex index eeb37c2..dcf9b18 100644 --- a/examples/send_from_file/lib/send_from_file/peer_handler.ex +++ b/examples/send_from_file/lib/send_from_file/peer_handler.ex @@ -63,6 +63,7 @@ defmodule SendFromFile.PeerHandler do video_payloader = VP8.Payloader.new(800) {:ok, audio_reader} = Ogg.Reader.open(@audio_file) + audio_payloader = Opus.Payloader.new() state = %{ peer_connection: pc, @@ -71,6 +72,7 @@ defmodule SendFromFile.PeerHandler do video_reader: video_reader, video_payloader: video_payloader, audio_reader: audio_reader, + audio_payloader: audio_payloader, next_video_timestamp: Enum.random(0..@max_rtp_timestamp), next_audio_timestamp: Enum.random(0..@max_rtp_timestamp), next_video_sequence_number: Enum.random(0..@max_rtp_seq_no), @@ -158,7 +160,7 @@ defmodule SendFromFile.PeerHandler do # and time spent on reading and parsing the file Process.send_after(self(), :send_audio, duration) - rtp_packet = Opus.Payloader.payload(packet) + {[rtp_packet], payloader} = Opus.Payloader.payload(state.audio_payloader, packet) rtp_packet = %{ rtp_packet @@ -177,6 +179,7 @@ defmodule SendFromFile.PeerHandler do state = %{ state | audio_reader: reader, + audio_payloader: payloader, next_audio_timestamp: next_timestamp, next_audio_sequence_number: next_sequence_number } diff --git a/lib/ex_webrtc/rtp/depayloader.ex b/lib/ex_webrtc/rtp/depayloader.ex new file mode 100644 index 0000000..11294ea --- /dev/null +++ b/lib/ex_webrtc/rtp/depayloader.ex @@ -0,0 +1,23 @@ +defmodule ExWebRTC.RTP.Depayloader do + @moduledoc """ + Behaviour for ExWebRTC Depayloaders. + """ + + @type depayloader :: struct() + + @doc """ + Creates a new depayloader struct. + + Refer to the modules implementing the behaviour for available options. + """ + @callback new(options :: any()) :: depayloader() + + @doc """ + Processes binary data from a single RTP packet, and outputs a frame if assembled. + + Returns the frame (or `nil` if a frame could not be decoded yet) + together with the updated depayloader struct. + """ + @callback depayload(depayloader(), packet :: ExRTP.Packet.t()) :: + {binary() | nil, depayloader()} +end diff --git a/lib/ex_webrtc/rtp/opus/depayloader.ex b/lib/ex_webrtc/rtp/opus/depayloader.ex index ad89c5b..89e9d11 100644 --- a/lib/ex_webrtc/rtp/opus/depayloader.ex +++ b/lib/ex_webrtc/rtp/opus/depayloader.ex @@ -7,9 +7,31 @@ defmodule ExWebRTC.RTP.Opus.Depayloader do alias ExRTP.Packet + @behaviour ExWebRTC.RTP.Depayloader + + @opaque t :: %__MODULE__{} + + @enforce_keys [] + defstruct @enforce_keys + + @doc """ + Creates a new Opus depayloader struct. + + Does not take any options/parameters. + """ + @impl true + @spec new(any()) :: t() + def new(_unused \\ nil) do + %__MODULE__{} + end + @doc """ Takes Opus packet out of an RTP packet. + + Always returns a binary as the first element. """ - @spec depayload(Packet.t()) :: binary() - def depayload(%Packet{payload: payload}), do: payload + @impl true + @spec depayload(t(), Packet.t()) :: {binary(), t()} + def depayload(%__MODULE__{} = depayloader, %Packet{payload: payload}), + do: {payload, depayloader} end diff --git a/lib/ex_webrtc/rtp/opus/payloader.ex b/lib/ex_webrtc/rtp/opus/payloader.ex index ef1f3f7..ed91b5f 100644 --- a/lib/ex_webrtc/rtp/opus/payloader.ex +++ b/lib/ex_webrtc/rtp/opus/payloader.ex @@ -5,13 +5,33 @@ defmodule ExWebRTC.RTP.Opus.Payloader do Based on [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587). """ + @behaviour ExWebRTC.RTP.Payloader + + @opaque t :: %__MODULE__{} + + @enforce_keys [] + defstruct @enforce_keys + + @doc """ + Creates a new Opus payloader struct. + + Does not take any options/parameters. + """ + @impl true + @spec new(any()) :: t() + def new(_unused \\ nil) do + %__MODULE__{} + end + @doc """ Packs Opus packet into an RTP packet. Fields from RTP header like ssrc, timestamp etc. are set to 0. + Always returns a single RTP packet. """ - @spec payload(binary()) :: ExRTP.Packet.t() - def payload(packet) when packet != <<>> do - ExRTP.Packet.new(packet) + @impl true + @spec payload(t(), binary()) :: {[ExRTP.Packet.t()], t()} + def payload(%__MODULE__{} = payloader, packet) when packet != <<>> do + {[ExRTP.Packet.new(packet)], payloader} end end diff --git a/lib/ex_webrtc/rtp/payloader.ex b/lib/ex_webrtc/rtp/payloader.ex new file mode 100644 index 0000000..c3a4fd9 --- /dev/null +++ b/lib/ex_webrtc/rtp/payloader.ex @@ -0,0 +1,21 @@ +defmodule ExWebRTC.RTP.Payloader do + @moduledoc """ + Behaviour for ExWebRTC Payloaders. + """ + + @type payloader :: struct() + + @doc """ + Creates a new payloader struct. + + Refer to the modules implementing the behaviour for available options. + """ + @callback new(options :: any()) :: payloader() + + @doc """ + Packs a frame into one or more RTP packets. + + Returns the packets together with the updated payloader struct. + """ + @callback payload(payloader(), frame :: binary()) :: {[ExRTP.Packet.t()], payloader()} +end diff --git a/lib/ex_webrtc/rtp/vp8/depayloader.ex b/lib/ex_webrtc/rtp/vp8/depayloader.ex index c56cf15..4456143 100644 --- a/lib/ex_webrtc/rtp/vp8/depayloader.ex +++ b/lib/ex_webrtc/rtp/vp8/depayloader.ex @@ -4,6 +4,9 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). """ + + @behaviour ExWebRTC.RTP.Depayloader + require Logger alias ExWebRTC.RTP.VP8.Payload @@ -15,17 +18,30 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do defstruct [:current_frame, :current_timestamp] - @spec new() :: t() - def new() do + @doc """ + Creates a new VP8 depayloader struct. + + Does not take any options/parameters. + """ + @impl true + @spec new(any()) :: t() + def new(_unused \\ nil) do %__MODULE__{} end - @spec write(t(), ExRTP.Packet.t()) :: {:ok, t()} | {:ok, binary(), t()} - def write(depayloader, packet) + @doc """ + Reassembles VP8 frames from subsequent RTP packets. + + Returns the frame (or `nil` if a frame could not be decoded yet) + together with the updated depayloader struct. + """ + @impl true + @spec depayload(t(), ExRTP.Packet.t()) :: {binary() | nil, t()} + def depayload(depayloader, packet) - def write(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {:ok, depayloader} + def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader} - def write(depayloader, packet) do + def depayload(depayloader, packet) do case Payload.parse(packet.payload) do {:ok, vp8_payload} -> do_write(depayloader, packet, vp8_payload) @@ -80,10 +96,10 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do case {depayloader.current_frame, packet.marker} do {current_frame, true} when current_frame != nil -> - {:ok, current_frame, %{depayloader | current_frame: nil, current_timestamp: nil}} + {current_frame, %{depayloader | current_frame: nil, current_timestamp: nil}} _ -> - {:ok, depayloader} + {nil, depayloader} end end end diff --git a/lib/ex_webrtc/rtp/vp8/payloader.ex b/lib/ex_webrtc/rtp/vp8/payloader.ex index 27b24d1..09447be 100644 --- a/lib/ex_webrtc/rtp/vp8/payloader.ex +++ b/lib/ex_webrtc/rtp/vp8/payloader.ex @@ -8,6 +8,8 @@ defmodule ExWebRTC.RTP.VP8.Payloader do does not pay attention to VP8 partition boundaries (see RFC 7741 sec. 4.4). """ + @behaviour ExWebRTC.RTP.Payloader + @first_chunk_descriptor <<0::1, 0::1, 0::1, 1::1, 0::1, 0::3>> @next_chunk_descriptor <<0::1, 0::1, 0::1, 0::1, 0::1, 0::3>> @@ -18,8 +20,16 @@ defmodule ExWebRTC.RTP.VP8.Payloader do max_payload_size: non_neg_integer() } - defstruct [:max_payload_size] + @enforce_keys [:max_payload_size] + defstruct @enforce_keys + + @doc """ + Creates a new VP8 payloader struct. + The parameter `max_payload_size` determines the maximum size of a single RTP packet + outputted by the payloader. It must be greater than `100`, and is set to `1000` by default. + """ + @impl true @spec new(non_neg_integer()) :: t() def new(max_payload_size \\ 1000) when max_payload_size > 100 do %__MODULE__{max_payload_size: max_payload_size} @@ -30,8 +40,9 @@ defmodule ExWebRTC.RTP.VP8.Payloader do Fields from RTP header like ssrc, timestamp etc. are set to 0. """ + @impl true @spec payload(t(), frame :: binary()) :: {[ExRTP.Packet.t()], t()} - def payload(payloader, frame) when frame != <<>> do + def payload(%__MODULE__{} = payloader, frame) when frame != <<>> do rtp_payloads = chunk(frame, payloader.max_payload_size - @desc_size_bytes) [first_rtp_payload | next_rtp_payloads] = rtp_payloads diff --git a/test/ex_webrtc/rtp/vp8/depayloader_test.exs b/test/ex_webrtc/rtp/vp8/depayloader_test.exs index 7ff1c46..ada0ddc 100644 --- a/test/ex_webrtc/rtp/vp8/depayloader_test.exs +++ b/test/ex_webrtc/rtp/vp8/depayloader_test.exs @@ -14,8 +14,8 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, marker: true) - assert {:ok, ^data, %{current_frame: nil, current_timestamp: nil} = depayloader} = - Depayloader.write(depayloader, packet) + assert {^data, %{current_frame: nil, current_timestamp: nil} = depayloader} = + Depayloader.depayload(depayloader, packet) # packet that doesn't start a new frame vp8_payload = %Payload{n: 0, s: 0, pid: 0, payload: data} @@ -23,8 +23,8 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload) - assert {:ok, %{current_frame: nil, current_timestamp: nil} = depayloader} = - Depayloader.write(depayloader, packet) + assert {nil, %{current_frame: nil, current_timestamp: nil} = depayloader} = + Depayloader.depayload(depayloader, packet) # packet that starts a new frame without finishing the previous one vp8_payload = %Payload{n: 0, s: 1, pid: 0, payload: data} @@ -32,8 +32,8 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload) - assert {:ok, %{current_frame: ^data, current_timestamp: 0} = depayloader} = - Depayloader.write(depayloader, packet) + assert {nil, %{current_frame: ^data, current_timestamp: 0} = depayloader} = + Depayloader.depayload(depayloader, packet) data2 = data <> <<0>> vp8_payload = %Payload{n: 0, s: 1, pid: 0, payload: data2} @@ -41,8 +41,8 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, timestamp: 3000) - assert {:ok, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} = - Depayloader.write(depayloader, packet) + assert {nil, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} = + Depayloader.depayload(depayloader, packet) # packet with timestamp from a new frame that is not a beginning of this frame data2 = data @@ -51,7 +51,7 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, timestamp: 6000) - assert {:ok, %{current_frame: nil, current_timestamp: nil}} = - Depayloader.write(depayloader, packet) + assert {nil, %{current_frame: nil, current_timestamp: nil}} = + Depayloader.depayload(depayloader, packet) end end From fa717faf924584de7741a1f34044645bdfedf632 Mon Sep 17 00:00:00 2001 From: Jakub Pisarek <99591440+sgfn@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:57:39 +0200 Subject: [PATCH 2/4] Review fixes --- examples/save_to_file/lib/save_to_file/peer_handler.ex | 2 +- lib/ex_webrtc/rtp/depayloader.ex | 2 +- lib/ex_webrtc/rtp/opus/depayloader.ex | 3 +-- lib/ex_webrtc/rtp/opus/payloader.ex | 3 +-- lib/ex_webrtc/rtp/vp8/depayloader.ex | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/save_to_file/lib/save_to_file/peer_handler.ex b/examples/save_to_file/lib/save_to_file/peer_handler.ex index 0ecc520..b991f08 100644 --- a/examples/save_to_file/lib/save_to_file/peer_handler.ex +++ b/examples/save_to_file/lib/save_to_file/peer_handler.ex @@ -54,7 +54,7 @@ defmodule SaveToFile.PeerHandler do video_writer: nil, video_depayloader: nil, audio_writer: nil, - audio_payloader: nil, + audio_depayloader: nil, frames_cnt: 0 } diff --git a/lib/ex_webrtc/rtp/depayloader.ex b/lib/ex_webrtc/rtp/depayloader.ex index 11294ea..99c8d58 100644 --- a/lib/ex_webrtc/rtp/depayloader.ex +++ b/lib/ex_webrtc/rtp/depayloader.ex @@ -15,7 +15,7 @@ defmodule ExWebRTC.RTP.Depayloader do @doc """ Processes binary data from a single RTP packet, and outputs a frame if assembled. - Returns the frame (or `nil` if a frame could not be decoded yet) + Returns the frame (or `nil` if a frame could not be depayloaded yet) together with the updated depayloader struct. """ @callback depayload(depayloader(), packet :: ExRTP.Packet.t()) :: diff --git a/lib/ex_webrtc/rtp/opus/depayloader.ex b/lib/ex_webrtc/rtp/opus/depayloader.ex index 89e9d11..0a9a4c1 100644 --- a/lib/ex_webrtc/rtp/opus/depayloader.ex +++ b/lib/ex_webrtc/rtp/opus/depayloader.ex @@ -11,8 +11,7 @@ defmodule ExWebRTC.RTP.Opus.Depayloader do @opaque t :: %__MODULE__{} - @enforce_keys [] - defstruct @enforce_keys + defstruct [] @doc """ Creates a new Opus depayloader struct. diff --git a/lib/ex_webrtc/rtp/opus/payloader.ex b/lib/ex_webrtc/rtp/opus/payloader.ex index ed91b5f..81b188b 100644 --- a/lib/ex_webrtc/rtp/opus/payloader.ex +++ b/lib/ex_webrtc/rtp/opus/payloader.ex @@ -9,8 +9,7 @@ defmodule ExWebRTC.RTP.Opus.Payloader do @opaque t :: %__MODULE__{} - @enforce_keys [] - defstruct @enforce_keys + defstruct [] @doc """ Creates a new Opus payloader struct. diff --git a/lib/ex_webrtc/rtp/vp8/depayloader.ex b/lib/ex_webrtc/rtp/vp8/depayloader.ex index 4456143..d1aabde 100644 --- a/lib/ex_webrtc/rtp/vp8/depayloader.ex +++ b/lib/ex_webrtc/rtp/vp8/depayloader.ex @@ -32,7 +32,7 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do @doc """ Reassembles VP8 frames from subsequent RTP packets. - Returns the frame (or `nil` if a frame could not be decoded yet) + Returns the frame (or `nil` if a frame could not be depayloaded yet) together with the updated depayloader struct. """ @impl true From b142d047a74456d380bd05fd0bcdf8ccbbde4e51 Mon Sep 17 00:00:00 2001 From: Jakub Pisarek <99591440+sgfn@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:49:34 +0200 Subject: [PATCH 3/4] Dynamic dispatch for (de)payloaders (#152) --- .../lib/save_to_file/peer_handler.ex | 13 +++--- .../lib/send_from_file/peer_handler.ex | 10 ++--- lib/ex_webrtc/rtp/depayloader.ex | 41 +++++++++++++++-- lib/ex_webrtc/rtp/payloader.ex | 39 ++++++++++++++-- test/ex_webrtc/rtp/depayloader_test.exs | 44 +++++++++++++++++++ test/ex_webrtc/rtp/payloader_test.exs | 41 +++++++++++++++++ 6 files changed, 172 insertions(+), 16 deletions(-) create mode 100644 test/ex_webrtc/rtp/depayloader_test.exs create mode 100644 test/ex_webrtc/rtp/payloader_test.exs diff --git a/examples/save_to_file/lib/save_to_file/peer_handler.ex b/examples/save_to_file/lib/save_to_file/peer_handler.ex index b991f08..ca32a73 100644 --- a/examples/save_to_file/lib/save_to_file/peer_handler.ex +++ b/examples/save_to_file/lib/save_to_file/peer_handler.ex @@ -10,7 +10,7 @@ defmodule SaveToFile.PeerHandler do } alias ExWebRTC.Media.{IVF, Ogg} - alias ExWebRTC.RTP.{Opus, VP8} + alias ExWebRTC.RTP.Depayloader @behaviour WebSock @@ -141,9 +141,11 @@ defmodule SaveToFile.PeerHandler do timebase_num: 1 ) + {:ok, video_depayloader} = @video_codecs |> hd() |> Depayloader.new() + state = %{ state - | video_depayloader: VP8.Depayloader.new(), + | video_depayloader: video_depayloader, video_writer: video_writer, video_track_id: id } @@ -154,10 +156,11 @@ defmodule SaveToFile.PeerHandler do defp handle_webrtc_msg({:track, %MediaStreamTrack{kind: :audio, id: id}}, state) do # by default uses 1 mono channel and 48k clock rate {:ok, audio_writer} = Ogg.Writer.open(@audio_file) + {:ok, audio_depayloader} = @audio_codecs |> hd() |> Depayloader.new() state = %{ state - | audio_depayloader: Opus.Depayloader.new(), + | audio_depayloader: audio_depayloader, audio_writer: audio_writer, audio_track_id: id } @@ -173,7 +176,7 @@ defmodule SaveToFile.PeerHandler do defp handle_webrtc_msg({:rtp, id, nil, packet}, %{video_track_id: id} = state) do state = - case VP8.Depayloader.depayload(state.video_depayloader, packet) do + case Depayloader.depayload(state.video_depayloader, packet) do {nil, video_depayloader} -> %{state | video_depayloader: video_depayloader} @@ -193,7 +196,7 @@ defmodule SaveToFile.PeerHandler do end defp handle_webrtc_msg({:rtp, id, nil, packet}, %{audio_track_id: id} = state) do - {opus_packet, depayloader} = Opus.Depayloader.depayload(state.audio_depayloader, packet) + {opus_packet, depayloader} = Depayloader.depayload(state.audio_depayloader, packet) {:ok, audio_writer} = Ogg.Writer.write_packet(state.audio_writer, opus_packet) {:ok, %{state | audio_depayloader: depayloader, audio_writer: audio_writer}} diff --git a/examples/send_from_file/lib/send_from_file/peer_handler.ex b/examples/send_from_file/lib/send_from_file/peer_handler.ex index dcf9b18..bbc2f91 100644 --- a/examples/send_from_file/lib/send_from_file/peer_handler.ex +++ b/examples/send_from_file/lib/send_from_file/peer_handler.ex @@ -12,7 +12,7 @@ defmodule SendFromFile.PeerHandler do } alias ExWebRTC.Media.{IVF, Ogg} - alias ExWebRTC.RTP.{Opus, VP8} + alias ExWebRTC.RTP.Payloader @behaviour WebSock @@ -60,10 +60,10 @@ defmodule SendFromFile.PeerHandler do {:ok, _sender} = PeerConnection.add_track(pc, audio_track) {:ok, _header, video_reader} = IVF.Reader.open(@video_file) - video_payloader = VP8.Payloader.new(800) + {:ok, video_payloader} = @video_codecs |> hd() |> Payloader.new(800) {:ok, audio_reader} = Ogg.Reader.open(@audio_file) - audio_payloader = Opus.Payloader.new() + {:ok, audio_payloader} = @audio_codecs |> hd() |> Payloader.new() state = %{ peer_connection: pc, @@ -114,7 +114,7 @@ defmodule SendFromFile.PeerHandler do case IVF.Reader.next_frame(state.video_reader) do {:ok, frame} -> - {rtp_packets, payloader} = VP8.Payloader.payload(state.video_payloader, frame.data) + {rtp_packets, payloader} = Payloader.payload(state.video_payloader, frame.data) # 3_000 = 90_000 (VP8 clock rate) / 30 FPS next_sequence_number = @@ -160,7 +160,7 @@ defmodule SendFromFile.PeerHandler do # and time spent on reading and parsing the file Process.send_after(self(), :send_audio, duration) - {[rtp_packet], payloader} = Opus.Payloader.payload(state.audio_payloader, packet) + {[rtp_packet], payloader} = Payloader.payload(state.audio_payloader, packet) rtp_packet = %{ rtp_packet diff --git a/lib/ex_webrtc/rtp/depayloader.ex b/lib/ex_webrtc/rtp/depayloader.ex index 99c8d58..3037515 100644 --- a/lib/ex_webrtc/rtp/depayloader.ex +++ b/lib/ex_webrtc/rtp/depayloader.ex @@ -1,14 +1,14 @@ defmodule ExWebRTC.RTP.Depayloader do @moduledoc """ - Behaviour for ExWebRTC Depayloaders. + Dispatcher module and behaviour for ExWebRTC Depayloaders. """ + alias ExWebRTC.RTPCodecParameters + @type depayloader :: struct() @doc """ Creates a new depayloader struct. - - Refer to the modules implementing the behaviour for available options. """ @callback new(options :: any()) :: depayloader() @@ -20,4 +20,39 @@ defmodule ExWebRTC.RTP.Depayloader do """ @callback depayload(depayloader(), packet :: ExRTP.Packet.t()) :: {binary() | nil, depayloader()} + + @doc """ + Creates a new depayloader struct that matches the passed codec parameters. + + Refer to the modules implementing the behaviour for available options. + """ + @spec new(RTPCodecParameters.t(), any()) :: + {:ok, depayloader()} | {:error, :no_depayloader_for_codec} + def new(codec_params, options \\ nil) do + with {:ok, module} <- match_depayloader_module(codec_params.mime_type) do + depayloader = if is_nil(options), do: module.new(), else: module.new(options) + + {:ok, depayloader} + end + end + + @doc """ + Processes binary data from a single RTP packet using the depayloader's module, + and outputs a frame if assembled. + + Returns the frame (or `nil` if a frame could not be depayloaded yet) + together with the updated depayloader struct. + """ + @spec depayload(depayloader(), ExRTP.Packet.t()) :: {binary() | nil, depayloader()} + def depayload(%module{} = depayloader, packet) do + module.depayload(depayloader, packet) + end + + defp match_depayloader_module(mime_type) do + case String.downcase(mime_type) do + "video/vp8" -> {:ok, ExWebRTC.RTP.VP8.Depayloader} + "audio/opus" -> {:ok, ExWebRTC.RTP.Opus.Depayloader} + _other -> {:error, :no_depayloader_for_codec} + end + end end diff --git a/lib/ex_webrtc/rtp/payloader.ex b/lib/ex_webrtc/rtp/payloader.ex index c3a4fd9..8f17135 100644 --- a/lib/ex_webrtc/rtp/payloader.ex +++ b/lib/ex_webrtc/rtp/payloader.ex @@ -1,14 +1,14 @@ defmodule ExWebRTC.RTP.Payloader do @moduledoc """ - Behaviour for ExWebRTC Payloaders. + Dispatcher module and behaviour for ExWebRTC Payloaders. """ + alias ExWebRTC.RTPCodecParameters + @type payloader :: struct() @doc """ Creates a new payloader struct. - - Refer to the modules implementing the behaviour for available options. """ @callback new(options :: any()) :: payloader() @@ -18,4 +18,37 @@ defmodule ExWebRTC.RTP.Payloader do Returns the packets together with the updated payloader struct. """ @callback payload(payloader(), frame :: binary()) :: {[ExRTP.Packet.t()], payloader()} + + @doc """ + Creates a new payloader struct that matches the passed codec parameters. + + Refer to the modules implementing the behaviour for available options. + """ + @spec new(RTPCodecParameters.t(), any()) :: + {:ok, payloader()} | {:error, :no_payloader_for_codec} + def new(codec_params, options \\ nil) do + with {:ok, module} <- match_payloader_module(codec_params.mime_type) do + payloader = if is_nil(options), do: module.new(), else: module.new(options) + + {:ok, payloader} + end + end + + @doc """ + Packs a frame into one or more RTP packets using the payloader's module. + + Returns the packets together with the updated payloader struct. + """ + @spec payload(payloader(), binary()) :: {[ExRTP.Packet.t()], payloader()} + def payload(%module{} = payloader, frame) do + module.payload(payloader, frame) + end + + defp match_payloader_module(mime_type) do + case String.downcase(mime_type) do + "video/vp8" -> {:ok, ExWebRTC.RTP.VP8.Payloader} + "audio/opus" -> {:ok, ExWebRTC.RTP.Opus.Payloader} + _other -> {:error, :no_payloader_for_codec} + end + end end diff --git a/test/ex_webrtc/rtp/depayloader_test.exs b/test/ex_webrtc/rtp/depayloader_test.exs new file mode 100644 index 0000000..2f6ba3f --- /dev/null +++ b/test/ex_webrtc/rtp/depayloader_test.exs @@ -0,0 +1,44 @@ +defmodule ExWebRTC.RTP.DepayloaderTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTPCodecParameters + alias ExWebRTC.RTP.Depayloader + alias ExWebRTC.RTP.{Opus, VP8} + + @packet %ExRTP.Packet{ + payload_type: 96, + sequence_number: 0, + timestamp: 0, + ssrc: 0, + payload: <<0, 1, 2, 3>> + } + + test "creates a VP8 depayloader and dispatches calls to its module" do + assert {:ok, depayloader} = + %RTPCodecParameters{payload_type: 96, mime_type: "video/VP8", clock_rate: 90_000} + |> Depayloader.new() + + assert Depayloader.depayload(depayloader, @packet) == + VP8.Depayloader.depayload(depayloader, @packet) + end + + test "creates an Opus depayloader and dispatches calls to its module" do + assert {:ok, depayloader} = + %RTPCodecParameters{ + payload_type: 96, + mime_type: "audio/opus", + clock_rate: 48_000, + channels: 2 + } + |> Depayloader.new() + + assert Depayloader.depayload(depayloader, @packet) == + Opus.Depayloader.depayload(depayloader, @packet) + end + + test "returns error if no depayloader exists for given codec" do + assert {:error, :no_depayloader_for_codec} = + %RTPCodecParameters{payload_type: 97, mime_type: "video/H264", clock_rate: 90_000} + |> Depayloader.new() + end +end diff --git a/test/ex_webrtc/rtp/payloader_test.exs b/test/ex_webrtc/rtp/payloader_test.exs new file mode 100644 index 0000000..2f317de --- /dev/null +++ b/test/ex_webrtc/rtp/payloader_test.exs @@ -0,0 +1,41 @@ +defmodule ExWebRTC.RTP.PayloaderTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTPCodecParameters + alias ExWebRTC.RTP.Payloader + alias ExWebRTC.RTP.{Opus, VP8} + + @frame <<0, 1, 2, 3>> + + test "creates a VP8 payloader and dispatches calls to its module" do + assert {:ok, _payloader} = + %RTPCodecParameters{payload_type: 96, mime_type: "video/VP8", clock_rate: 90_000} + |> Payloader.new() + + # with options + assert {:ok, payloader} = + %RTPCodecParameters{payload_type: 96, mime_type: "video/VP8", clock_rate: 90_000} + |> Payloader.new(800) + + assert Payloader.payload(payloader, @frame) == VP8.Payloader.payload(payloader, @frame) + end + + test "creates an Opus payloader and dispatches calls to its module" do + assert {:ok, payloader} = + %RTPCodecParameters{ + payload_type: 111, + mime_type: "audio/opus", + clock_rate: 48_000, + channels: 2 + } + |> Payloader.new() + + assert Payloader.payload(payloader, @frame) == Opus.Payloader.payload(payloader, @frame) + end + + test "returns error if no payloader exists for given codec" do + assert {:error, :no_payloader_for_codec} = + %RTPCodecParameters{payload_type: 97, mime_type: "video/H264", clock_rate: 90_000} + |> Payloader.new() + end +end From 9488e17502d9029671a89958236ad957726e2162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Fri, 9 Aug 2024 12:55:34 +0200 Subject: [PATCH 4/4] Hide VP8 and Opus Payloader/Depayloader modules (#154) --- .../lib/send_from_file/peer_handler.ex | 2 +- lib/ex_webrtc/rtp/depayloader.ex | 44 ++++++------------- lib/ex_webrtc/rtp/depayloader_behaviour.ex | 19 ++++++++ lib/ex_webrtc/rtp/opus/depayloader.ex | 29 ++++-------- lib/ex_webrtc/rtp/opus/payloader.ex | 29 ++++-------- lib/ex_webrtc/rtp/payloader.ex | 44 ++++++++----------- lib/ex_webrtc/rtp/payloader_behaviour.ex | 17 +++++++ lib/ex_webrtc/rtp/vp8/depayloader.ex | 40 ++++++----------- lib/ex_webrtc/rtp/vp8/payload.ex | 9 ++-- lib/ex_webrtc/rtp/vp8/payloader.ex | 40 ++++++----------- test/ex_webrtc/rtp/depayloader_test.exs | 5 +-- test/ex_webrtc/rtp/payloader_test.exs | 7 ++- test/ex_webrtc/rtp/vp8/depayloader_test.exs | 15 ++++--- test/ex_webrtc/rtp/vp8/payloader_test.exs | 6 +-- 14 files changed, 132 insertions(+), 174 deletions(-) create mode 100644 lib/ex_webrtc/rtp/depayloader_behaviour.ex create mode 100644 lib/ex_webrtc/rtp/payloader_behaviour.ex diff --git a/examples/send_from_file/lib/send_from_file/peer_handler.ex b/examples/send_from_file/lib/send_from_file/peer_handler.ex index bbc2f91..02dbb37 100644 --- a/examples/send_from_file/lib/send_from_file/peer_handler.ex +++ b/examples/send_from_file/lib/send_from_file/peer_handler.ex @@ -60,7 +60,7 @@ defmodule SendFromFile.PeerHandler do {:ok, _sender} = PeerConnection.add_track(pc, audio_track) {:ok, _header, video_reader} = IVF.Reader.open(@video_file) - {:ok, video_payloader} = @video_codecs |> hd() |> Payloader.new(800) + {:ok, video_payloader} = @video_codecs |> hd() |> Payloader.new(max_payload_size: 800) {:ok, audio_reader} = Ogg.Reader.open(@audio_file) {:ok, audio_payloader} = @audio_codecs |> hd() |> Payloader.new() diff --git a/lib/ex_webrtc/rtp/depayloader.ex b/lib/ex_webrtc/rtp/depayloader.ex index 3037515..33f0cce 100644 --- a/lib/ex_webrtc/rtp/depayloader.ex +++ b/lib/ex_webrtc/rtp/depayloader.ex @@ -1,57 +1,41 @@ defmodule ExWebRTC.RTP.Depayloader do @moduledoc """ - Dispatcher module and behaviour for ExWebRTC Depayloaders. - """ - - alias ExWebRTC.RTPCodecParameters - - @type depayloader :: struct() + RTP depayloader. - @doc """ - Creates a new depayloader struct. + It unpacks RTP pakcets into audio/video frames. """ - @callback new(options :: any()) :: depayloader() - @doc """ - Processes binary data from a single RTP packet, and outputs a frame if assembled. + alias ExWebRTC.RTPCodecParameters - Returns the frame (or `nil` if a frame could not be depayloaded yet) - together with the updated depayloader struct. - """ - @callback depayload(depayloader(), packet :: ExRTP.Packet.t()) :: - {binary() | nil, depayloader()} + @opaque depayloader :: struct() @doc """ - Creates a new depayloader struct that matches the passed codec parameters. - - Refer to the modules implementing the behaviour for available options. + Creates a new depayloader that matches the passed codec parameters. """ - @spec new(RTPCodecParameters.t(), any()) :: + @spec new(RTPCodecParameters.t()) :: {:ok, depayloader()} | {:error, :no_depayloader_for_codec} - def new(codec_params, options \\ nil) do - with {:ok, module} <- match_depayloader_module(codec_params.mime_type) do - depayloader = if is_nil(options), do: module.new(), else: module.new(options) - + def new(codec_params) do + with {:ok, module} <- to_depayloader_module(codec_params.mime_type) do + depayloader = module.new() {:ok, depayloader} end end @doc """ - Processes binary data from a single RTP packet using the depayloader's module, - and outputs a frame if assembled. + Processes binary data from a single RTP packet, and outputs a frame if assembled. Returns the frame (or `nil` if a frame could not be depayloaded yet) - together with the updated depayloader struct. + together with the updated depayloader. """ @spec depayload(depayloader(), ExRTP.Packet.t()) :: {binary() | nil, depayloader()} def depayload(%module{} = depayloader, packet) do module.depayload(depayloader, packet) end - defp match_depayloader_module(mime_type) do + defp to_depayloader_module(mime_type) do case String.downcase(mime_type) do - "video/vp8" -> {:ok, ExWebRTC.RTP.VP8.Depayloader} - "audio/opus" -> {:ok, ExWebRTC.RTP.Opus.Depayloader} + "video/vp8" -> {:ok, ExWebRTC.RTP.Depayloader.VP8} + "audio/opus" -> {:ok, ExWebRTC.RTP.Depayloader.Opus} _other -> {:error, :no_depayloader_for_codec} end end diff --git a/lib/ex_webrtc/rtp/depayloader_behaviour.ex b/lib/ex_webrtc/rtp/depayloader_behaviour.ex new file mode 100644 index 0000000..3bae5cf --- /dev/null +++ b/lib/ex_webrtc/rtp/depayloader_behaviour.ex @@ -0,0 +1,19 @@ +defmodule ExWebRTC.RTP.Depayloader.Behaviour do + @moduledoc false + + @type depayloader :: struct() + + @doc """ + Creates a new depayloader struct. + """ + @callback new() :: depayloader() + + @doc """ + Processes binary data from a single RTP packet, and outputs a frame if assembled. + + Returns the frame (or `nil` if a frame could not be depayloaded yet) + together with the updated depayloader struct. + """ + @callback depayload(depayloader(), packet :: ExRTP.Packet.t()) :: + {binary() | nil, depayloader()} +end diff --git a/lib/ex_webrtc/rtp/opus/depayloader.ex b/lib/ex_webrtc/rtp/opus/depayloader.ex index 0a9a4c1..38b75c8 100644 --- a/lib/ex_webrtc/rtp/opus/depayloader.ex +++ b/lib/ex_webrtc/rtp/opus/depayloader.ex @@ -1,34 +1,23 @@ -defmodule ExWebRTC.RTP.Opus.Depayloader do - @moduledoc """ - Decapsualtes Opus audio out of RTP packet. - - Based on [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587). - """ +defmodule ExWebRTC.RTP.Depayloader.Opus do + @moduledoc false + # Decapsualtes Opus audio out of RTP packet. + # + # Based on [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587). alias ExRTP.Packet - @behaviour ExWebRTC.RTP.Depayloader + @behaviour ExWebRTC.RTP.Depayloader.Behaviour - @opaque t :: %__MODULE__{} + @type t :: %__MODULE__{} defstruct [] - @doc """ - Creates a new Opus depayloader struct. - - Does not take any options/parameters. - """ @impl true - @spec new(any()) :: t() - def new(_unused \\ nil) do + @spec new() :: t() + def new() do %__MODULE__{} end - @doc """ - Takes Opus packet out of an RTP packet. - - Always returns a binary as the first element. - """ @impl true @spec depayload(t(), Packet.t()) :: {binary(), t()} def depayload(%__MODULE__{} = depayloader, %Packet{payload: payload}), diff --git a/lib/ex_webrtc/rtp/opus/payloader.ex b/lib/ex_webrtc/rtp/opus/payloader.ex index 81b188b..a0e8215 100644 --- a/lib/ex_webrtc/rtp/opus/payloader.ex +++ b/lib/ex_webrtc/rtp/opus/payloader.ex @@ -1,33 +1,20 @@ -defmodule ExWebRTC.RTP.Opus.Payloader do - @moduledoc """ - Encapsulates Opus audio packet into an RTP packet. +defmodule ExWebRTC.RTP.Payloader.Opus do + @moduledoc false + # Encapsulates Opus audio packet into an RTP packet. + # + # Based on [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587). - Based on [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587). - """ + @behaviour ExWebRTC.RTP.Payloader.Behaviour - @behaviour ExWebRTC.RTP.Payloader - - @opaque t :: %__MODULE__{} + @type t :: %__MODULE__{} defstruct [] - @doc """ - Creates a new Opus payloader struct. - - Does not take any options/parameters. - """ @impl true - @spec new(any()) :: t() - def new(_unused \\ nil) do + def new(_max_payload_size) do %__MODULE__{} end - @doc """ - Packs Opus packet into an RTP packet. - - Fields from RTP header like ssrc, timestamp etc. are set to 0. - Always returns a single RTP packet. - """ @impl true @spec payload(t(), binary()) :: {[ExRTP.Packet.t()], t()} def payload(%__MODULE__{} = payloader, packet) when packet != <<>> do diff --git a/lib/ex_webrtc/rtp/payloader.ex b/lib/ex_webrtc/rtp/payloader.ex index 8f17135..08ac1e3 100644 --- a/lib/ex_webrtc/rtp/payloader.ex +++ b/lib/ex_webrtc/rtp/payloader.ex @@ -1,53 +1,45 @@ defmodule ExWebRTC.RTP.Payloader do @moduledoc """ - Dispatcher module and behaviour for ExWebRTC Payloaders. - """ - - alias ExWebRTC.RTPCodecParameters + RTP payloader. - @type payloader :: struct() - - @doc """ - Creates a new payloader struct. + It packs audio/video frames into one or more RTP packets. """ - @callback new(options :: any()) :: payloader() - @doc """ - Packs a frame into one or more RTP packets. + alias ExWebRTC.RTPCodecParameters - Returns the packets together with the updated payloader struct. - """ - @callback payload(payloader(), frame :: binary()) :: {[ExRTP.Packet.t()], payloader()} + @opaque payloader :: struct() @doc """ - Creates a new payloader struct that matches the passed codec parameters. + Creates a new payloader that matches the passed codec parameters. - Refer to the modules implementing the behaviour for available options. + Opts: + * max_payload_size - determines the maximum size of a single RTP packet outputted by the payloader. + It must be greater than `100`, and is set to `1000` by default. """ - @spec new(RTPCodecParameters.t(), any()) :: + @spec new(RTPCodecParameters.t(), max_payload_size: integer()) :: {:ok, payloader()} | {:error, :no_payloader_for_codec} - def new(codec_params, options \\ nil) do - with {:ok, module} <- match_payloader_module(codec_params.mime_type) do - payloader = if is_nil(options), do: module.new(), else: module.new(options) - + def new(codec_params, opts \\ []) do + with {:ok, module} <- to_payloader_module(codec_params.mime_type) do + max_payload_size = opts[:max_payload_size] || 1000 + payloader = module.new(max_payload_size) {:ok, payloader} end end @doc """ - Packs a frame into one or more RTP packets using the payloader's module. + Packs a frame into one or more RTP packets. - Returns the packets together with the updated payloader struct. + Returns the packets together with the updated payloader. """ @spec payload(payloader(), binary()) :: {[ExRTP.Packet.t()], payloader()} def payload(%module{} = payloader, frame) do module.payload(payloader, frame) end - defp match_payloader_module(mime_type) do + defp to_payloader_module(mime_type) do case String.downcase(mime_type) do - "video/vp8" -> {:ok, ExWebRTC.RTP.VP8.Payloader} - "audio/opus" -> {:ok, ExWebRTC.RTP.Opus.Payloader} + "video/vp8" -> {:ok, ExWebRTC.RTP.Payloader.VP8} + "audio/opus" -> {:ok, ExWebRTC.RTP.Payloader.Opus} _other -> {:error, :no_payloader_for_codec} end end diff --git a/lib/ex_webrtc/rtp/payloader_behaviour.ex b/lib/ex_webrtc/rtp/payloader_behaviour.ex new file mode 100644 index 0000000..81c3df4 --- /dev/null +++ b/lib/ex_webrtc/rtp/payloader_behaviour.ex @@ -0,0 +1,17 @@ +defmodule ExWebRTC.RTP.Payloader.Behaviour do + @moduledoc false + + @type payloader :: struct() + + @doc """ + Creates a new payloader struct. + """ + @callback new(max_payload_size :: integer()) :: payloader() + + @doc """ + Packs a frame into one or more RTP packets. + + Returns the packets together with the updated payloader struct. + """ + @callback payload(payloader(), frame :: binary()) :: {[ExRTP.Packet.t()], payloader()} +end diff --git a/lib/ex_webrtc/rtp/vp8/depayloader.ex b/lib/ex_webrtc/rtp/vp8/depayloader.ex index d1aabde..42b6b2c 100644 --- a/lib/ex_webrtc/rtp/vp8/depayloader.ex +++ b/lib/ex_webrtc/rtp/vp8/depayloader.ex @@ -1,42 +1,28 @@ -defmodule ExWebRTC.RTP.VP8.Depayloader do - @moduledoc """ - Reassembles VP8 frames from RTP packets. +defmodule ExWebRTC.RTP.Depayloader.VP8 do + @moduledoc false + # Reassembles VP8 frames from RTP packets. + # + # Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). - Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). - """ - - @behaviour ExWebRTC.RTP.Depayloader + @behaviour ExWebRTC.RTP.Depayloader.Behaviour require Logger alias ExWebRTC.RTP.VP8.Payload - @opaque t() :: %__MODULE__{ - current_frame: nil, - current_timestamp: nil - } + @type t() :: %__MODULE__{ + current_frame: nil, + current_timestamp: nil + } defstruct [:current_frame, :current_timestamp] - @doc """ - Creates a new VP8 depayloader struct. - - Does not take any options/parameters. - """ @impl true - @spec new(any()) :: t() - def new(_unused \\ nil) do + def new() do %__MODULE__{} end - @doc """ - Reassembles VP8 frames from subsequent RTP packets. - - Returns the frame (or `nil` if a frame could not be depayloaded yet) - together with the updated depayloader struct. - """ @impl true - @spec depayload(t(), ExRTP.Packet.t()) :: {binary() | nil, t()} def depayload(depayloader, packet) def depayload(depayloader, %ExRTP.Packet{payload: <<>>, padding: true}), do: {nil, depayloader} @@ -44,7 +30,7 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do def depayload(depayloader, packet) do case Payload.parse(packet.payload) do {:ok, vp8_payload} -> - do_write(depayloader, packet, vp8_payload) + do_depayload(depayloader, packet, vp8_payload) {:error, reason} -> Logger.warning(""" @@ -56,7 +42,7 @@ defmodule ExWebRTC.RTP.VP8.Depayloader do end end - defp do_write(depayloader, packet, vp8_payload) do + defp do_depayload(depayloader, packet, vp8_payload) do depayloader = case {depayloader.current_frame, vp8_payload} do {nil, %Payload{s: 1, pid: 0}} -> diff --git a/lib/ex_webrtc/rtp/vp8/payload.ex b/lib/ex_webrtc/rtp/vp8/payload.ex index 48338f0..eb74684 100644 --- a/lib/ex_webrtc/rtp/vp8/payload.ex +++ b/lib/ex_webrtc/rtp/vp8/payload.ex @@ -1,9 +1,8 @@ defmodule ExWebRTC.RTP.VP8.Payload do - @moduledoc """ - Defines VP8 payload structure stored in RTP packet payload. - - Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). - """ + @moduledoc false + # Defines VP8 payload structure stored in RTP packet payload. + # + # Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). @type t() :: %__MODULE__{ n: 0 | 1, diff --git a/lib/ex_webrtc/rtp/vp8/payloader.ex b/lib/ex_webrtc/rtp/vp8/payloader.ex index 09447be..bcf374e 100644 --- a/lib/ex_webrtc/rtp/vp8/payloader.ex +++ b/lib/ex_webrtc/rtp/vp8/payloader.ex @@ -1,14 +1,13 @@ -defmodule ExWebRTC.RTP.VP8.Payloader do - @moduledoc """ - Encapsulates VP8 video frames into RTP packets. +defmodule ExWebRTC.RTP.Payloader.VP8 do + @moduledoc false + # Encapsulates VP8 video frames into RTP packets. + # + # Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). + # + # It does not support `X` bit right now, in particular it + # does not pay attention to VP8 partition boundaries (see RFC 7741 sec. 4.4). - Based on [RFC 7741: RTP Payload Format for VP8 Video](https://datatracker.ietf.org/doc/html/rfc7741). - - It does not support `X` bit right now, in particular it - does not pay attention to VP8 partition boundaries (see RFC 7741 sec. 4.4). - """ - - @behaviour ExWebRTC.RTP.Payloader + @behaviour ExWebRTC.RTP.Payloader.Behaviour @first_chunk_descriptor <<0::1, 0::1, 0::1, 1::1, 0::1, 0::3>> @@ -16,32 +15,19 @@ defmodule ExWebRTC.RTP.VP8.Payloader do @desc_size_bytes 1 - @opaque t() :: %__MODULE__{ - max_payload_size: non_neg_integer() - } + @type t() :: %__MODULE__{ + max_payload_size: non_neg_integer() + } @enforce_keys [:max_payload_size] defstruct @enforce_keys - @doc """ - Creates a new VP8 payloader struct. - - The parameter `max_payload_size` determines the maximum size of a single RTP packet - outputted by the payloader. It must be greater than `100`, and is set to `1000` by default. - """ @impl true - @spec new(non_neg_integer()) :: t() - def new(max_payload_size \\ 1000) when max_payload_size > 100 do + def new(max_payload_size) when max_payload_size > 100 do %__MODULE__{max_payload_size: max_payload_size} end - @doc """ - Packs VP8 frame into one or more RTP packets. - - Fields from RTP header like ssrc, timestamp etc. are set to 0. - """ @impl true - @spec payload(t(), frame :: binary()) :: {[ExRTP.Packet.t()], t()} def payload(%__MODULE__{} = payloader, frame) when frame != <<>> do rtp_payloads = chunk(frame, payloader.max_payload_size - @desc_size_bytes) diff --git a/test/ex_webrtc/rtp/depayloader_test.exs b/test/ex_webrtc/rtp/depayloader_test.exs index 2f6ba3f..01c35be 100644 --- a/test/ex_webrtc/rtp/depayloader_test.exs +++ b/test/ex_webrtc/rtp/depayloader_test.exs @@ -3,7 +3,6 @@ defmodule ExWebRTC.RTP.DepayloaderTest do alias ExWebRTC.RTPCodecParameters alias ExWebRTC.RTP.Depayloader - alias ExWebRTC.RTP.{Opus, VP8} @packet %ExRTP.Packet{ payload_type: 96, @@ -19,7 +18,7 @@ defmodule ExWebRTC.RTP.DepayloaderTest do |> Depayloader.new() assert Depayloader.depayload(depayloader, @packet) == - VP8.Depayloader.depayload(depayloader, @packet) + Depayloader.VP8.depayload(depayloader, @packet) end test "creates an Opus depayloader and dispatches calls to its module" do @@ -33,7 +32,7 @@ defmodule ExWebRTC.RTP.DepayloaderTest do |> Depayloader.new() assert Depayloader.depayload(depayloader, @packet) == - Opus.Depayloader.depayload(depayloader, @packet) + Depayloader.Opus.depayload(depayloader, @packet) end test "returns error if no depayloader exists for given codec" do diff --git a/test/ex_webrtc/rtp/payloader_test.exs b/test/ex_webrtc/rtp/payloader_test.exs index 2f317de..ea7fd76 100644 --- a/test/ex_webrtc/rtp/payloader_test.exs +++ b/test/ex_webrtc/rtp/payloader_test.exs @@ -3,7 +3,6 @@ defmodule ExWebRTC.RTP.PayloaderTest do alias ExWebRTC.RTPCodecParameters alias ExWebRTC.RTP.Payloader - alias ExWebRTC.RTP.{Opus, VP8} @frame <<0, 1, 2, 3>> @@ -15,9 +14,9 @@ defmodule ExWebRTC.RTP.PayloaderTest do # with options assert {:ok, payloader} = %RTPCodecParameters{payload_type: 96, mime_type: "video/VP8", clock_rate: 90_000} - |> Payloader.new(800) + |> Payloader.new(max_payload_size: 800) - assert Payloader.payload(payloader, @frame) == VP8.Payloader.payload(payloader, @frame) + assert Payloader.payload(payloader, @frame) == Payloader.VP8.payload(payloader, @frame) end test "creates an Opus payloader and dispatches calls to its module" do @@ -30,7 +29,7 @@ defmodule ExWebRTC.RTP.PayloaderTest do } |> Payloader.new() - assert Payloader.payload(payloader, @frame) == Opus.Payloader.payload(payloader, @frame) + assert Payloader.payload(payloader, @frame) == Payloader.Opus.payload(payloader, @frame) end test "returns error if no payloader exists for given codec" do diff --git a/test/ex_webrtc/rtp/vp8/depayloader_test.exs b/test/ex_webrtc/rtp/vp8/depayloader_test.exs index ada0ddc..ff65619 100644 --- a/test/ex_webrtc/rtp/vp8/depayloader_test.exs +++ b/test/ex_webrtc/rtp/vp8/depayloader_test.exs @@ -1,10 +1,11 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do use ExUnit.Case, async: true - alias ExWebRTC.RTP.VP8.{Payload, Depayloader} + alias ExWebRTC.RTP.Depayloader + alias ExWebRTC.RTP.VP8.Payload test "write/2" do - depayloader = Depayloader.new() + depayloader = Depayloader.VP8.new() # random vp8 data, not necessarily correct data = <<0, 1, 2, 3>> @@ -15,7 +16,7 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, marker: true) assert {^data, %{current_frame: nil, current_timestamp: nil} = depayloader} = - Depayloader.depayload(depayloader, packet) + Depayloader.VP8.depayload(depayloader, packet) # packet that doesn't start a new frame vp8_payload = %Payload{n: 0, s: 0, pid: 0, payload: data} @@ -24,7 +25,7 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload) assert {nil, %{current_frame: nil, current_timestamp: nil} = depayloader} = - Depayloader.depayload(depayloader, packet) + Depayloader.VP8.depayload(depayloader, packet) # packet that starts a new frame without finishing the previous one vp8_payload = %Payload{n: 0, s: 1, pid: 0, payload: data} @@ -33,7 +34,7 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload) assert {nil, %{current_frame: ^data, current_timestamp: 0} = depayloader} = - Depayloader.depayload(depayloader, packet) + Depayloader.VP8.depayload(depayloader, packet) data2 = data <> <<0>> vp8_payload = %Payload{n: 0, s: 1, pid: 0, payload: data2} @@ -42,7 +43,7 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, timestamp: 3000) assert {nil, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} = - Depayloader.depayload(depayloader, packet) + Depayloader.VP8.depayload(depayloader, packet) # packet with timestamp from a new frame that is not a beginning of this frame data2 = data @@ -52,6 +53,6 @@ defmodule ExWebRTC.RTP.VP8.DepayloaderTest do packet = ExRTP.Packet.new(vp8_payload, timestamp: 6000) assert {nil, %{current_frame: nil, current_timestamp: nil}} = - Depayloader.depayload(depayloader, packet) + Depayloader.VP8.depayload(depayloader, packet) end end diff --git a/test/ex_webrtc/rtp/vp8/payloader_test.exs b/test/ex_webrtc/rtp/vp8/payloader_test.exs index cd64260..3721715 100644 --- a/test/ex_webrtc/rtp/vp8/payloader_test.exs +++ b/test/ex_webrtc/rtp/vp8/payloader_test.exs @@ -2,17 +2,17 @@ defmodule ExWebRTC.RTP.VP8.PayloaderTest do use ExUnit.Case, async: true alias ExWebRTC.Media.IVF.Reader - alias ExWebRTC.RTP.VP8.Payloader + alias ExWebRTC.RTP.Payloader test "payload vp8 video" do # video frames in the fixture are mostly 500+ bytes - vp8_payloader = Payloader.new(200) + vp8_payloader = Payloader.VP8.new(200) {:ok, _header, ivf_reader} = Reader.open("test/fixtures/ivf/vp8_correct.ivf") for _i <- 0..28, reduce: vp8_payloader do vp8_payloader -> {:ok, frame} = Reader.next_frame(ivf_reader) - {rtp_packets, vp8_payloader} = Payloader.payload(vp8_payloader, frame.data) + {rtp_packets, vp8_payloader} = Payloader.VP8.payload(vp8_payloader, frame.data) # assert all packets but last are 200 bytes rtp_packets