Merge pull request #81 from shiguredo/feature/add-on-signaling

on_signaling_message を追加します

CHANGES.md

@@ -11,6 +11,8 @@
 
 ## develop
 
+- [ADD] on_signaling_message コールバックを追加する
+  - @tnoho
 - [UPDATE] Sora C++ SDK のバージョンを `2024.8.0` に上げる
   - @torikizi
 - [UPDATE] libwebrtc のバージョンを `m128.6613.2.0` に上げる

VERSION

@@ -1,4 +1,4 @@
-SORA_CPP_SDK_VERSION=2024.8.0-canary.0
+SORA_CPP_SDK_VERSION=2024.8.0-canary.1
 WEBRTC_BUILD_VERSION=m128.6613.2.0
 BOOST_VERSION=1.85.0
 CMAKE_VERSION=3.29.6

examples/src/media/__init__.py

@@ -14,7 +14,9 @@
     SoraAudioSink,
     SoraConnection,
     SoraMediaTrack,
+    SoraSignalingDirection,
     SoraSignalingErrorCode,
+    SoraSignalingType,
     SoraVideoFrame,
     SoraVideoSink,
 )
@@ -95,6 +97,16 @@ def __init__(
         self._closed: Event = Event()
         self._default_connection_timeout_s: float = 10.0
 
+        # メッセージ
+        self._connect_message: Optional[dict[str, Any]] = None
+        self._redirect_message: Optional[dict[str, Any]] = None
+        self._offer_message: Optional[dict[str, Any]] = None
+        self._answer_message: Optional[dict[str, Any]] = None
+        self._candidate_messages: list[dict[str, Any]] = []
+        self._re_offer_messages: list[dict[str, Any]] = []
+        self._re_answer_messages: list[dict[str, Any]] = []
+
+        self._connection.on_signaling_message = self._on_signaling_message
         self._connection.on_set_offer = self._on_set_offer
         self._connection.on_switched = self._on_switched
         self._connection.on_notify = self._on_notify
@@ -131,6 +143,34 @@ def get_stats(self):
         raw_stats = self._connection.get_stats()
         return json.loads(raw_stats)
 
+    @property
+    def connect_message(self) -> Optional[dict[str, Any]]:
+        return self._connect_message
+
+    @property
+    def redirect_message(self) -> Optional[dict[str, Any]]:
+        return self._redirect_message
+
+    @property
+    def offer_message(self) -> Optional[dict[str, Any]]:
+        return self._offer_message
+
+    @property
+    def answer_message(self) -> Optional[dict[str, Any]]:
+        return self._answer_message
+
+    @property
+    def candidate_messages(self) -> list[dict[str, Any]]:
+        return self._candidate_messages
+
+    @property
+    def re_offer_messages(self) -> list[dict[str, Any]]:
+        return self._re_offer_messages
+
+    @property
+    def re_answer_messages(self) -> list[dict[str, Any]]:
+        return self._re_answer_messages
+
     @property
     def connected(self) -> bool:
         return self._connected.is_set()
@@ -149,6 +189,40 @@ def _fake_video_loop(self):
             time.sleep(1.0 / 30)
             self._video_source.on_captured(numpy.zeros((480, 640, 3), dtype=numpy.uint8))
 
+    def _on_signaling_message(
+        self,
+        signaling_type: SoraSignalingType,
+        signaling_direction: SoraSignalingDirection,
+        raw_message: str,
+    ):
+        print(raw_message)
+        message: dict[str, Any] = json.loads(raw_message)
+        match message["type"]:
+            case "connect":
+                assert signaling_type == SoraSignalingType.WEBSOCKET
+                assert signaling_direction == SoraSignalingDirection.SENT
+                self._connect_message = message
+            case "redirect":
+                assert signaling_type == SoraSignalingType.WEBSOCKET
+                assert signaling_direction == SoraSignalingDirection.RECEIVED
+                self._redirect_message = message
+            case "offer":
+                assert signaling_type == SoraSignalingType.WEBSOCKET
+                assert signaling_direction == SoraSignalingDirection.RECEIVED
+                self._offer_message = message
+            case "answer":
+                assert signaling_type == SoraSignalingType.WEBSOCKET
+                assert signaling_direction == SoraSignalingDirection.SENT
+                self._answer_message = message
+            case "candidate":
+                self._candidate_messages.append(message)
+            case "re-offer":
+                self._re_offer_messages.append(message)
+            case "re-answer":
+                self._re_answer_messages.append(message)
+            case _:
+                NotImplementedError(f"Unknown signaling message type: {message['type']}")
+
     def _on_set_offer(self, raw_message: str) -> None:
         """
         オファー設定イベントを処理します。

examples/tests/test_signaling_message.py

@@ -0,0 +1,30 @@
+import sys
+import time
+import uuid
+
+from media import Sendonly
+
+
+def test_signaling_message(setup):
+    signaling_urls = setup.get("signaling_urls")
+    channel_id_prefix = setup.get("channel_id_prefix")
+    metadata = setup.get("metadata")
+
+    channel_id = f"{channel_id_prefix}_{__name__}_{sys._getframe().f_code.co_name}_{uuid.uuid4()}"
+
+    sendonly = Sendonly(
+        signaling_urls,
+        channel_id,
+        audio=False,
+        video=True,
+        metadata=metadata,
+    )
+    sendonly.connect(fake_audio=True)
+
+    time.sleep(5)
+
+    assert sendonly.connect_message is not None
+    assert sendonly.offer_message is not None
+    assert sendonly.answer_message is not None
+
+    sendonly.disconnect()

src/sora_connection.cpp

@@ -3,6 +3,9 @@
 #include <chrono>
 #include <stdexcept>
 
+// WebRTC
+#include <rtc_base/crypto_random.h>
+
 // Sora C++ SDK
 #include <sora/rtc_stats.h>
 
@@ -186,6 +189,14 @@ void SoraConnection::OnSwitched(std::string text) {
   }
 }
 
+void SoraConnection::OnSignalingMessage(sora::SoraSignalingType type,
+                                        sora::SoraSignalingDirection direction,
+                                        std::string message) {
+  if (on_signaling_message_) {
+    on_signaling_message_(type, direction, message);
+  }
+}
+
 void SoraConnection::OnTrack(
     rtc::scoped_refptr<webrtc::RtpTransceiverInterface> transceiver) {
   if (on_track_) {

src/sora_connection.h

@@ -102,13 +102,19 @@ class SoraConnection : public sora::SoraSignalingObserver,
   void OnPush(std::string text) override;
   void OnMessage(std::string label, std::string data) override;
   void OnSwitched(std::string text) override;
+  void OnSignalingMessage(sora::SoraSignalingType type,
+                          sora::SoraSignalingDirection direction,
+                          std::string message) override;
   void OnTrack(
       rtc::scoped_refptr<webrtc::RtpTransceiverInterface> transceiver) override;
   void OnRemoveTrack(
       rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) override;
   void OnDataChannel(std::string label) override;
 
   // sora::SoraSignalingObserver のコールバック関数が呼び出された時に対応して呼び出す Python の関数を保持する
+  std::function<
+      void(sora::SoraSignalingType, sora::SoraSignalingDirection, std::string)>
+      on_signaling_message_;
   std::function<void(std::string)> on_set_offer_;
   std::function<void(sora::SoraSignalingErrorCode, std::string)> on_disconnect_;
   std::function<void(std::string)> on_notify_;

src/sora_sdk_ext.cpp

@@ -103,6 +103,12 @@ int connection_tp_traverse(PyObject* self, visitproc visit, void* arg) {
     Py_VISIT(on_disconnect.ptr());
   }
 
+  if (conn->on_signaling_message_) {
+    nb::object on_disconnect =
+        nb::cast(conn->on_signaling_message_, nb::rv_policy::none);
+    Py_VISIT(on_disconnect.ptr());
+  }
+
   if (conn->on_notify_) {
     nb::object on_notify = nb::cast(conn->on_notify_, nb::rv_policy::none);
     Py_VISIT(on_notify.ptr());
@@ -162,6 +168,16 @@ NB_MODULE(sora_sdk_ext, m) {
              sora::SoraSignalingErrorCode::PEER_CONNECTION_STATE_FAILED)
       .value("ICE_FAILED", sora::SoraSignalingErrorCode::ICE_FAILED);
 
+  nb::enum_<sora::SoraSignalingType>(m, "SoraSignalingType",
+                                     nb::is_arithmetic())
+      .value("WEBSOCKET", sora::SoraSignalingType::WEBSOCKET)
+      .value("DATACHANNEL", sora::SoraSignalingType::DATACHANNEL);
+
+  nb::enum_<sora::SoraSignalingDirection>(m, "SoraSignalingDirection",
+                                          nb::is_arithmetic())
+      .value("SENT", sora::SoraSignalingDirection::SENT)
+      .value("RECEIVED", sora::SoraSignalingDirection::RECEIVED);
+
   nb::enum_<webrtc::MediaStreamTrackInterface::TrackState>(m, "SoraTrackState",
                                                            nb::is_arithmetic())
       .value("LIVE", webrtc::MediaStreamTrackInterface::TrackState::kLive)
@@ -286,6 +302,7 @@ NB_MODULE(sora_sdk_ext, m) {
       .def("get_stats", &SoraConnection::GetStats)
       .def_rw("on_set_offer", &SoraConnection::on_set_offer_)
       .def_rw("on_disconnect", &SoraConnection::on_disconnect_)
+      .def_rw("on_signaling_message", &SoraConnection::on_signaling_message_)
       .def_rw("on_notify", &SoraConnection::on_notify_)
       .def_rw("on_push", &SoraConnection::on_push_)
       .def_rw("on_message", &SoraConnection::on_message_)

src/sora_vad.cpp

@@ -8,7 +8,6 @@
 #include <modules/audio_processing/agc2/agc2_common.h>
 #include <modules/audio_processing/agc2/cpu_features.h>
 #include <modules/audio_processing/agc2/rnn_vad/common.h>
-#include <modules/audio_processing/include/audio_frame_view.h>
 
 SoraVAD::SoraVAD() {
   vad_ = std::make_unique<webrtc::VoiceActivityDetectorWrapper>(
@@ -36,7 +35,5 @@ float SoraVAD::Analyze(std::shared_ptr<SoraAudioFrame> frame) {
         webrtc::StreamConfig(frame->sample_rate_hz(), frame->num_channels());
   }
   audio_buffer_->CopyFrom(frame->RawData(), vad_input_config_);
-  return vad_->Analyze(webrtc::AudioFrameView<const float>(
-      audio_buffer_->channels(), audio_buffer_->num_channels(),
-      audio_buffer_->num_frames()).view());
+  return vad_->Analyze(audio_buffer_->view());
 }