Merge branch 'develop' into feature/e2e-test

.github/workflows/build.yml

@@ -56,7 +56,9 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install libva-dev libdrm-dev
         if: ${{ matrix.platform.os == 'ubuntu' }}
-      - run: curl -sSf https://rye-up.com/get | RYE_INSTALL_OPTION="--yes" bash
+      - uses: eifinger/setup-rye@v3
+        with:
+          version: 'latest'
       - run: echo "$HOME/.rye/shims" >> $GITHUB_PATH
       - run: rye pin cpython@${{ matrix.python_version }}
       - run: rye sync
@@ -112,7 +114,7 @@ jobs:
           path: "dist/"
 
   build_windows:
-    runs-on: windows-2019
+    runs-on: windows-2022
     timeout-minutes: 60
     strategy:
       fail-fast: false
@@ -139,6 +141,20 @@ jobs:
           name: windows-2019_python-${{ matrix.python_version }}
           path: dist/
 
+  slack_notify_failed:
+    needs: [build_linux_macos, build_windows]
+    runs-on: ubuntu-latest
+    if: failure()
+    steps:
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_CHANNEL: sora-python-sdk
+          SLACK_COLOR: danger
+          SLACK_ICON_EMOJI: ":japanese_ogre:"
+          SLACK_TITLE: Build failed
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
   publish_wheel:
     if: contains(github.ref, 'tags/202')
     needs:

.vscode/settings.json

@@ -19,6 +19,8 @@
     "editor.formatOnSave": true
   },
   "files.associations": {
+    "*.cs": "csharp",
+    "CMakeLists.txt": "cmake",
     "algorithm": "cpp",
     "__bits": "cpp",
     "__config": "cpp",
@@ -119,7 +121,9 @@
     "strstream": "cpp",
     "typeindex": "cpp",
     "source_location": "cpp",
-    "__memory": "cpp"
+    "__memory": "cpp",
+    "compare": "cpp",
+    "concepts": "cpp"
   },
   "C_Cpp.errorSquiggles": "disabled",
   // nanobind 周りでエラーが消えないので全部消す

CHANGES.md

@@ -16,14 +16,18 @@
 
 ## develop
 
+- [UPDATE] nanobind を `2.0.0` に上げて固定する
+  - @melpon
 - [UPDATE] Sora C++ SDK のバージョンを `2024.6.1` に上げる
   - @voluntas
 - [FIX] SoraAudioSink.read が timeout を無視して失敗を返すケースがあったので修正する
   - @enm10k
+- [FIX] SoraAudioSink.read が timeout を無視するケースがある問題を修正した結果、 read の実行タイミングによってはクラッシュするようになったので修正する
+  - @enm10k
 
 ## 2024.2.0
 
-**2024-04-09**
+**日時**: 2024-04-09
 
 - [ADD] Sora Python SDK Samples を `examples` に移動する
   - @voluntas
@@ -38,15 +42,19 @@
   - @voluntas
 - [UPDATE] ruff の最小を ``0.3.0` に上げる
   - @voluntas
-- [UPDATE] Sora C++ SDK のバージョンを `2024.6.0` に上げる
+- [UPDATE] Sora C++ SDK のバージョンを `2024.7.0-canary.2` に上げる
   - libwebrtc で `cricket::MediaEngineDependencies` が廃止された変更に追従する
-  - WEBRTC_BUILD_VERSION を `m122.6261.1.0` に上げる
+  - WEBRTC_BUILD_VERSION を `m125.6422.2.1` に上げる
     - Ubuntu のビルドを通すために、 \_\_assertion_handler というファイルをコピーする処理を追加した
-  - BOOST_VERSION を `1.84.0` に上げる
-  - @enm10k
+    - rtc::TaskQueue が廃止され、webrtc::TaskQueueBase を直接利用する方式変更に追従した
+  - BOOST_VERSION を `1.85.0` に上げる
+  - @enm10k @melpon
 - [UPDATE] Intel VPL を利用した H.265 に対応
   - Sora C++ SDK のバージョンを `2024.6.0` に上げることで対応
   - @enm10k
+- [UPDATE] Github Actions のビルドで windows-2022 を利用する
+  - Sora CPP SDK 2024.7.0-canary (libwebrtc m125) から windows-2022 でビルドする
+  - @miosakuma
 - [FIX] pyproject.toml の `[rye.tool]` virtual = true を削除する
   - virtual = true は pip version 24 からはデフォルトが wheel が削除されるようになったための暫定対応だった
   - そのために pyproject.toml の `build-system.requires` に wheel と setuptools を指定する

VERSION

@@ -1,5 +1,5 @@
-SORA_CPP_SDK_VERSION=2024.6.1
-WEBRTC_BUILD_VERSION=m122.6261.1.0
-BOOST_VERSION=1.84.0
+SORA_CPP_SDK_VERSION=2024.7.0-canary.2
+WEBRTC_BUILD_VERSION=m125.6422.2.1
+BOOST_VERSION=1.85.0
 CMAKE_VERSION=3.28.1
 OPENH264_VERSION=v2.4.1

examples/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "sora-sdk-samples"
-version = "2024.2.0"
+version = "2024.3.0.dev3"
 description = "Sora Python SDK Samples"
 authors = [{ name = "Shiguredo Inc." }]
 dependencies = [
     "opencv-python~=4.9.0.80",
     "opencv-python-headless~=4.9.0.80",
     "sounddevice~=0.4.6",
-    "sora-sdk>=2024.2.0",
+    "sora-sdk>=2024.3.0.dev3",
     "mediapipe~=0.10.1",
     "python-dotenv>=1.0.1",
 ]

examples/requirements-dev.lock

@@ -6,6 +6,7 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
 
 -e file:.
 absl-py==1.4.0
@@ -63,7 +64,7 @@ python-dotenv==1.0.1
 ruff==0.3.0
 six==1.16.0
     # via python-dateutil
-sora-sdk==2024.2.0
+sora-sdk==2024.3.0.dev3
     # via sora-sdk-samples
 sounddevice==0.4.6
     # via mediapipe

examples/requirements.lock

@@ -6,6 +6,7 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
 
 -e file:.
 absl-py==1.4.0
@@ -59,7 +60,7 @@ python-dotenv==1.0.1
     # via sora-sdk-samples
 six==1.16.0
     # via python-dateutil
-sora-sdk==2024.2.0
+sora-sdk==2024.3.0.dev3
     # via sora-sdk-samples
 sounddevice==0.4.6
     # via mediapipe

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "sora_sdk"
 authors = [{ name = "Shiguredo Inc.", email = "contact+pypi@shiguredo.jp" }]
-version = "2024.3.0.dev1"
+version = "2024.3.0.dev3"
 description = "WebRTC SFU Sora Python SDK"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -57,7 +57,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.rye]
 dev-dependencies = [
-    "nanobind~=1.9.2",
+    "nanobind~=2.0.0",
     "setuptools>=69.2",
     "build~=1.1.1",
     "wheel~=0.43.0",

requirements-dev.lock

@@ -13,7 +13,7 @@ auditwheel==6.0.0
 build==1.1.1
 iniconfig==2.0.0
     # via pytest
-nanobind==1.9.2
+nanobind==2.0.0
 numpy==1.26.4
     # via opencv-python
 opencv-python==4.9.0.80
@@ -29,6 +29,6 @@ pyproject-hooks==1.1.0
     # via build
 pytest==8.2.1
 python-dotenv==1.0.1
-ruff==0.4.4
+ruff==0.4.6
 setuptools==70.0.0
 wheel==0.43.0

run.py

@@ -1,4 +1,5 @@
 import hashlib
+import multiprocessing
 import os
 import platform
 import shutil
@@ -763,7 +764,7 @@ def main():
         mkdir_p(sora_build_dir)
         with cd(sora_build_dir):
             cmd(["cmake", BASE_DIR, *cmake_args])
-            cmd(["cmake", "--build", ".", "--config", configuration])
+            cmd(["cmake", "--build", ".", "--config", configuration, f"-j{multiprocessing.cpu_count()}"])
 
         for file in os.listdir(sora_src_dir):
             if file.startswith("sora_sdk_ext.") and (

scripts/package.ubuntu-20.04_armv8_jetson.sh

@@ -25,7 +25,7 @@ apt-get update
 DEBIAN_FRONTEND=noninteractive apt-get install -y \
   curl
 
-curl -sSf https://rye-up.com/get | RYE_INSTALL_OPTION="--yes" bash
+curl -sSf https://rye.astral.sh/get | RYE_INSTALL_OPTION="--yes" bash
 . /root/.rye/env
 
 cd /root/sora-python-sdk

src/dummy_audio_mixer.cpp

@@ -52,12 +52,10 @@ DummyAudioMixer::DummyAudioMixer(webrtc::TaskQueueFactory* task_queue_factory)
    * sora::SoraClientContextConfig::use_audio_device を false にした際に設定される、
    * webrtc::AudioDeviceDummy はループを回さないため、ここでループを作ることとした。
    */
-  task_queue_ =
-      std::make_unique<rtc::TaskQueue>(task_queue_factory_->CreateTaskQueue(
-          "TestAudioDeviceModuleImpl",
-          webrtc::TaskQueueFactory::Priority::NORMAL));
+  task_queue_ = task_queue_factory_->CreateTaskQueue(
+      "TestAudioDeviceModuleImpl", webrtc::TaskQueueFactory::Priority::NORMAL);
 
-  webrtc::RepeatingTaskHandle::Start(task_queue_->Get(), [this]() {
+  webrtc::RepeatingTaskHandle::Start(task_queue_.get(), [this]() {
     ProcessAudio();
     // オーディオフレームは 10 ms ごとに処理するため 10000 us を指定する
     return webrtc::TimeDelta::Micros(10000);

src/dummy_audio_mixer.h

@@ -7,8 +7,9 @@
 #include <api/audio/audio_frame.h>
 #include <api/audio/audio_mixer.h>
 #include <api/scoped_refptr.h>
+#include <api/task_queue/task_queue_base.h>
+#include <api/task_queue/task_queue_factory.h>
 #include <rtc_base/synchronization/mutex.h>
-#include <rtc_base/task_queue.h>
 #include <rtc_base/task_utils/repeating_task.h>
 #include <rtc_base/thread_annotations.h>
 
@@ -44,7 +45,7 @@ class DummyAudioMixer : public webrtc::AudioMixer {
  private:
   void ProcessAudio();
   const webrtc::TaskQueueFactory* task_queue_factory_;
-  std::unique_ptr<rtc::TaskQueue> task_queue_;
+  std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter> task_queue_;
 
   mutable webrtc::Mutex mutex_;
   std::vector<std::unique_ptr<SourceStatus>> audio_source_list_

src/dynamic_h264_decoder.cpp

@@ -4,6 +4,7 @@
 
 // WebRTC
 #include <api/video/i420_buffer.h>
+#include <modules/video_coding/include/video_error_codes.h>
 #include <rtc_base/logging.h>
 #include <third_party/libyuv/include/libyuv.h>
 
@@ -33,8 +34,7 @@ bool DynamicH264Decoder::Configure(const Settings& settings) {
     Release();
     return false;
   }
-  destroy_decoder_ =
-      (DestroyDecoderFunc)::dlsym(handle, "WelsDestroyDecoder");
+  destroy_decoder_ = (DestroyDecoderFunc)::dlsym(handle, "WelsDestroyDecoder");
   if (destroy_decoder_ == nullptr) {
     RTC_LOG(LS_ERROR) << "Failed to dlsym(WelsDestroyDecoder)";
     Release();

src/dynamic_h264_encoder.cpp

@@ -14,18 +14,21 @@
 
 #include "dynamic_h264_encoder.h"
 
-#include <dlfcn.h>
-
 #include <algorithm>
 #include <limits>
 #include <string>
 
+// Linux
+#include <dlfcn.h>
+
 // WebRTC
 #include <absl/strings/match.h>
 #include <absl/types/optional.h>
 #include <api/video/video_codec_constants.h>
 #include <api/video_codecs/scalability_mode.h>
 #include <common_video/libyuv/include/webrtc_libyuv.h>
+#include <modules/video_coding/include/video_codec_interface.h>
+#include <modules/video_coding/include/video_error_codes.h>
 #include <modules/video_coding/svc/create_scalability_structure.h>
 #include <modules/video_coding/utility/simulcast_rate_allocator.h>
 #include <modules/video_coding/utility/simulcast_utility.h>
@@ -173,22 +176,17 @@ static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) {
   }
 }
 
-DynamicH264Encoder::DynamicH264Encoder(const cricket::VideoCodec& codec,
+DynamicH264Encoder::DynamicH264Encoder(const Environment& env,
+                                       H264EncoderSettings settings,
                                        std::string openh264)
-    : packetization_mode_(H264PacketizationMode::SingleNalUnit),
+    : env_(env),
+      packetization_mode_(settings.packetization_mode),
       max_payload_size_(0),
       number_of_cores_(0),
       encoded_image_callback_(nullptr),
       has_reported_init_(false),
       has_reported_error_(false),
       openh264_(std::move(openh264)) {
-  RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
-  std::string packetization_mode_string;
-  if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
-                     &packetization_mode_string) &&
-      packetization_mode_string == "1") {
-    packetization_mode_ = H264PacketizationMode::NonInterleaved;
-  }
   downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
   encoded_images_.reserve(kMaxSimulcastStreams);
   encoders_.reserve(kMaxSimulcastStreams);
@@ -544,7 +542,7 @@ int32_t DynamicH264Encoder::Encode(
 
     encoded_images_[i]._encodedWidth = configurations_[i].width;
     encoded_images_[i]._encodedHeight = configurations_[i].height;
-    encoded_images_[i].SetRtpTimestamp(input_frame.timestamp());
+    encoded_images_[i].SetRtpTimestamp(input_frame.rtp_timestamp());
     encoded_images_[i].SetColorSpace(input_frame.color_space());
     encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
     encoded_images_[i].SetSimulcastIndex(configurations_[i].simulcast_idx);
@@ -576,10 +574,20 @@ int32_t DynamicH264Encoder::Encode(
         codec_specific.codecSpecific.H264.base_layer_sync =
             tid > 0 && tid < tl0sync_limit_[i];
         if (svc_controllers_[i]) {
+          if (encoded_images_[i]._frameType == VideoFrameType::kVideoFrameKey) {
+            // Reset the ScalableVideoController on key frame
+            // to reset the expected dependency structure.
+            layer_frames =
+                svc_controllers_[i]->NextFrameConfig(/* restart= */ true);
+            RTC_CHECK_EQ(layer_frames.size(), 1);
+            RTC_DCHECK_EQ(layer_frames[0].TemporalId(), 0);
+            RTC_DCHECK_EQ(layer_frames[0].IsKeyframe(), true);
+          }
+
           if (layer_frames[0].TemporalId() != tid) {
             RTC_LOG(LS_WARNING)
-                << "Encoder produced a frame for layer S" << (i + 1) << "T"
-                << tid + 1 << " that wasn't requested.";
+                << "Encoder produced a frame with temporal id " << tid
+                << ", expected " << layer_frames[0].TemporalId() << ".";
             continue;
           }
           encoded_images_[i].SetTemporalIndex(tid);