Merge pull request #29 from shiguredo/feature/openh264

OpenH264 に対応

CHANGES.md

@@ -11,6 +11,9 @@
 
 ## develop
 
+- [ADD] OpenH264 に対応（Windows を除く）
+ - @melpon
+
 ## 2023.1.2
 
 **2023-06-28**

CMakeLists.txt

@@ -1,5 +1,5 @@
+cmake_minimum_required(VERSION 3.18)
 project(sora_sdk)
-cmake_minimum_required(VERSION 3.18...3.23)
 
 # Only interpret if() arguments as variables or keywords when unquoted.
 cmake_policy(SET CMP0054 NEW)
@@ -105,8 +105,10 @@ elseif(TARGET_OS STREQUAL "windows")
  # 文字コードを utf-8 として扱うのと、シンボルテーブル数を増やす
  target_compile_options(sora_sdk_ext PRIVATE /utf-8 /bigobj)
  # CRTライブラリを静的リンクさせる
- # MSVC_RUNTIME_LIBRARY で設定ても反映されないため CMAKE_CXX_FLAGS を用いた
- string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
+ set_property(TARGET sora_sdk_ext PROPERTY
+ MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+ set_property(TARGET nanobind-static PROPERTY
+ MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
  target_compile_definitions(sora_sdk_ext
  PRIVATE
  _CONSOLE
@@ -116,6 +118,16 @@ elseif(TARGET_OS STREQUAL "windows")
  HAVE_SNPRINTF
  )
 endif()
+
+# Windows 以外は OpenH264 の動的呼び出しに対応する
+if (NOT TARGET_OS STREQUAL "windows")
+ target_include_directories(sora_sdk_ext PRIVATE ${OPENH264_DIR}/include)
+ target_sources(sora_sdk_ext
+ PRIVATE
+ src/dynamic_h264_decoder.cpp
+ src/dynamic_h264_encoder.cpp)
+endif()
+
 target_link_libraries(sora_sdk_ext PRIVATE Sora::sora)
 
 install(TARGETS sora_sdk_ext LIBRARY DESTINATION .)

VERSION

@@ -3,3 +3,4 @@ WEBRTC_BUILD_VERSION=m114.5735.2.0
 BOOST_VERSION=1.82.0
 LYRA_VERSION=1.3.0
 CMAKE_VERSION=3.26.4
+OPENH264_VERSION=v2.3.1

requirements-dev.lock

@@ -9,7 +9,6 @@
 -e file:.
 auditwheel==5.4.0
 build==0.10.0
-colorama==0.4.6
 exceptiongroup==1.1.1
 iniconfig==2.0.0
 nanobind==1.4.0

run.py

@@ -9,7 +9,8 @@
 import zipfile
 from typing import Callable, Dict, List, NamedTuple, Optional, Union
 
-from pypath import get_python_version, get_python_include_dir, get_python_library
+from pypath import (get_python_include_dir, get_python_library,
+ get_python_version)
 
 
 def mkdir_p(path: str):
@@ -431,6 +432,17 @@ def install_cmake(version, source_dir, install_dir, platform: str, ext):
  extract(path, install_dir, 'cmake')
 
 
+@versioned
+def install_openh264(version, source_dir, install_dir):
+ rm_rf(os.path.join(source_dir, 'openh264'))
+ rm_rf(os.path.join(install_dir, 'openh264'))
+ git_clone_shallow('https://github.com/cisco/openh264.git',
+ version, os.path.join(source_dir, 'openh264'))
+ with cd(os.path.join(source_dir, 'openh264')):
+ cmd([
+ 'make', f'PREFIX={os.path.join(install_dir, "openh264")}', 'install-headers'])
+
+
 class PlatformTarget(object):
  def __init__(self, os, osver, arch):
  self.os = os
@@ -615,6 +627,16 @@ def install_deps(build_platform: PlatformTarget, target_platform: PlatformTarget
  else:
  add_path(os.path.join(install_dir, 'cmake', 'bin'))
 
+ if build_platform.os != 'windows':
+ # OpenH264
+ install_openh264_args = {
+ 'version': version['OPENH264_VERSION'],
+ 'version_file': os.path.join(install_dir, 'openh264.version'),
+ 'source_dir': source_dir,
+ 'install_dir': install_dir,
+ }
+ install_openh264(**install_openh264_args)
+
 
 def cmake_path(path: str) -> str:
  return path.replace('\\', '/')
@@ -672,6 +694,8 @@ def main():
  f"-DWEBRTC_LIBRARY_DIR={cmake_path(webrtc_info.webrtc_library_dir)}")
  cmake_args.append(
  f"-DSORA_DIR={cmake_path(os.path.join(install_dir, 'sora'))}")
+ cmake_args.append(
+ f"-DOPENH264_DIR={cmake_path(os.path.join(install_dir, 'openh264'))}")
  python_version = get_python_version()
  cmake_args.append(f"-DPYTHON_VERSION_STRING={python_version}")
  cmake_args.append(f"-DPYTHON_INCLUDE_DIR={get_python_include_dir(python_version)}")

src/dynamic_h264_decoder.cpp

@@ -0,0 +1,139 @@
+#include "dynamic_h264_decoder.h"
+
+#include <dlfcn.h>
+
+// WebRTC
+#include <api/video/i420_buffer.h>
+#include <rtc_base/logging.h>
+#include <third_party/libyuv/include/libyuv.h>
+
+// OpenH264
+#include <wels/codec_api.h>
+
+namespace webrtc {
+
+DynamicH264Decoder::DynamicH264Decoder(std::string openh264)
+ : openh264_(std::move(openh264)) {}
+DynamicH264Decoder::~DynamicH264Decoder() {
+ Release();
+}
+
+bool DynamicH264Decoder::Configure(const Settings& settings) {
+ Release();
+
+ void* handle = ::dlopen(openh264_.c_str(), RTLD_LAZY);
+ if (handle == nullptr) {
+ RTC_LOG(LS_ERROR) << "Failed to dlopen";
+ return false;
+ }
+ openh264_handle_ = handle;
+ create_decoder_ = (CreateDecoderFunc)::dlsym(handle, "WelsCreateDecoder");
+ if (create_decoder_ == nullptr) {
+ RTC_LOG(LS_ERROR) << "Failed to dlsym(WelsCreateDecoder)";
+ Release();
+ return false;
+ }
+ destroy_decoder_ =
+ (DestroyDecoderFunc)::dlsym(handle, "WelsDestroyDecoder");
+ if (destroy_decoder_ == nullptr) {
+ RTC_LOG(LS_ERROR) << "Failed to dlsym(WelsDestroyDecoder)";
+ Release();
+ return false;
+ }
+
+ ISVCDecoder* decoder = nullptr;
+ int r = create_decoder_(&decoder);
+ if (r != 0) {
+ RTC_LOG(LS_ERROR) << "Failed to WelsCreateDecoder: r=" << r;
+ Release();
+ return false;
+ }
+
+ SDecodingParam param = {};
+ r = decoder->Initialize(&param);
+ if (r != 0) {
+ RTC_LOG(LS_ERROR) << "Failed to ISVCDecoder::Initialize: r=" << r;
+ Release();
+ return false;
+ }
+ decoder_ = decoder;
+
+ return true;
+}
+int32_t DynamicH264Decoder::Release() {
+ if (decoder_ != nullptr) {
+ decoder_->Uninitialize();
+ destroy_decoder_(decoder_);
+ decoder_ = nullptr;
+ }
+
+ if (openh264_handle_ != nullptr) {
+ ::dlclose(openh264_handle_);
+ openh264_handle_ = nullptr;
+ }
+
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t DynamicH264Decoder::RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) {
+ callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t DynamicH264Decoder::Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t render_time_ms) {
+ if (decoder_ == nullptr) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+
+ h264_bitstream_parser_.ParseBitstream(input_image);
+ absl::optional<int> qp = h264_bitstream_parser_.GetLastSliceQp();
+
+ std::array<std::uint8_t*, 3> yuv;
+ SBufferInfo info = {};
+ int r = decoder_->DecodeFrameNoDelay(input_image.data(), input_image.size(),
+ yuv.data(), &info);
+ if (r != 0) {
+ RTC_LOG(LS_ERROR) << "Failed to ISVCDecoder::DecodeFrameNoDelay: r=" << r;
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ if (info.iBufferStatus == 0) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ int width_y = info.UsrData.sSystemBuffer.iWidth;
+ int height_y = info.UsrData.sSystemBuffer.iHeight;
+ int width_uv = (width_y + 1) / 2;
+ int height_uv = (height_y + 1) / 2;
+ int stride_y = info.UsrData.sSystemBuffer.iStride[0];
+ int stride_uv = info.UsrData.sSystemBuffer.iStride[1];
+ rtc::scoped_refptr<webrtc::I420Buffer> i420_buffer(
+ webrtc::I420Buffer::Create(width_y, height_y));
+ libyuv::I420Copy(yuv[0], stride_y, yuv[1], stride_uv, yuv[2], stride_uv,
+ i420_buffer->MutableDataY(), i420_buffer->StrideY(),
+ i420_buffer->MutableDataU(), i420_buffer->StrideU(),
+ i420_buffer->MutableDataV(), i420_buffer->StrideV(), width_y,
+ height_y);
+
+ webrtc::VideoFrame video_frame =
+ webrtc::VideoFrame::Builder()
+ .set_video_frame_buffer(i420_buffer)
+ .set_timestamp_rtp(input_image.Timestamp())
+ .build();
+ if (input_image.ColorSpace() != nullptr) {
+ video_frame.set_color_space(*input_image.ColorSpace());
+ }
+
+ callback_->Decoded(video_frame, absl::nullopt, qp);
+
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+const char* DynamicH264Decoder::ImplementationName() const {
+ return "OpenH264";
+}
+
+} // namespace webrtc

src/dynamic_h264_decoder.h

@@ -0,0 +1,51 @@
+#ifndef DYNAMIC_H264_DECODER_H_
+#define DYNAMIC_H264_DECODER_H_
+
+#include <memory>
+
+// WebRTC
+#include <common_video/h264/h264_bitstream_parser.h>
+#include <modules/video_coding/codecs/h264/include/h264.h>
+
+class ISVCDecoder;
+
+namespace webrtc {
+
+class DynamicH264Decoder : public H264Decoder {
+ public:
+ static std::unique_ptr<VideoDecoder> Create(std::string openh264) {
+ return std::unique_ptr<VideoDecoder>(
+ new DynamicH264Decoder(std::move(openh264)));
+ }
+
+ DynamicH264Decoder(std::string openh264);
+ ~DynamicH264Decoder() override;
+
+ bool Configure(const Settings& settings) override;
+ int32_t Release() override;
+
+ int32_t RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) override;
+
+ int32_t Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t render_time_ms = -1) override;
+
+ const char* ImplementationName() const override;
+
+ private:
+ DecodedImageCallback* callback_ = nullptr;
+ ISVCDecoder* decoder_ = nullptr;
+ webrtc::H264BitstreamParser h264_bitstream_parser_;
+
+ std::string openh264_;
+ void* openh264_handle_ = nullptr;
+ using CreateDecoderFunc = int (*)(ISVCDecoder**);
+ using DestroyDecoderFunc = void (*)(ISVCDecoder*);
+ CreateDecoderFunc create_decoder_ = nullptr;
+ DestroyDecoderFunc destroy_decoder_ = nullptr;
+};
+
+} // namespace webrtc
+
+#endif