diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection.h index f946af5fa..03a8f5d67 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection.h @@ -244,6 +244,33 @@ class PeerConnection : public webrtc::PeerConnectionObserver, /// connection. bool IsLocalVideoTrackEnabled() const noexcept; + /// Rounding mode of video frame height for |SetFrameHeightRoundMode()|. + /// This is only used on HoloLens 1 (UWP x86). + enum class FrameHeightRoundMode { + /// Leave frames unchanged. + kNone = 0, + + /// Crop frame height to the nearest multiple of 16. + /// ((height - nearestLowerMultipleOf16) / 2) rows are cropped from the top + /// and (height - nearestLowerMultipleOf16 - croppedRowsTop) rows are + /// cropped from the bottom. + kCrop = 1, + + /// Pad frame height to the nearest multiple of 16. + /// ((nearestHigherMultipleOf16 - height) / 2) rows are added symmetrically + /// at the top and (nearestHigherMultipleOf16 - height - addedRowsTop) rows + /// are added symmetrically at the bottom. + kPad = 2 + }; + + /// [HoloLens 1 only] + /// Use this function to select whether resolutions where height is not + /// multiple of 16 should be cropped, padded or left unchanged. Defaults to + /// FrameHeightRoundMode::kCrop to avoid severe artifacts produced by the + /// H.264 hardware encoder. The default value is applied when creating the + /// first peer connection, so can be overridden after it. + static void SetFrameHeightRoundMode(FrameHeightRoundMode value); + // // Audio // diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp index 38c07b4a5..04b6045f4 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp @@ -1278,6 +1278,12 @@ mrsResult MRS_CALL mrsSdpForceCodecs(const char* message, return MRS_SUCCESS; } +void MRS_CALL +mrsSetFrameHeightRoundMode(FrameHeightRoundMode value) { + PeerConnection::SetFrameHeightRoundMode( + (PeerConnection::FrameHeightRoundMode)value); +} + void MRS_CALL mrsMemCpy(void* dst, const void* src, uint64_t size) { memcpy(dst, src, static_cast(size)); } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.h index 129d86976..596480f65 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.h @@ -680,6 +680,13 @@ MRS_API mrsResult MRS_CALL mrsSdpForceCodecs(const char* message, char* buffer, uint64_t* buffer_size); +/// Must be the same as PeerConnection::FrameHeightRoundMode. +enum class FrameHeightRoundMode : int32_t { NONE = 0, CROP = 1, PAD = 2}; + +/// See PeerConnection::SetFrameHeightRoundMode. +MRS_API void MRS_CALL +mrsSetFrameHeightRoundMode(FrameHeightRoundMode value); + // // Generic utilities // diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp index 30a0f0c8c..150c29761 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp @@ -16,6 +16,93 @@ #include +#if defined(_M_IX86) /* x86 */ && defined(WINAPI_FAMILY) && \ + (WINAPI_FAMILY == WINAPI_FAMILY_APP) /* UWP app */ && \ + defined(_WIN32_WINNT_WIN10) && \ + _WIN32_WINNT >= _WIN32_WINNT_WIN10 /* Win10 */ + +// Defined in +// external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/third_party/winuwp_h264/H264Encoder/H264Encoder.cc +static constexpr int kFrameHeightCrop = 1; +extern int webrtc__WinUWPH264EncoderImpl__frame_height_round_mode; + +#include +#include +#include +#include + +namespace { + +bool CheckIfHololens() { + // The best way to check if we are running on Hololens is checking if this is + // a x86 Windows device with a transparent holographic display (AR). + + using namespace Microsoft::WRL; + using namespace Microsoft::WRL::Wrappers; + using namespace ABI::Windows::Foundation; + using namespace ABI::Windows::Graphics::Holographic; + +#define RETURN_IF_ERROR(...) \ + if (FAILED(__VA_ARGS__)) { \ + return false; \ + } + + RoInitializeWrapper initialize(RO_INIT_MULTITHREADED); + + // HolographicSpace.IsAvailable + ComPtr holo_space_statics; + RETURN_IF_ERROR(GetActivationFactory( + HStringReference( + RuntimeClass_Windows_Graphics_Holographic_HolographicSpace) + .Get(), + &holo_space_statics)); + boolean is_holo_space_available; + RETURN_IF_ERROR( + holo_space_statics->get_IsAvailable(&is_holo_space_available)); + if (!is_holo_space_available) { + // Not a holographic device. + return false; + } + + // HolographicDisplay.GetDefault().IsOpaque + ComPtr holo_display_statics; + RETURN_IF_ERROR(GetActivationFactory( + HStringReference( + RuntimeClass_Windows_Graphics_Holographic_HolographicDisplay) + .Get(), + &holo_display_statics)); + ComPtr holo_display; + RETURN_IF_ERROR(holo_display_statics->GetDefault(&holo_display)); + boolean is_opaque; + RETURN_IF_ERROR(holo_display->get_IsOpaque(&is_opaque)); + // Hololens if not opaque (otherwise VR). + return !is_opaque; +#undef RETURN_IF_ERROR +} + +bool IsHololens() { + static bool is_hololens = CheckIfHololens(); + return is_hololens; +} +} // namespace + +namespace Microsoft::MixedReality::WebRTC { +void PeerConnection::SetFrameHeightRoundMode(FrameHeightRoundMode value) { + if (IsHololens()) { + webrtc__WinUWPH264EncoderImpl__frame_height_round_mode = (int)value; + } +} +} // namespace Microsoft::MixedReality::WebRTC + +#else + +namespace Microsoft::MixedReality::WebRTC { +void PeerConnection::SetFrameHeightRoundMode(FrameHeightRoundMode /*value*/) { +} +} // namespace Microsoft::MixedReality::WebRTC + +#endif + namespace { /// Simple observer utility delegating to a given callback on success. @@ -85,6 +172,10 @@ rtc::scoped_refptr PeerConnection::create( webrtc::PeerConnectionFactoryInterface& factory, const webrtc::PeerConnectionInterface::RTCConfiguration& config, mrsPeerConnectionInteropHandle interop_handle) { + // Set the default value for the HL1 workaround before creating any + // connection. This has no effect on other platforms. + SetFrameHeightRoundMode(FrameHeightRoundMode::kCrop); + // Create the PeerConnection object rtc::scoped_refptr peer = new rtc::RefCountedObject(interop_handle); diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs index 45006c5eb..245554237 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs @@ -22,7 +22,7 @@ internal struct mrsBool /// Attribute to decorate managed delegates used as native callbacks (reverse P/Invoke). /// Required by Mono in Ahead-Of-Time (AOT) compiling, and Unity with the IL2CPP backend. /// - /// + /// /// This attribute is required by Mono AOT and Unity IL2CPP, but not by .NET Core or Framework. /// The implementation was copied from the Mono source code (https://github.com/mono/mono). /// The type argument does not seem to be used anywhere in the code, and a stub implementation @@ -85,13 +85,13 @@ public static unsafe extern uint SdpForceCodecs(string message, SdpFilter audioF /// /// Unsafe utility to copy a memory block with stride. - /// + /// /// This utility loops over the rows of the input memory block, and copy them to the output /// memory block, then increment the read and write pointers by the source and destination /// strides, respectively. For each row, exactly bytes are copied, /// even if the row stride is higher. The extra bytes in the destination buffer past the row /// size until the row stride are left untouched. - /// + /// /// This is equivalent to the following pseudo-code: /// /// for (int row = 0; row < elem_count; ++row) { @@ -153,5 +153,12 @@ public static void ThrowOnErrorCode(uint res) throw new ArgumentOutOfRangeException("Invalid ID passed to AddDataChannelAsync()."); } } + + /// + /// See . + /// + /// + [DllImport(dllPath, CallingConvention = CallingConvention.StdCall, EntryPoint = "mrsSetFrameHeightRoundMode")] + public static unsafe extern void SetFrameHeightRoundMode(PeerConnection.FrameHeightRoundMode value); } } diff --git a/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs b/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs index 92bf83cad..ceee80386 100644 --- a/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs +++ b/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs @@ -1301,6 +1301,46 @@ public static Task> GetVideoCaptureFormatsAsync(string }); } + /// + /// Frame height round mode. + /// + /// + public enum FrameHeightRoundMode + { + /// + /// Leave frames unchanged. + /// + None = 0, + + /// + /// Crop frame height to the nearest multiple of 16. + /// ((height - nearestLowerMultipleOf16) / 2) rows are cropped from the top and + /// (height - nearestLowerMultipleOf16 - croppedRowsTop) rows are cropped from the bottom. + /// + Crop = 1, + + /// + /// Pad frame height to the nearest multiple of 16. + /// ((nearestHigherMultipleOf16 - height) / 2) rows are added symmetrically at the top and + /// (nearestHigherMultipleOf16 - height - addedRowsTop) rows are added symmetrically at the bottom. + /// + Pad = 2 + } + + /// + /// [HoloLens 1 only] + /// Use this function to select whether resolutions where height is not multiple of 16 + /// should be cropped, padded or left unchanged. + /// Default is to avoid severe artifacts produced by + /// the H.264 hardware encoder on HoloLens 1. + /// This has no effect on other platforms. + /// + /// The rounding mode for video frames. + public static void SetFrameHeightRoundMode(FrameHeightRoundMode value) + { + Utils.SetFrameHeightRoundMode(value); + } + internal void OnConnected() { IsConnected = true;