From 101382afd0feb63243b391e59c529376018d4020 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Tue, 15 Oct 2024 19:00:07 +0800 Subject: [PATCH] RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 v6.0.157 v7.0.18 (#4160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## How to reproduce? 1. Refer this commit, which contains the web demo to capture screen as video stream through RTC. 2. Copy the `trunk/research/players/whip.html` and `trunk/research/players/js/srs.sdk.js` to replace the `develop` branch source code. 3. `./configure && make` 4. `./objs/srs -c conf/rtc2rtmp.conf` 5. open `http://localhost:8080/players/whip.html?schema=http` 6. check `Screen` radio option. 7. click `publish`, then check the screen to share. 8. play the rtmp live stream: `rtmp://localhost/live/livestream` 9. check the video stuttering. ## Cause When capture screen by the chrome web browser, which send RTP packet with empty payload frequently, then all the cached RTP packets are dropped before next key frame arrive in this case. The OBS screen stream and camera stream do not have such problem. ## Add screen stream to WHIP demo >Screenshot 2024-08-28 at 2 49 46 PM --------- Co-authored-by: winlin --- trunk/doc/CHANGELOG.md | 3 ++ trunk/research/players/js/srs.sdk.js | 46 ++++++++++++++++++++-------- trunk/research/players/whip.html | 15 ++++++--- trunk/src/app/srs_app_rtc_source.cpp | 17 ++++++++++ trunk/src/core/srs_core_version5.hpp | 2 +- trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/core/srs_core_version7.hpp | 2 +- 7 files changed, 67 insertions(+), 20 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 18a7b36a43..c831ec94bb 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 7.0 Changelog +* v7.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v7.0.18 (#4160) * v7.0, 2024-10-15, Merge [#3979](https://github.com/ossrs/srs/pull/3979): ST: Use clock_gettime to prevent time jumping backwards. v7.0.17 (#3979) * v7.0, 2024-09-09, Merge [#4158](https://github.com/ossrs/srs/pull/4158): Proxy: Support proxy server for SRS. v7.0.16 (#4158) * v7.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v7.0.15 (#4171) @@ -29,6 +30,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v6.0.157 (#4160) * v6.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v6.0.156 (#4171) * v6.0, 2024-09-01, Merge [#4165](https://github.com/ossrs/srs/pull/4165): FLV: Refine source and http handler. v6.0.155 (#4165) * v6.0, 2024-09-01, Merge [#4166](https://github.com/ossrs/srs/pull/4166): Edge: Fix flv edge crash when http unmount. v6.0.154 (#4166) @@ -189,6 +191,7 @@ The changelog for SRS. ## SRS 5.0 Changelog +* v5.0, 2024-10-15, Merge [#4160](https://github.com/ossrs/srs/pull/4160): RTC2RTMP: Fix screen sharing stutter caused by packet loss. v5.0.216 (#4160) * v5.0, 2024-09-09, Merge [#4171](https://github.com/ossrs/srs/pull/4171): Heartbeat: Report ports for proxy server. v5.0.215 (#4171) * v5.0, 2024-07-24, Merge [#4126](https://github.com/ossrs/srs/pull/4126): Edge: Improve stability for state and fd closing. v5.0.214 (#4126) * v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057) diff --git a/trunk/research/players/js/srs.sdk.js b/trunk/research/players/js/srs.sdk.js index 2a597883a9..0e0c4fd479 100644 --- a/trunk/research/players/js/srs.sdk.js +++ b/trunk/research/players/js/srs.sdk.js @@ -527,36 +527,56 @@ function SrsRtcWhipWhepAsync() { // @url The WebRTC url to publish with, for example: // http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream // @options The options to control playing, supports: - // videoOnly: boolean, whether only play video, default to false. - // audioOnly: boolean, whether only play audio, default to false. + // camera: boolean, whether capture video from camera, default to true. + // screen: boolean, whether capture video from screen, default to false. + // audio: boolean, whether play audio, default to true. self.publish = async function (url, options) { if (url.indexOf('/whip/') === -1) throw new Error(`invalid WHIP url ${url}`); - if (options?.videoOnly && options?.audioOnly) throw new Error(`The videoOnly and audioOnly in options can't be true at the same time`); + const hasAudio = options?.audio ?? true; + const useCamera = options?.camera ?? true; + const useScreen = options?.screen ?? false; + + if (!hasAudio && !useCamera && !useScreen) throw new Error(`The camera, screen and audio can't be false at the same time`); - if (!options?.videoOnly) { + if (hasAudio) { self.pc.addTransceiver("audio", {direction: "sendonly"}); } else { self.constraints.audio = false; } - if (!options?.audioOnly) { + if (useCamera || useScreen) { self.pc.addTransceiver("video", {direction: "sendonly"}); - } else { + } + + if (!useCamera) { self.constraints.video = false; } if (!navigator.mediaDevices && window.location.protocol === 'http:' && window.location.hostname !== 'localhost') { throw new SrsError('HttpsRequiredError', `Please use HTTPS or localhost to publish, read https://github.com/ossrs/srs/issues/2762#issuecomment-983147576`); } - var stream = await navigator.mediaDevices.getUserMedia(self.constraints); - // @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addStream#Migrating_to_addTrack - stream.getTracks().forEach(function (track) { - self.pc.addTrack(track); + if (useScreen) { + const displayStream = await navigator.mediaDevices.getDisplayMedia({ + video: true + }); + // @see https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addStream#Migrating_to_addTrack + displayStream.getTracks().forEach(function (track) { + self.pc.addTrack(track); + // Notify about local track when stream is ok. + self.ontrack && self.ontrack({track: track}); + }); + } - // Notify about local track when stream is ok. - self.ontrack && self.ontrack({track: track}); - }); + if (useCamera || hasAudio) { + const userStream = await navigator.mediaDevices.getUserMedia(self.constraints); + + userStream.getTracks().forEach(function (track) { + self.pc.addTrack(track); + // Notify about local track when stream is ok. + self.ontrack && self.ontrack({track: track}); + }); + } var offer = await self.pc.createOffer(); await self.pc.setLocalDescription(offer); diff --git a/trunk/research/players/whip.html b/trunk/research/players/whip.html index 60067fe7be..11e4ac9932 100644 --- a/trunk/research/players/whip.html +++ b/trunk/research/players/whip.html @@ -62,10 +62,16 @@
Controls: + +
@@ -113,8 +119,9 @@ // For example: webrtc://r.ossrs.net/live/livestream var url = $("#txt_url").val(); sdk.publish(url, { - videoOnly: $('#ch_videoonly').prop('checked'), - audioOnly: $('#ch_audioonly').prop('checked'), + camera: $('#ra_camera').prop('checked'), + screen: $('#ra_screen').prop('checked'), + audio: $('#ch_audio').prop('checked') }).then(function(session){ $('#sessionid').html(session.sessionid); $('#simulator-drop').attr('href', session.simulator + '?drop=1&username=' + session.sessionid); diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 8aebf5d530..5ad61e3f94 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1775,6 +1775,23 @@ srs_error_t SrsRtcFrameBuilder::packet_video_rtmp(const uint16_t start, const ui if (0 == nb_payload) { srs_warn("empty nalu"); + + // The chrome web browser send RTP packet with empty payload frequently, + // reset header_sn_, lost_sn_ and continue to found next frame in this case, + // otherwise, all the cached RTP packets are dropped before next key frame arrive. + header_sn_ = end + 1; + uint16_t tail_sn = 0; + int sn = find_next_lost_sn(header_sn_, tail_sn); + if (-1 == sn) { + if (check_frame_complete(header_sn_, tail_sn)) { + err = packet_video_rtmp(header_sn_, tail_sn); + } + } else if (-2 == sn) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "video cache is overflow"); + } else { + lost_sn_ = sn; + } + return err; } diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp index 791b3ae6ed..1f5323e3f6 100644 --- a/trunk/src/core/srs_core_version5.hpp +++ b/trunk/src/core/srs_core_version5.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 5 #define VERSION_MINOR 0 -#define VERSION_REVISION 215 +#define VERSION_REVISION 216 #endif diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index 69725cd535..f7d8dce7b1 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 156 +#define VERSION_REVISION 157 #endif diff --git a/trunk/src/core/srs_core_version7.hpp b/trunk/src/core/srs_core_version7.hpp index 67461fab92..c4d31842cf 100644 --- a/trunk/src/core/srs_core_version7.hpp +++ b/trunk/src/core/srs_core_version7.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 7 #define VERSION_MINOR 0 -#define VERSION_REVISION 17 +#define VERSION_REVISION 18 #endif \ No newline at end of file