ossrs · winlinvip · Nov 23, 2022 · Nov 23, 2022
diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md
@@ -8,6 +8,7 @@ The changelog for SRS.
 
 ## SRS 6.0 Changelog
 
+* v6.0, 2022-11-23, Merge [#3275](https://github.com/ossrs/srs/pull/3275): H265: Support HEVC over HTTP-TS. v6.0.4
 * v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3
 * v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2
 * v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1

diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp
@@ -555,7 +555,7 @@ srs_error_t SrsHlsMuxer::flush_audio(SrsTsMessageCache* cache)
     return err;
 }
 
-srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache)
+srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame)
 {
     srs_error_t err = srs_success;
 
@@ -573,6 +573,12 @@ srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache)
 
     // update the duration of segment.
     current->append(cache->video->dts / 90);
+
+    // The video codec might change during streaming. Note that the frame might be NULL, when reap segment.
+    if (frame && frame->vcodec()) {
+        SrsTsContextWriter* tscw = current->tscw;
+        tscw->update_video_codec(frame->vcodec()->id);
+    }
 
     if ((err = current->tscw->write_video(cache->video)) != srs_success) {
         return srs_error_wrap(err, "hls: write video");
@@ -1025,7 +1031,7 @@ srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts)
     }
 
     // flush video when got one
-    if ((err = muxer->flush_video(tsmc)) != srs_success) {
+    if ((err = muxer->flush_video(tsmc, frame)) != srs_success) {
         return srs_error_wrap(err, "hls: flush video");
     }
 
@@ -1057,7 +1063,7 @@ srs_error_t SrsHlsController::reap_segment()
     }
 
     // segment open, flush video first.
-    if ((err = muxer->flush_video(tsmc)) != srs_success) {
+    if ((err = muxer->flush_video(tsmc, NULL)) != srs_success) {
         return srs_error_wrap(err, "hls: flush video");
     }
 

diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp
@@ -193,7 +193,7 @@ class SrsHlsMuxer
     // Whether current hls muxer is pure audio mode.
     virtual bool pure_audio();
     virtual srs_error_t flush_audio(SrsTsMessageCache* cache);
-    virtual srs_error_t flush_video(SrsTsMessageCache* cache);
+    virtual srs_error_t flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame);
     // Close segment(ts).
     virtual srs_error_t segment_close();
 private:

diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp
@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    3
+#define VERSION_REVISION    4
 
 #endif
diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp
@@ -620,11 +620,11 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size)
     // For HEVC(H.265), try to parse the IDR from NALUs.
     if (c && c->id == SrsVideoCodecIdHEVC) {
 #ifdef SRS_H265
-        SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1);
+        SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(bytes[0]);
         has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
         return err;
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -854,7 +854,7 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp)
         }
         return err;
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -1373,7 +1373,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream)
         // TODO: FIXME: Might need to guess format?
         return do_avc_demux_ibmf_format(stream);
 #else
-        return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled");
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
 #endif
     }
 
@@ -1500,6 +1500,7 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream)
     // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20
     for (int i = 0; i < PictureLength;) {
         // unsigned int((NAL_unit_length+1)*8) NALUnitLength;
+        // TODO: FIXME: Should ignore error? See https://github.com/ossrs/srs-gb28181/commit/a13b9b54938a14796abb9011e7a8ee779439a452
         if (!stream->require(vcodec->NAL_unit_length + 1)) {
             return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d",
                 PictureLength, i, vcodec->NAL_unit_length, stream->left());

diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp
@@ -468,6 +468,7 @@ enum SrsHevcNaluType {
     SrsHevcNaluType_UNSPECIFIED_63,
     SrsHevcNaluType_INVALID,
 };
+#define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1)
 
 struct SrsHevcNalData {
     uint16_t nal_unit_length;

diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp
@@ -266,7 +266,8 @@
     XX(ERROR_INOTIFY_OPENFD                , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \
     XX(ERROR_INOTIFY_WATCH                 , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \
     XX(ERROR_HTTP_URL_UNESCAPE             , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \
-    XX(ERROR_HTTP_WITH_BODY                , 3097, "HttpWithBody", "Failed for HTTP body")
+    XX(ERROR_HTTP_WITH_BODY                , 3097, "HttpWithBody", "Failed for HTTP body") \
+    XX(ERROR_HEVC_DISABLED                 , 3098, "HevcDisabled", "HEVC is disabled")
 
 /**************************************************/
 /* HTTP/StreamConverter protocol error. */

diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp
@@ -43,6 +43,9 @@ string srs_ts_stream2string(SrsTsStream stream)
         case SrsTsStreamAudioAC3: return "AC3";
         case SrsTsStreamAudioDTS: return "AudioDTS";
         case SrsTsStreamVideoH264: return "H.264";
+#ifdef SRS_H265
+        case SrsTsStreamVideoHEVC: return "H.265";
+#endif
         case SrsTsStreamVideoMpeg4: return "MP4";
         case SrsTsStreamAudioMpeg4: return "MP4A";
         default: return "Other";
@@ -285,6 +288,14 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
             vs = SrsTsStreamVideoH264;
             video_pid = TS_VIDEO_AVC_PID;
             break;
+        case SrsVideoCodecIdHEVC:
+#ifdef SRS_H265
+            vs = SrsTsStreamVideoHEVC;
+            video_pid = TS_VIDEO_AVC_PID;
+            break;
+#else
+            return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
+#endif
         case SrsVideoCodecIdDisabled:
             vs = SrsTsStreamReserved;
             break;
@@ -296,7 +307,6 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
         case SrsVideoCodecIdOn2VP6:
         case SrsVideoCodecIdOn2VP6WithAlphaChannel:
         case SrsVideoCodecIdScreenVideoVersion2:
-        case SrsVideoCodecIdHEVC:
         case SrsVideoCodecIdAV1:
             vs = SrsTsStreamReserved;
             break;
@@ -335,10 +345,13 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
         return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec=%d, acodec=%d", vc, ac);
     }
 
-    // when any codec changed, write PAT/PMT table.
+    // When any codec changed, write PAT/PMT table.
     if (vcodec != vc || acodec != ac) {
-        vcodec = vc;
-        acodec = ac;
+        if (vcodec != SrsVideoCodecIdReserved || acodec != SrsAudioCodecIdReserved1) {
+            srs_trace("TS: Refresh PMT when vcodec=%d=>%d, acodec=%d=>%d", vcodec, vc, acodec, ac);
+        }
+        vcodec = vc; acodec = ac;
+
         if ((err = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != srs_success) {
             return srs_error_wrap(err, "ts: encode PAT/PMT");
         }
@@ -355,8 +368,12 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr
 srs_error_t SrsTsContext::encode_pat_pmt(ISrsStreamWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as)
 {
     srs_error_t err = srs_success;
-
-    if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) {
+
+    bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    if (!codec_ok) {
         return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no PID, vs=%d, as=%d", vs, as);
     }
 
@@ -425,8 +442,12 @@ srs_error_t SrsTsContext::encode_pes(ISrsStreamWriter* writer, SrsTsMessage* msg
     if (msg->payload->length() == 0) {
         return err;
     }
-
-    if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) {
+
+    bool codec_ok = (sid == SrsTsStreamVideoH264 || sid == SrsTsStreamAudioAAC || sid == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (sid == SrsTsStreamVideoHEVC);
+#endif
+    if (!codec_ok) {
         srs_info("ts: ignore the unknown stream, sid=%d", sid);
         return err;
     }
@@ -750,10 +771,14 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context,
     pmt->current_next_indicator = 1;
     pmt->section_number = 0;
     pmt->last_section_number = 0;
-
-    // must got one valid codec.
-    srs_assert(vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
-
+
+    // Here we must get the correct codec.
+    bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    srs_assert(codec_ok);
+
     // if mp3 or aac specified, use audio to carry pcr.
     if (as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3) {
         // use audio to carray pcr by default.
@@ -762,8 +787,12 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context,
         pmt->infos.push_back(new SrsTsPayloadPMTESInfo(as, apid));
     }
 
-    // if h.264 specified, use video to carry pcr.
-    if (vs == SrsTsStreamVideoH264) {
+    // If h.264/h.265 specified, use video to carry pcr.
+    codec_ok = (vs == SrsTsStreamVideoH264);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC);
+#endif
+    if (codec_ok) {
         pmt->PCR_PID = vpid;
         pmt->infos.push_back(new SrsTsPayloadPMTESInfo(vs, vpid));
     }
@@ -2533,6 +2562,9 @@ srs_error_t SrsTsPayloadPMT::psi_decode(SrsBuffer* stream)
         // update the apply pid table
         switch (info->stream_type) {
             case SrsTsStreamVideoH264:
+#ifdef SRS_H265
+            case SrsTsStreamVideoHEVC:
+#endif
             case SrsTsStreamVideoMpeg4:
                 packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type);
                 break;
@@ -2616,6 +2648,9 @@ srs_error_t SrsTsPayloadPMT::psi_encode(SrsBuffer* stream)
         // update the apply pid table
         switch (info->stream_type) {
             case SrsTsStreamVideoH264:
+#ifdef SRS_H265
+            case SrsTsStreamVideoHEVC:
+#endif
             case SrsTsStreamVideoMpeg4:
                 packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type);
                 break;
@@ -2685,6 +2720,11 @@ SrsVideoCodecId SrsTsContextWriter::video_codec()
     return vcodec;
 }
 
+void SrsTsContextWriter::update_video_codec(SrsVideoCodecId v)
+{
+    vcodec = v;
+}
+
 SrsEncFileWriter::SrsEncFileWriter()
 {
     memset(iv,0,16);
@@ -2832,8 +2872,17 @@ srs_error_t SrsTsMessageCache::cache_video(SrsVideoFrame* frame, int64_t dts)
     video->dts = dts;
     video->pts = video->dts + frame->cts * 90;
     video->sid = SrsTsPESStreamIdVideoCommon;
-
-    // write video to cache.
+
+    // Write H.265 video frame to cache.
+    if (frame && frame->vcodec()->id == SrsVideoCodecIdHEVC) {
+#ifdef SRS_H265
+        return do_cache_hevc(frame);
+#else
+        return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled");
+#endif
+    }
+
+    // Write H.264 video frame to cache.
     if ((err = do_cache_avc(frame)) != srs_success) {
         return srs_error_wrap(err, "ts: cache avc");
     }
@@ -2924,7 +2973,7 @@ srs_error_t SrsTsMessageCache::do_cache_aac(SrsAudioFrame* frame)
     return err;
 }
 
-void srs_avc_insert_aud(SrsSimpleStream* payload, bool& aud_inserted)
+void srs_avc_insert_aud(SrsSimpleStream* payload, bool aud_inserted)
 {
     // mux the samples in annexb format,
     // ISO_IEC_14496-10-AVC-2012.pdf, page 324.
@@ -3064,6 +3113,52 @@ srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame)
     return err;
 }
 
+#ifdef SRS_H265
+srs_error_t SrsTsMessageCache::do_cache_hevc(SrsVideoFrame* frame)
+{
+    srs_error_t err = srs_success;
+
+    // Whether aud inserted.
+    bool aud_inserted = false;
+
+    SrsVideoCodecConfig* codec = frame->vcodec();
+    srs_assert(codec);
+
+    bool is_sps_pps_appended = false;
+
+    // all sample use cont nalu header, except the sps-pps before IDR frame.
+    for (int i = 0; i < frame->nb_samples; i++) {
+        SrsSample* sample = &frame->samples[i];
+        int32_t size = sample->size;
+
+        if (!sample->bytes || size <= 0) {
+            return srs_error_new(ERROR_HLS_AVC_SAMPLE_SIZE, "ts: invalid avc sample length=%d", size);
+        }
+
+        // Insert aud before NALU for HEVC.
+        SrsHevcNaluType nalu_type = (SrsHevcNaluType)SrsHevcNaluTypeParse(sample->bytes[0]);
+        bool is_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23);
+        if (is_idr && !frame->has_sps_pps && !is_sps_pps_appended) {
+            for (size_t i = 0; i < codec->hevc_dec_conf_record_.nalu_vec.size(); i++) {
+                const SrsHevcHvccNalu& nalu = codec->hevc_dec_conf_record_.nalu_vec[i];
+                if (nalu.num_nalus <= 0 || nalu.nal_data_vec.empty()) continue;
+
+                srs_avc_insert_aud(video->payload, aud_inserted);
+                const SrsHevcNalData& data = nalu.nal_data_vec.at(0);
+                video->payload->append((char*)&data.nal_unit_data[0], (int)data.nal_unit_data.size());
+                is_sps_pps_appended = true;
+            }
+        }
+
+        // Insert the NALU to video in annexb.
+        srs_avc_insert_aud(video->payload, aud_inserted);
+        video->payload->append(sample->bytes, sample->size);
+    }
+
+    return err;
+}
+#endif
+
 SrsTsTransmuxer::SrsTsTransmuxer()
 {
     writer = NULL;
@@ -3158,10 +3253,17 @@ srs_error_t SrsTsTransmuxer::write_video(int64_t timestamp, char* data, int size
     if (format->video->frame_type == SrsVideoAvcFrameTypeVideoInfoFrame) {
         return err;
     }
-
-    if (format->vcodec->id != SrsVideoCodecIdAVC) {
+
+    bool codec_ok = (format->vcodec->id != SrsVideoCodecIdAVC);
+#ifdef SRS_H265
+    codec_ok = codec_ok ? : (format->vcodec->id != SrsVideoCodecIdHEVC);
+#endif
+    if (!codec_ok) {
         return err;
     }
+
+    // The video codec might change during streaming.
+    tscw->update_video_codec(format->vcodec->id);
 
     // ignore sequence header
     if (format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) {