From bbe39aeb1391ae643fe816245cefe0036b31e398 Mon Sep 17 00:00:00 2001 From: ZSC714725 Date: Fri, 15 Dec 2023 09:26:04 +0800 Subject: [PATCH] lalmax support safari hevc(whep) --- rtc/packer.go | 153 +++++++++++++++++++++++++++++++++++++++++++-- rtc/server.go | 6 ++ rtc/whepsession.go | 28 ++++++++- 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/rtc/packer.go b/rtc/packer.go index facaad0..ec35bbb 100644 --- a/rtc/packer.go +++ b/rtc/packer.go @@ -2,15 +2,24 @@ package rtc import ( "fmt" + "math/rand" "github.com/pion/rtp" - "github.com/pion/webrtc/v3" "github.com/q191201771/lal/pkg/avc" "github.com/q191201771/lal/pkg/base" + "github.com/q191201771/lal/pkg/hevc" "github.com/q191201771/lal/pkg/rtprtcp" "github.com/q191201771/naza/pkg/nazalog" ) +const ( + PacketH264 = "H264" + PacketHEVC = "HEVC" + PacketSafariHevc = "SafariHevc" + PacketPCMA = "PCMA" + PacketPCMU = "PCMU" +) + type Packer struct { enc IRtpEncoder } @@ -19,12 +28,14 @@ func NewPacker(mimeType string, codec []byte) *Packer { p := &Packer{} switch mimeType { - case webrtc.MimeTypeH264: + case PacketH264: p.enc = NewH264RtpEncoder(codec) - case webrtc.MimeTypePCMA: + case PacketPCMA: p.enc = NewG711RtpEncoder(8) - case webrtc.MimeTypePCMU: + case PacketPCMU: p.enc = NewG711RtpEncoder(0) + case PacketSafariHevc: + p.enc = NewSafariHEVCRtpEncoder(codec) } return p } @@ -108,7 +119,7 @@ func (enc *H264RtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) { pkts = append(pkts, &newRtpPkt) } - if len(rtpPkts) == 0 { + if len(pkts) == 0 { return nil, fmt.Errorf("Packetize failed") } @@ -148,9 +159,139 @@ func (enc *G711RtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) { pkts = append(pkts, &newRtpPkt) } - if len(rtpPkts) == 0 { + if len(pkts) == 0 { + return nil, fmt.Errorf("Packetize failed") + } + + return pkts, nil +} + +type SafariHEVCRtpEncoder struct { + IRtpEncoder + vps []byte + sps []byte + pps []byte + payloadType int + ssrc int + seqId uint16 + tsBase int64 +} + +func NewSafariHEVCRtpEncoder(codec []byte) *SafariHEVCRtpEncoder { + vps, sps, pps, err := hevc.ParseVpsSpsPpsFromSeqHeader(codec) + if err != nil { + nazalog.Error(err) + return nil + } + + return &SafariHEVCRtpEncoder{ + vps: vps, + sps: sps, + pps: pps, + payloadType: 98, + ssrc: 0, + seqId: uint16(rand.Int() % 65536), + tsBase: -1, + } +} + +func (enc *SafariHEVCRtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) { + var pkts []*rtp.Packet + var out []byte + var keyFrame bool + + if enc.tsBase == -1 { + enc.tsBase = int64(msg.Dts()) + } + + err := avc.IterateNaluAvcc(msg.Payload[5:], func(nal []byte) { + t := hevc.ParseNaluType(nal[0]) + if t == hevc.NaluTypeSei { + return + } + + if hevc.IsIrapNalu(t) { + keyFrame = true + out = append(out, avc.NaluStartCode3...) + out = append(out, enc.vps...) + out = append(out, avc.NaluStartCode3...) + out = append(out, enc.sps...) + out = append(out, avc.NaluStartCode3...) + out = append(out, enc.pps...) + } + + out = append(out, avc.NaluStartCode3...) + out = append(out, nal...) + }) + + if err != nil { + return nil, fmt.Errorf("Packetize failed") + } + + if len(out) == 0 { + return nil, fmt.Errorf("Packetize failed") + } + + payloads := enc.doPacketNaluForSafariHevc(out, keyFrame) + for i, payload := range payloads { + var pkt rtp.Packet + pkt.Version = 2 + pkt.Timestamp = uint32((int64(msg.Dts()) - enc.tsBase) * 90) + pkt.PayloadType = uint8(enc.payloadType) + pkt.SSRC = uint32(enc.ssrc) + + if i == len(payloads)-1 { + pkt.Marker = true + } + + pkt.SequenceNumber = enc.seqId + enc.seqId += 1 + pkt.Payload = payload + + pkts = append(pkts, &pkt) + } + + if len(pkts) == 0 { return nil, fmt.Errorf("Packetize failed") } return pkts, nil } + +func (enc *SafariHEVCRtpEncoder) doPacketNaluForSafariHevc(nalu []byte, keyFrame bool) [][]byte { + var rtpPayloads [][]byte + + naluLen := len(nalu) + maxPayloadSize := 1200 + splitNum := naluLen/maxPayloadSize + 1 + remainder := naluLen % splitNum + referenceLen := naluLen / splitNum + dataPos := 0 + + for i := splitNum; i > 0; i-- { + tmpLen := referenceLen + if i < remainder { + tmpLen++ + } + buf := make([]byte, tmpLen+1) + if keyFrame { + if i == splitNum { + buf[0] = 3 + } else { + buf[0] = 1 + } + } else { + if i == splitNum { + buf[0] = 2 + } else { + buf[0] = 0 + } + } + copy(buf[1:], nalu[dataPos:dataPos+tmpLen]) + dataPos += tmpLen + + rtpPayloads = append(rtpPayloads, buf) + } + + return rtpPayloads +} diff --git a/rtc/server.go b/rtc/server.go index cef88f7..3ece4af 100644 --- a/rtc/server.go +++ b/rtc/server.go @@ -4,6 +4,7 @@ import ( config "lalmax/conf" "net" "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/pion/ice/v2" @@ -140,6 +141,11 @@ func (s *RtcServer) HandleWHEP(c *gin.Context) { return } + userAgent := c.Request.UserAgent() + if strings.Contains(userAgent, "Safari") { + whepsession.SetRemoteSafari(true) + } + sdp := whepsession.GetAnswerSDP(string(body)) if sdp == "" { c.Status(http.StatusInternalServerError) diff --git a/rtc/whepsession.go b/rtc/whepsession.go index 3df548c..a850601 100644 --- a/rtc/whepsession.go +++ b/rtc/whepsession.go @@ -21,6 +21,7 @@ type whepSession struct { audiopacker *Packer msgChan chan base.RtmpMsg closeChan chan bool + remoteSafari bool } func NewWhepSession(streamid string, pc *peerConnection, lalServer logic.ILalServer) *whepSession { @@ -41,6 +42,10 @@ func NewWhepSession(streamid string, pc *peerConnection, lalServer logic.ILalSer } } +func (conn *whepSession) SetRemoteSafari(val bool) { + conn.remoteSafari = val +} + func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) { var err error @@ -59,7 +64,24 @@ func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) { return } - conn.videopacker = NewPacker(webrtc.MimeTypeH264, videoHeader.Payload) + conn.videopacker = NewPacker(PacketH264, videoHeader.Payload) + } else if videoHeader.IsHevcKeySeqHeader() { + if conn.remoteSafari { + // hevc暂时只支持对接Safari hevc + conn.videoTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH265}, "video", "lalmax") + if err != nil { + nazalog.Error(err) + return + } + + _, err = conn.pc.AddTrack(conn.videoTrack) + if err != nil { + nazalog.Error(err) + return + } + + conn.videopacker = NewPacker(PacketSafariHevc, videoHeader.Payload) + } } } @@ -75,7 +97,7 @@ func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) { return } - mimeType = webrtc.MimeTypePCMA + mimeType = PacketPCMA case base.RtmpSoundFormatG711U: conn.audioTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU}, "audio", "lalmax") if err != nil { @@ -83,7 +105,7 @@ func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) { return } - mimeType = webrtc.MimeTypePCMU + mimeType = PacketPCMU default: nazalog.Error("unsupport audio codeid:", audioId) }