lalmax support safari hevc(whep)

rtc/packer.go

@@ -2,15 +2,24 @@ package rtc
 
 import (
 	"fmt"
+	"math/rand"
 
 	"github.com/pion/rtp"
-	"github.com/pion/webrtc/v3"
 	"github.com/q191201771/lal/pkg/avc"
 	"github.com/q191201771/lal/pkg/base"
+	"github.com/q191201771/lal/pkg/hevc"
 	"github.com/q191201771/lal/pkg/rtprtcp"
 	"github.com/q191201771/naza/pkg/nazalog"
 )
 
+const (
+	PacketH264       = "H264"
+	PacketHEVC       = "HEVC"
+	PacketSafariHevc = "SafariHevc"
+	PacketPCMA       = "PCMA"
+	PacketPCMU       = "PCMU"
+)
+
 type Packer struct {
 	enc IRtpEncoder
 }
@@ -19,12 +28,14 @@ func NewPacker(mimeType string, codec []byte) *Packer {
 	p := &Packer{}
 
 	switch mimeType {
-	case webrtc.MimeTypeH264:
+	case PacketH264:
 		p.enc = NewH264RtpEncoder(codec)
-	case webrtc.MimeTypePCMA:
+	case PacketPCMA:
 		p.enc = NewG711RtpEncoder(8)
-	case webrtc.MimeTypePCMU:
+	case PacketPCMU:
 		p.enc = NewG711RtpEncoder(0)
+	case PacketSafariHevc:
+		p.enc = NewSafariHEVCRtpEncoder(codec)
 	}
 	return p
 }
@@ -108,7 +119,7 @@ func (enc *H264RtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) {
 		pkts = append(pkts, &newRtpPkt)
 	}
 
-	if len(rtpPkts) == 0 {
+	if len(pkts) == 0 {
 		return nil, fmt.Errorf("Packetize failed")
 	}
 
@@ -148,9 +159,139 @@ func (enc *G711RtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) {
 		pkts = append(pkts, &newRtpPkt)
 	}
 
-	if len(rtpPkts) == 0 {
+	if len(pkts) == 0 {
+		return nil, fmt.Errorf("Packetize failed")
+	}
+
+	return pkts, nil
+}
+
+type SafariHEVCRtpEncoder struct {
+	IRtpEncoder
+	vps         []byte
+	sps         []byte
+	pps         []byte
+	payloadType int
+	ssrc        int
+	seqId       uint16
+	tsBase      int64
+}
+
+func NewSafariHEVCRtpEncoder(codec []byte) *SafariHEVCRtpEncoder {
+	vps, sps, pps, err := hevc.ParseVpsSpsPpsFromSeqHeader(codec)
+	if err != nil {
+		nazalog.Error(err)
+		return nil
+	}
+
+	return &SafariHEVCRtpEncoder{
+		vps:         vps,
+		sps:         sps,
+		pps:         pps,
+		payloadType: 98,
+		ssrc:        0,
+		seqId:       uint16(rand.Int() % 65536),
+		tsBase:      -1,
+	}
+}
+
+func (enc *SafariHEVCRtpEncoder) Encode(msg base.RtmpMsg) ([]*rtp.Packet, error) {
+	var pkts []*rtp.Packet
+	var out []byte
+	var keyFrame bool
+
+	if enc.tsBase == -1 {
+		enc.tsBase = int64(msg.Dts())
+	}
+
+	err := avc.IterateNaluAvcc(msg.Payload[5:], func(nal []byte) {
+		t := hevc.ParseNaluType(nal[0])
+		if t == hevc.NaluTypeSei {
+			return
+		}
+
+		if hevc.IsIrapNalu(t) {
+			keyFrame = true
+			out = append(out, avc.NaluStartCode3...)
+			out = append(out, enc.vps...)
+			out = append(out, avc.NaluStartCode3...)
+			out = append(out, enc.sps...)
+			out = append(out, avc.NaluStartCode3...)
+			out = append(out, enc.pps...)
+		}
+
+		out = append(out, avc.NaluStartCode3...)
+		out = append(out, nal...)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("Packetize failed")
+	}
+
+	if len(out) == 0 {
+		return nil, fmt.Errorf("Packetize failed")
+	}
+
+	payloads := enc.doPacketNaluForSafariHevc(out, keyFrame)
+	for i, payload := range payloads {
+		var pkt rtp.Packet
+		pkt.Version = 2
+		pkt.Timestamp = uint32((int64(msg.Dts()) - enc.tsBase) * 90)
+		pkt.PayloadType = uint8(enc.payloadType)
+		pkt.SSRC = uint32(enc.ssrc)
+
+		if i == len(payloads)-1 {
+			pkt.Marker = true
+		}
+
+		pkt.SequenceNumber = enc.seqId
+		enc.seqId += 1
+		pkt.Payload = payload
+
+		pkts = append(pkts, &pkt)
+	}
+
+	if len(pkts) == 0 {
 		return nil, fmt.Errorf("Packetize failed")
 	}
 
 	return pkts, nil
 }
+
+func (enc *SafariHEVCRtpEncoder) doPacketNaluForSafariHevc(nalu []byte, keyFrame bool) [][]byte {
+	var rtpPayloads [][]byte
+
+	naluLen := len(nalu)
+	maxPayloadSize := 1200
+	splitNum := naluLen/maxPayloadSize + 1
+	remainder := naluLen % splitNum
+	referenceLen := naluLen / splitNum
+	dataPos := 0
+
+	for i := splitNum; i > 0; i-- {
+		tmpLen := referenceLen
+		if i < remainder {
+			tmpLen++
+		}
+		buf := make([]byte, tmpLen+1)
+		if keyFrame {
+			if i == splitNum {
+				buf[0] = 3
+			} else {
+				buf[0] = 1
+			}
+		} else {
+			if i == splitNum {
+				buf[0] = 2
+			} else {
+				buf[0] = 0
+			}
+		}
+		copy(buf[1:], nalu[dataPos:dataPos+tmpLen])
+		dataPos += tmpLen
+
+		rtpPayloads = append(rtpPayloads, buf)
+	}
+
+	return rtpPayloads
+}

rtc/server.go

@@ -4,6 +4,7 @@ import (
 	config "lalmax/conf"
 	"net"
 	"net/http"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 	"github.com/pion/ice/v2"
@@ -140,6 +141,11 @@ func (s *RtcServer) HandleWHEP(c *gin.Context) {
 		return
 	}
 
+	userAgent := c.Request.UserAgent()
+	if strings.Contains(userAgent, "Safari") {
+		whepsession.SetRemoteSafari(true)
+	}
+
 	sdp := whepsession.GetAnswerSDP(string(body))
 	if sdp == "" {
 		c.Status(http.StatusInternalServerError)

rtc/whepsession.go

@@ -21,6 +21,7 @@ type whepSession struct {
 	audiopacker  *Packer
 	msgChan      chan base.RtmpMsg
 	closeChan    chan bool
+	remoteSafari bool
 }
 
 func NewWhepSession(streamid string, pc *peerConnection, lalServer logic.ILalServer) *whepSession {
@@ -41,6 +42,10 @@ func NewWhepSession(streamid string, pc *peerConnection, lalServer logic.ILalSer
 	}
 }
 
+func (conn *whepSession) SetRemoteSafari(val bool) {
+	conn.remoteSafari = val
+}
+
 func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) {
 	var err error
 
@@ -59,7 +64,24 @@ func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) {
 				return
 			}
 
-			conn.videopacker = NewPacker(webrtc.MimeTypeH264, videoHeader.Payload)
+			conn.videopacker = NewPacker(PacketH264, videoHeader.Payload)
+		} else if videoHeader.IsHevcKeySeqHeader() {
+			if conn.remoteSafari {
+				// hevc暂时只支持对接Safari hevc
+				conn.videoTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH265}, "video", "lalmax")
+				if err != nil {
+					nazalog.Error(err)
+					return
+				}
+
+				_, err = conn.pc.AddTrack(conn.videoTrack)
+				if err != nil {
+					nazalog.Error(err)
+					return
+				}
+
+				conn.videopacker = NewPacker(PacketSafariHevc, videoHeader.Payload)
+			}
 		}
 	}
 
@@ -75,15 +97,15 @@ func (conn *whepSession) GetAnswerSDP(offer string) (sdp string) {
 				return
 			}
 
-			mimeType = webrtc.MimeTypePCMA
+			mimeType = PacketPCMA
 		case base.RtmpSoundFormatG711U:
 			conn.audioTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU}, "audio", "lalmax")
 			if err != nil {
 				nazalog.Error(err)
 				return
 			}
 
-			mimeType = webrtc.MimeTypePCMU
+			mimeType = PacketPCMU
 		default:
 			nazalog.Error("unsupport audio codeid:", audioId)
 		}