From dffe63f1bc8730f81da580f6f9891fe26468ce08 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Sun, 6 Mar 2022 17:33:16 +0100 Subject: [PATCH] add SPS and PTS before IDRs of all incoming H264 streams; stop filtering H264 inside single protocols --- internal/core/rtmp_conn.go | 53 +++----------------------- internal/core/rtmp_source.go | 18 ++------- internal/core/stream.go | 29 ++++++++++++++ internal/hls/client_video_processor.go | 19 +-------- internal/hls/muxer_ts_generator.go | 6 +-- 5 files changed, 41 insertions(+), 84 deletions(-) diff --git a/internal/core/rtmp_conn.go b/internal/core/rtmp_conn.go index 46ab5e0e..83e8221f 100644 --- a/internal/core/rtmp_conn.go +++ b/internal/core/rtmp_conn.go @@ -342,36 +342,11 @@ func (c *rtmpConn) runRead(ctx context.Context) error { continue } - var filteredNALUs [][]byte - - for _, nalu := range pair.data.h264NALUs { - typ := h264.NALUType(nalu[0] & 0x1F) - - switch typ { - case h264.NALUTypeSPS, h264.NALUTypePPS: - // added automatically before every IDR - continue - - case h264.NALUTypeAccessUnitDelimiter: - // not needed - continue - - case h264.NALUTypeIDR: - // add SPS and PPS before every IDR - // TODO: send H264DecoderConfig instead of NALUs? - filteredNALUs = append(filteredNALUs, videoTrack.SPS(), videoTrack.PPS()) - } - - filteredNALUs = append(filteredNALUs, nalu) - } - - if filteredNALUs == nil { - continue - } + // TODO: send H264DecoderConfig instead of NALUs? // wait until we receive an IDR if !videoFirstIDRFound { - if !h264.IDRPresent(filteredNALUs) { + if !h264.IDRPresent(pair.data.h264NALUs) { continue } @@ -380,7 +355,7 @@ func (c *rtmpConn) runRead(ctx context.Context) error { videoDTSEst = h264.NewDTSEstimator() } - data, err := h264.EncodeAVCC(filteredNALUs) + data, err := h264.EncodeAVCC(pair.data.h264NALUs) if err != nil { return err } @@ -559,25 +534,9 @@ func (c *rtmpConn) runPublish(ctx context.Context) error { return err } - var outNALUs [][]byte - - for _, nalu := range nalus { - typ := h264.NALUType(nalu[0] & 0x1F) - if typ == h264.NALUTypeAccessUnitDelimiter { - // not needed - continue - } - - outNALUs = append(outNALUs, nalu) - } - - if len(outNALUs) == 0 { - continue - } - pts := pkt.Time + pkt.CTime - pkts, err := h264Encoder.Encode(outNALUs, pts) + pkts, err := h264Encoder.Encode(nalus, pts) if err != nil { return fmt.Errorf("error while encoding H264: %v", err) } @@ -592,8 +551,8 @@ func (c *rtmpConn) runPublish(ctx context.Context) error { } else { rres.stream.writeData(videoTrackID, &data{ rtp: pkt, - ptsEqualsDTS: h264.IDRPresent(outNALUs), - h264NALUs: outNALUs, + ptsEqualsDTS: h264.IDRPresent(nalus), + h264NALUs: nalus, h264PTS: pts, }) } diff --git a/internal/core/rtmp_source.go b/internal/core/rtmp_source.go index 6e3bccf5..80df3e4d 100644 --- a/internal/core/rtmp_source.go +++ b/internal/core/rtmp_source.go @@ -184,21 +184,9 @@ func (s *rtmpSource) runInner() bool { return err } - var outNALUs [][]byte - for _, nalu := range nalus { - // remove SPS, PPS and AUD, not needed by RTSP / RTMP - typ := h264.NALUType(nalu[0] & 0x1F) - switch typ { - case h264.NALUTypeSPS, h264.NALUTypePPS, h264.NALUTypeAccessUnitDelimiter: - continue - } - - outNALUs = append(outNALUs, nalu) - } - pts := pkt.Time + pkt.CTime - pkts, err := h264Encoder.Encode(outNALUs, pts) + pkts, err := h264Encoder.Encode(nalus, pts) if err != nil { return fmt.Errorf("error while encoding H264: %v", err) } @@ -213,8 +201,8 @@ func (s *rtmpSource) runInner() bool { } else { res.stream.writeData(videoTrackID, &data{ rtp: pkt, - ptsEqualsDTS: h264.IDRPresent(outNALUs), - h264NALUs: outNALUs, + ptsEqualsDTS: h264.IDRPresent(nalus), + h264NALUs: nalus, h264PTS: pts, }) } diff --git a/internal/core/stream.go b/internal/core/stream.go index 4308acd4..b87975ed 100644 --- a/internal/core/stream.go +++ b/internal/core/stream.go @@ -98,10 +98,39 @@ func (s *stream) updateH264TrackParameters(h264track *gortsplib.TrackH264, nalus } } +// remux is needed to +// - fix corrupted streams +// - make streams compatible with all protocols +func (s *stream) remuxH264NALUs(h264track *gortsplib.TrackH264, data *data) { + var filteredNALUs [][]byte //nolint:prealloc + + for _, nalu := range data.h264NALUs { + typ := h264.NALUType(nalu[0] & 0x1F) + switch typ { + case h264.NALUTypeSPS, h264.NALUTypePPS: + // remove since they're automatically added before every IDR + continue + + case h264.NALUTypeAccessUnitDelimiter: + // remove since it is not needed + continue + + case h264.NALUTypeIDR: + // add SPS and PPS before every IDR + filteredNALUs = append(filteredNALUs, h264track.SPS(), h264track.PPS()) + } + + filteredNALUs = append(filteredNALUs, nalu) + } + + data.h264NALUs = filteredNALUs +} + func (s *stream) writeData(trackID int, data *data) { track := s.rtspStream.Tracks()[trackID] if h264track, ok := track.(*gortsplib.TrackH264); ok { s.updateH264TrackParameters(h264track, data.h264NALUs) + s.remuxH264NALUs(h264track, data) } // forward to RTSP readers diff --git a/internal/hls/client_video_processor.go b/internal/hls/client_video_processor.go index 17a82460..c7fbfece 100644 --- a/internal/hls/client_video_processor.go +++ b/internal/hls/client_video_processor.go @@ -89,24 +89,7 @@ func (p *clientVideoProcessor) doProcess( return nil } - outNALUs := make([][]byte, 0, len(nalus)) - - for _, nalu := range nalus { - typ := h264.NALUType(nalu[0] & 0x1F) - - if typ == h264.NALUTypeAccessUnitDelimiter { - // remove since it's not needed - continue - } - - outNALUs = append(outNALUs, nalu) - } - - if len(outNALUs) == 0 { - return nil - } - - p.onData(pts, outNALUs) + p.onData(pts, nalus) return nil } diff --git a/internal/hls/muxer_ts_generator.go b/internal/hls/muxer_ts_generator.go index 46db3057..e1074284 100644 --- a/internal/hls/muxer_ts_generator.go +++ b/internal/hls/muxer_ts_generator.go @@ -115,11 +115,9 @@ func (m *muxerTSGenerator) writeH264(pts time.Duration, nalus [][]byte) error { dts := m.videoDTSEst.Feed(pts) // prepend an AUD. This is required by video.js and iOS - filteredNALUs := [][]byte{ - {byte(h264.NALUTypeAccessUnitDelimiter), 240}, - } + nalus = append([][]byte{{byte(h264.NALUTypeAccessUnitDelimiter), 240}}, nalus...) - enc, err := h264.EncodeAnnexB(filteredNALUs) + enc, err := h264.EncodeAnnexB(nalus) if err != nil { if m.currentSegment.buf.Len() > 0 { m.streamPlaylist.pushSegment(m.currentSegment)