From 66966de8bb66b101a3b1474d1a006f21959e9616 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Sun, 6 Aug 2023 18:11:47 +0200 Subject: [PATCH] disable check for missing key frames (#1904) --- internal/formatprocessor/av1.go | 28 ++-------------- internal/formatprocessor/av1_test.go | 43 ------------------------ internal/formatprocessor/h264.go | 37 ++++++--------------- internal/formatprocessor/h264_test.go | 48 --------------------------- internal/formatprocessor/h265.go | 37 ++++++--------------- internal/formatprocessor/h265_test.go | 36 -------------------- internal/formatprocessor/processor.go | 4 --- 7 files changed, 22 insertions(+), 211 deletions(-) diff --git a/internal/formatprocessor/av1.go b/internal/formatprocessor/av1.go index f4a4d9cdcc1..0a9ba5c41da 100644 --- a/internal/formatprocessor/av1.go +++ b/internal/formatprocessor/av1.go @@ -6,7 +6,6 @@ import ( "github.com/bluenviron/gortsplib/v3/pkg/formats" "github.com/bluenviron/gortsplib/v3/pkg/formats/rtpav1" - "github.com/bluenviron/mediacommon/pkg/codecs/av1" "github.com/pion/rtp" "github.com/bluenviron/mediamtx/internal/logger" @@ -24,10 +23,8 @@ type formatProcessorAV1 struct { format *formats.AV1 log logger.Writer - encoder *rtpav1.Encoder - decoder *rtpav1.Decoder - lastKeyFrameTimeReceived bool - lastKeyFrameTime time.Time + encoder *rtpav1.Encoder + decoder *rtpav1.Decoder } func newAV1( @@ -59,24 +56,6 @@ func (t *formatProcessorAV1) createEncoder() error { return t.encoder.Init() } -func (t *formatProcessorAV1) checkKeyFrameInterval(ntp time.Time, isKeyFrame bool) { - if !t.lastKeyFrameTimeReceived || isKeyFrame { - t.lastKeyFrameTimeReceived = true - t.lastKeyFrameTime = ntp - return - } - - if ntp.Sub(t.lastKeyFrameTime) >= maxKeyFrameInterval { - t.lastKeyFrameTime = ntp - t.log.Log(logger.Warn, "no AV1 key frames received in %v, stream can't be decoded", maxKeyFrameInterval) - } -} - -func (t *formatProcessorAV1) checkOBUs(ntp time.Time, obus [][]byte) { - containsKeyFrame, _ := av1.ContainsKeyFrame(obus) - t.checkKeyFrameInterval(ntp, containsKeyFrame) -} - func (t *formatProcessorAV1) Process(unit Unit, hasNonRTSPReaders bool) error { //nolint:dupl tunit := unit.(*UnitAV1) @@ -112,7 +91,6 @@ func (t *formatProcessorAV1) Process(unit Unit, hasNonRTSPReaders bool) error { } tunit.OBUs = obus - t.checkOBUs(tunit.NTP, obus) tunit.PTS = pts } @@ -120,8 +98,6 @@ func (t *formatProcessorAV1) Process(unit Unit, hasNonRTSPReaders bool) error { return nil } - t.checkOBUs(tunit.NTP, tunit.OBUs) - // encode into RTP pkts, err := t.encoder.Encode(tunit.OBUs, tunit.PTS) if err != nil { diff --git a/internal/formatprocessor/av1_test.go b/internal/formatprocessor/av1_test.go index f7e0a4ab8a7..136e2bf8332 100644 --- a/internal/formatprocessor/av1_test.go +++ b/internal/formatprocessor/av1_test.go @@ -1,44 +1 @@ package formatprocessor - -import ( - "testing" - "time" - - "github.com/bluenviron/gortsplib/v3/pkg/formats" - "github.com/stretchr/testify/require" -) - -func TestAV1KeyFrameWarning(t *testing.T) { //nolint:dupl - forma := &formats.AV1{ - PayloadTyp: 96, - } - - w := &testLogWriter{recv: make(chan string, 1)} - p, err := New(1472, forma, true, w) - require.NoError(t, err) - - ntp := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - err = p.Process(&UnitAV1{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - OBUs: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - ntp = ntp.Add(30 * time.Second) - err = p.Process(&UnitAV1{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - OBUs: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - logl := <-w.recv - require.Equal(t, "no AV1 key frames received in 10s, stream can't be decoded", logl) -} diff --git a/internal/formatprocessor/h264.go b/internal/formatprocessor/h264.go index 09e7e025636..b6ee5bed311 100644 --- a/internal/formatprocessor/h264.go +++ b/internal/formatprocessor/h264.go @@ -81,10 +81,8 @@ type formatProcessorH264 struct { format *formats.H264 log logger.Writer - encoder *rtph264.Encoder - decoder *rtph264.Decoder - lastKeyFrameTimeReceived bool - lastKeyFrameTime time.Time + encoder *rtph264.Encoder + decoder *rtph264.Decoder } func newH264( @@ -146,12 +144,12 @@ func (t *formatProcessorH264) updateTrackParametersFromRTPPacket(pkt *rtp.Packet } } -func (t *formatProcessorH264) updateTrackParametersFromNALUs(nalus [][]byte) { +func (t *formatProcessorH264) updateTrackParametersFromAU(au [][]byte) { sps := t.format.SPS pps := t.format.PPS update := false - for _, nalu := range nalus { + for _, nalu := range au { typ := h264.NALUType(nalu[0] & 0x1F) switch typ { @@ -174,24 +172,11 @@ func (t *formatProcessorH264) updateTrackParametersFromNALUs(nalus [][]byte) { } } -func (t *formatProcessorH264) checkKeyFrameInterval(ntp time.Time, isKeyFrame bool) { - if !t.lastKeyFrameTimeReceived || isKeyFrame { - t.lastKeyFrameTimeReceived = true - t.lastKeyFrameTime = ntp - return - } - - if ntp.Sub(t.lastKeyFrameTime) >= maxKeyFrameInterval { - t.lastKeyFrameTime = ntp - t.log.Log(logger.Warn, "no H264 key frames received in %v, stream can't be decoded", maxKeyFrameInterval) - } -} - -func (t *formatProcessorH264) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][]byte { +func (t *formatProcessorH264) remuxAccessUnit(au [][]byte) [][]byte { isKeyFrame := false n := 0 - for _, nalu := range nalus { + for _, nalu := range au { typ := h264.NALUType(nalu[0] & 0x1F) switch typ { @@ -214,8 +199,6 @@ func (t *formatProcessorH264) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][ n++ } - t.checkKeyFrameInterval(ntp, isKeyFrame) - if n == 0 { return nil } @@ -229,7 +212,7 @@ func (t *formatProcessorH264) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][ i = 2 } - for _, nalu := range nalus { + for _, nalu := range au { typ := h264.NALUType(nalu[0] & 0x1F) switch typ { @@ -294,7 +277,7 @@ func (t *formatProcessorH264) Process(unit Unit, hasNonRTSPReaders bool) error { return err } - tunit.AU = t.remuxAccessUnit(tunit.NTP, au) + tunit.AU = t.remuxAccessUnit(au) tunit.PTS = pts } @@ -303,8 +286,8 @@ func (t *formatProcessorH264) Process(unit Unit, hasNonRTSPReaders bool) error { return nil } } else { - t.updateTrackParametersFromNALUs(tunit.AU) - tunit.AU = t.remuxAccessUnit(tunit.NTP, tunit.AU) + t.updateTrackParametersFromAU(tunit.AU) + tunit.AU = t.remuxAccessUnit(tunit.AU) } // encode into RTP diff --git a/internal/formatprocessor/h264_test.go b/internal/formatprocessor/h264_test.go index de40dafbcb5..4c2e3e32b0e 100644 --- a/internal/formatprocessor/h264_test.go +++ b/internal/formatprocessor/h264_test.go @@ -2,26 +2,14 @@ package formatprocessor import ( "bytes" - "fmt" "testing" - "time" "github.com/bluenviron/gortsplib/v3/pkg/formats" "github.com/bluenviron/mediacommon/pkg/codecs/h264" "github.com/pion/rtp" "github.com/stretchr/testify/require" - - "github.com/bluenviron/mediamtx/internal/logger" ) -type testLogWriter struct { - recv chan string -} - -func (w *testLogWriter) Log(_ logger.Level, format string, args ...interface{}) { - w.recv <- fmt.Sprintf(format, args...) -} - func TestH264DynamicParams(t *testing.T) { forma := &formats.H264{ PayloadTyp: 96, @@ -207,39 +195,3 @@ func TestH264EmptyPacket(t *testing.T) { // if all NALUs have been removed, no RTP packets must be generated. require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets) } - -func TestH264KeyFrameWarning(t *testing.T) { - forma := &formats.H264{ - PayloadTyp: 96, - PacketizationMode: 1, - } - - w := &testLogWriter{recv: make(chan string, 1)} - p, err := New(1472, forma, true, w) - require.NoError(t, err) - - ntp := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - err = p.Process(&UnitH264{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - AU: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - ntp = ntp.Add(30 * time.Second) - err = p.Process(&UnitH264{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - AU: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - logl := <-w.recv - require.Equal(t, "no H264 key frames received in 10s, stream can't be decoded", logl) -} diff --git a/internal/formatprocessor/h265.go b/internal/formatprocessor/h265.go index a49ea24ad81..4027ca85034 100644 --- a/internal/formatprocessor/h265.go +++ b/internal/formatprocessor/h265.go @@ -88,10 +88,8 @@ type formatProcessorH265 struct { format *formats.H265 log logger.Writer - encoder *rtph265.Encoder - decoder *rtph265.Decoder - lastKeyFrameTimeReceived bool - lastKeyFrameTime time.Time + encoder *rtph265.Encoder + decoder *rtph265.Decoder } func newH265( @@ -160,13 +158,13 @@ func (t *formatProcessorH265) updateTrackParametersFromRTPPacket(pkt *rtp.Packet } } -func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) { +func (t *formatProcessorH265) updateTrackParametersFromAU(au [][]byte) { vps := t.format.VPS sps := t.format.SPS pps := t.format.PPS update := false - for _, nalu := range nalus { + for _, nalu := range au { typ := h265.NALUType((nalu[0] >> 1) & 0b111111) switch typ { @@ -195,24 +193,11 @@ func (t *formatProcessorH265) updateTrackParametersFromNALUs(nalus [][]byte) { } } -func (t *formatProcessorH265) checkKeyFrameInterval(ntp time.Time, isKeyFrame bool) { - if !t.lastKeyFrameTimeReceived || isKeyFrame { - t.lastKeyFrameTimeReceived = true - t.lastKeyFrameTime = ntp - return - } - - if ntp.Sub(t.lastKeyFrameTime) >= maxKeyFrameInterval { - t.lastKeyFrameTime = ntp - t.log.Log(logger.Warn, "no H265 key frames received in %v, stream can't be decoded", maxKeyFrameInterval) - } -} - -func (t *formatProcessorH265) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][]byte { +func (t *formatProcessorH265) remuxAccessUnit(au [][]byte) [][]byte { isKeyFrame := false n := 0 - for _, nalu := range nalus { + for _, nalu := range au { typ := h265.NALUType((nalu[0] >> 1) & 0b111111) switch typ { @@ -235,8 +220,6 @@ func (t *formatProcessorH265) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][ n++ } - t.checkKeyFrameInterval(ntp, isKeyFrame) - if n == 0 { return nil } @@ -251,7 +234,7 @@ func (t *formatProcessorH265) remuxAccessUnit(ntp time.Time, nalus [][]byte) [][ i = 3 } - for _, nalu := range nalus { + for _, nalu := range au { typ := h265.NALUType((nalu[0] >> 1) & 0b111111) switch typ { @@ -316,7 +299,7 @@ func (t *formatProcessorH265) Process(unit Unit, hasNonRTSPReaders bool) error { return err } - tunit.AU = t.remuxAccessUnit(tunit.NTP, au) + tunit.AU = t.remuxAccessUnit(au) tunit.PTS = pts } @@ -325,8 +308,8 @@ func (t *formatProcessorH265) Process(unit Unit, hasNonRTSPReaders bool) error { return nil } } else { - t.updateTrackParametersFromNALUs(tunit.AU) - tunit.AU = t.remuxAccessUnit(tunit.NTP, tunit.AU) + t.updateTrackParametersFromAU(tunit.AU) + tunit.AU = t.remuxAccessUnit(tunit.AU) } // encode into RTP diff --git a/internal/formatprocessor/h265_test.go b/internal/formatprocessor/h265_test.go index 6c276975a2c..83c27decb31 100644 --- a/internal/formatprocessor/h265_test.go +++ b/internal/formatprocessor/h265_test.go @@ -3,7 +3,6 @@ package formatprocessor import ( "bytes" "testing" - "time" "github.com/bluenviron/gortsplib/v3/pkg/formats" "github.com/bluenviron/mediacommon/pkg/codecs/h265" @@ -193,38 +192,3 @@ func TestH265EmptyPacket(t *testing.T) { // if all NALUs have been removed, no RTP packets must be generated. require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets) } - -func TestH265KeyFrameWarning(t *testing.T) { //nolint:dupl - forma := &formats.H265{ - PayloadTyp: 96, - } - - w := &testLogWriter{recv: make(chan string, 1)} - p, err := New(1472, forma, true, w) - require.NoError(t, err) - - ntp := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - err = p.Process(&UnitH265{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - AU: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - ntp = ntp.Add(30 * time.Second) - err = p.Process(&UnitH265{ - BaseUnit: BaseUnit{ - NTP: ntp, - }, - AU: [][]byte{ - {0x01}, - }, - }, false) - require.NoError(t, err) - - logl := <-w.recv - require.Equal(t, "no H265 key frames received in 10s, stream can't be decoded", logl) -} diff --git a/internal/formatprocessor/processor.go b/internal/formatprocessor/processor.go index d83c0587b6d..753b76913d0 100644 --- a/internal/formatprocessor/processor.go +++ b/internal/formatprocessor/processor.go @@ -10,10 +10,6 @@ import ( "github.com/bluenviron/mediamtx/internal/logger" ) -const ( - maxKeyFrameInterval = 10 * time.Second -) - // Processor cleans and normalizes streams. type Processor interface { // cleans and normalizes a data unit.