Skip to content

Commit

Permalink
try to fix DTS extraction of nvenc H264 streams (bluenviron/mediamtx#…
Browse files Browse the repository at this point in the history 
…1002)
  • Loading branch information
@aler9
aler9 committed Nov 1, 2022
1 parent dbb6934 commit 89aa62d
Show file tree
Hide file tree
Showing 2 changed files with 134 additions and 46 deletions.
66 changes: 32 additions & 34 deletions pkg/h264/dtsextractor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,13 +94,18 @@ func getPOCDiff(poc1 uint32, poc2 uint32, sps *SPS) int32 {
return diff
}

func getSEIPicTimingDPBOutputDelay(buf []byte, sps *SPS) (uint32, bool) {
type seiTimingInfo struct {
cpbRemovalDelay uint32
dpbOutputDelay uint32
}

func parseSEITimingInfo(buf []byte, sps *SPS) (*seiTimingInfo, bool) {
buf = AntiCompetitionRemove(buf)
pos := 1

for {
if pos >= (len(buf) - 1) {
return 0, false
return nil, false
}

payloadType := 0
Expand All @@ -127,48 +132,49 @@ func getSEIPicTimingDPBOutputDelay(buf []byte, sps *SPS) (uint32, bool) {
buf2 := buf[pos:]
pos2 := 0

// cpbRemovalDelay
_, err := bits.ReadBits(buf2, &pos2, int(sps.VUI.NalHRD.CpbRemovalDelayLengthMinus1+1))
ret := &seiTimingInfo{}

tmp, err := bits.ReadBits(buf2, &pos2, int(sps.VUI.NalHRD.CpbRemovalDelayLengthMinus1+1))
if err != nil {
return 0, false
return nil, false
}
ret.cpbRemovalDelay = uint32(tmp)

tmp, err := bits.ReadBits(buf2, &pos2, int(sps.VUI.NalHRD.DpbOutputDelayLengthMinus1+1))
tmp, err = bits.ReadBits(buf2, &pos2, int(sps.VUI.NalHRD.DpbOutputDelayLengthMinus1+1))
if err != nil {
return 0, false
return nil, false
}
dpbOutputDelay := uint32(tmp)
ret.dpbOutputDelay = uint32(tmp)

return dpbOutputDelay, true
return ret, true
}

pos += payloadSize
}
}

func findSEIPicTimingDPBOutputDelay(nalus [][]byte, sps *SPS) (uint32, bool) {
func findSEITimingInfo(nalus [][]byte, sps *SPS) (*seiTimingInfo, bool) {
for _, nalu := range nalus {
typ := NALUType(nalu[0] & 0x1F)
if typ == NALUTypeSEI {
ret, ok := getSEIPicTimingDPBOutputDelay(nalu, sps)
ret, ok := parseSEITimingInfo(nalu, sps)
if ok {
return ret, true
}
}
}
return 0, false
return nil, false
}

// DTSExtractor is a utility that allows to extract NALU DTS from PTS.
type DTSExtractor struct {
sps []byte
spsp *SPS
prevPTS time.Duration
prevDTS *time.Duration
prevPOCDiff int32
expectedPOC uint32
ptsDTSOffset time.Duration
firstDPBOutputDelay *uint32
sps []byte
spsp *SPS
prevPTS time.Duration
prevDTS *time.Duration
prevPOCDiff int32
expectedPOC uint32
ptsDTSOffset time.Duration
}

// NewDTSExtractor allocates a DTSExtractor.
Expand Down Expand Up @@ -255,31 +261,23 @@ func (d *DTSExtractor) extractInner(nalus [][]byte, pts time.Duration) (time.Dur

// DTS is computed from SEI
case d.spsp.VUI != nil && d.spsp.VUI.TimingInfo != nil && d.spsp.VUI.NalHRD != nil:
dpbOutputDelay, ok := findSEIPicTimingDPBOutputDelay(nalus, d.spsp)
ti, ok := findSEITimingInfo(nalus, d.spsp)
if !ok {
// some streams declare that they use SEI pic timings, but they don't.
// assume PTS = DTS.
return pts, 0, nil
}

// workaround for nvenc
// nvenc puts a wrong dpbOutputDelay in IDR frames that follows the first one.
// save the dpbOutputDelay of the first IDR frame and use if for subsequent
// IDR frames.
// nvenc puts a wrong dpbOutputDelay into timing infos of non-starting IDR frames
// https://forums.developer.nvidia.com/t/nvcodec-h-264-encoder-sei-pic-timing-dpb-output-delay/156050
// https://forums.developer.nvidia.com/t/h264-pic-timing-sei-message/71188
if idrPresent &&
d.spsp.VUI.NalHRD.CpbRemovalDelayLengthMinus1 == 15 &&
d.spsp.VUI.NalHRD.DpbOutputDelayLengthMinus1 == 5 {
if d.firstDPBOutputDelay == nil {
d.firstDPBOutputDelay = &dpbOutputDelay
} else {
dpbOutputDelay = *d.firstDPBOutputDelay
}
if idrPresent && ti.cpbRemovalDelay > 0 {
ti.dpbOutputDelay = 2
}

return pts - time.Duration(dpbOutputDelay)/2*time.Second*
time.Duration(d.spsp.VUI.TimingInfo.NumUnitsInTick)*2/time.Duration(d.spsp.VUI.TimingInfo.TimeScale), 0, nil
return pts - time.Duration(ti.dpbOutputDelay)*time.Second*
time.Duration(d.spsp.VUI.TimingInfo.NumUnitsInTick)/time.Duration(d.spsp.VUI.TimingInfo.TimeScale), 0, nil

// we assume PTS = DTS
default:
Expand Down
114 changes: 102 additions & 12 deletions pkg/h264/dtsextractor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,9 @@ func TestDTSExtractor(t *testing.T) {
[]sequenceSample{
{
[][]byte{
// SEI (buffering period)
{6, 0, 7, 128, 117, 48, 0, 0, 3, 0, 64, 128},
// SEI (pic timing)
{6, 1, 4, 0, 0, 8, 16, 128},
// SPS
{
{6, 0, 7, 128, 117, 48, 0, 0, 3, 0, 64, 128}, // SEI (buffering period)
{6, 1, 4, 0, 0, 8, 16, 128}, // SEI (pic timing)
{ // SPS
103, 100, 0, 42, 172, 44, 172, 7,
128, 34, 126, 92, 5, 168, 8, 8,
10, 0, 0, 7, 208, 0, 3, 169,
Expand All @@ -165,30 +162,123 @@ func TestDTSExtractor(t *testing.T) {
},
{
[][]byte{
// SEI
{6, 1, 4, 0, 2, 32, 16, 128},
{6, 1, 4, 0, 2, 32, 16, 128}, // SEI
},
66666666 * time.Nanosecond,
0,
},
{
[][]byte{
// SEI
{6, 1, 4, 0, 4, 0, 16, 128},
{6, 1, 4, 0, 4, 0, 16, 128}, // SEI
},
16666666 * time.Nanosecond,
16666666 * time.Nanosecond,
},
{
[][]byte{
// SEI
{6, 1, 4, 0, 6, 0, 16, 128},
{6, 1, 4, 0, 6, 0, 16, 128}, // SEI
},
33333333 * time.Nanosecond,
33333333 * time.Nanosecond,
},
},
},
{
"sei-based nvenc",
[]sequenceSample{
{
[][]byte{
{ // SPS
103, 100, 0, 42, 172, 44, 172, 7,
128, 34, 126, 92, 5, 168, 8, 8,
10, 0, 0, 7, 208, 0, 3, 169,
129, 192, 0, 0, 76, 75, 0, 0,
38, 37, 173, 222, 92, 20,
},
{6, 0, 7, 128, 175, 199, 128, 0, 0, 192, 128}, // SEI
{6, 1, 4, 0, 120, 40, 16, 128}, // SEI
{5}, // IDR
},
999 * time.Millisecond,
982333334 * time.Nanosecond,
},
{[][]byte{{6, 1, 4, 0, 2, 40, 16, 128}}, 1083 * time.Millisecond, 999666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 4, 0, 16, 128}}, 1016 * time.Millisecond, 1016 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 6, 0, 16, 128}}, 1033 * time.Millisecond, 1033 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 8, 0, 16, 128}}, 1050 * time.Millisecond, 1050 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 10, 0, 16, 128}}, 1066 * time.Millisecond, 1066 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 12, 40, 16, 128}}, 1166 * time.Millisecond, 1082666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 14, 0, 16, 128}}, 1100 * time.Millisecond, 1100 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 16, 0, 16, 128}}, 1116 * time.Millisecond, 1116 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 18, 0, 16, 128}}, 1133 * time.Millisecond, 1133 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 20, 0, 16, 128}}, 1150 * time.Millisecond, 1150 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 22, 40, 16, 128}}, 1249 * time.Millisecond, 1165666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 24, 0, 16, 128}}, 1183 * time.Millisecond, 1183 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 26, 0, 16, 128}}, 1200 * time.Millisecond, 1200 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 28, 0, 16, 128}}, 1216 * time.Millisecond, 1216 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 30, 0, 16, 128}}, 1233 * time.Millisecond, 1233 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 32, 40, 16, 128}}, 1333 * time.Millisecond, 1249666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 34, 0, 16, 128}}, 1266 * time.Millisecond, 1266 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 36, 0, 16, 128}}, 1283 * time.Millisecond, 1283 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 38, 0, 16, 128}}, 1300 * time.Millisecond, 1300 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 40, 0, 16, 128}}, 1316 * time.Millisecond, 1316 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 42, 40, 16, 128}}, 1416 * time.Millisecond, 1332666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 44, 0, 16, 128}}, 1350 * time.Millisecond, 1350 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 46, 0, 16, 128}}, 1366 * time.Millisecond, 1366 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 48, 0, 16, 128}}, 1383 * time.Millisecond, 1383 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 50, 0, 16, 128}}, 1400 * time.Millisecond, 1400 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 52, 40, 16, 128}}, 1499 * time.Millisecond, 1415666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 54, 0, 16, 128}}, 1433 * time.Millisecond, 1433 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 56, 0, 16, 128}}, 1450 * time.Millisecond, 1450 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 58, 0, 16, 128}}, 1466 * time.Millisecond, 1466 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 60, 0, 16, 128}}, 1483 * time.Millisecond, 1483 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 62, 40, 16, 128}}, 1583 * time.Millisecond, 1499666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 64, 0, 16, 128}}, 1516 * time.Millisecond, 1516 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 66, 0, 16, 128}}, 1533 * time.Millisecond, 1533 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 68, 0, 16, 128}}, 1550 * time.Millisecond, 1550 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 70, 0, 16, 128}}, 1566 * time.Millisecond, 1566 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 72, 40, 16, 128}}, 1666 * time.Millisecond, 1582666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 74, 0, 16, 128}}, 1600 * time.Millisecond, 1600 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 76, 0, 16, 128}}, 1616 * time.Millisecond, 1616 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 78, 0, 16, 128}}, 1633 * time.Millisecond, 1633 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 80, 0, 16, 128}}, 1650 * time.Millisecond, 1650 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 82, 40, 16, 128}}, 1749 * time.Millisecond, 1665666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 84, 0, 16, 128}}, 1683 * time.Millisecond, 1683 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 86, 0, 16, 128}}, 1700 * time.Millisecond, 1700 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 88, 0, 16, 128}}, 1716 * time.Millisecond, 1716 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 90, 0, 16, 128}}, 1733 * time.Millisecond, 1733 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 92, 40, 16, 128}}, 1833 * time.Millisecond, 1749666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 94, 0, 16, 128}}, 1766 * time.Millisecond, 1766 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 96, 0, 16, 128}}, 1783 * time.Millisecond, 1783 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 98, 0, 16, 128}}, 1800 * time.Millisecond, 1800 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 100, 0, 16, 128}}, 1816 * time.Millisecond, 1816 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 102, 40, 16, 128}}, 1916 * time.Millisecond, 1832666667 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 104, 0, 16, 128}}, 1850 * time.Millisecond, 1850 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 106, 0, 16, 128}}, 1866 * time.Millisecond, 1866 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 108, 0, 16, 128}}, 1883 * time.Millisecond, 1883 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 110, 0, 16, 128}}, 1900 * time.Millisecond, 1900 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 112, 32, 16, 128}}, 1982 * time.Millisecond, 1915333334 * time.Nanosecond},
{[][]byte{{6, 1, 4, 0, 114, 0, 16, 128}}, 1933 * time.Millisecond, 1933 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 116, 0, 16, 128}}, 1950 * time.Millisecond, 1950 * time.Millisecond},
{[][]byte{{6, 1, 4, 0, 118, 0, 16, 128}}, 1966 * time.Millisecond, 1966 * time.Millisecond},
{
[][]byte{
{ // SPS
103, 100, 0, 42, 172, 44, 172, 7,
128, 34, 126, 92, 5, 168, 8, 8,
10, 0, 0, 7, 208, 0, 3, 169,
129, 192, 0, 0, 76, 75, 0, 0,
38, 37, 173, 222, 92, 20,
},
{6, 0, 7, 128, 175, 199, 128, 0, 0, 192, 128}, // SEI
{6, 1, 4, 0, 120, 40, 16, 128}, // SEI
{5}, // IDR
},
1999 * time.Millisecond,
1982333334 * time.Nanosecond,
},
},
},
} {
t.Run(ca.name, func(t *testing.T) {
ex := NewDTSExtractor()
Expand Down

0 comments on commit 89aa62d

Please sign in to comment.