Skip to content

Commit

Permalink
fix DTS extraction of QSV HEVC streams (bluenviron/mediamtx#3285) (#128)
Browse files Browse the repository at this point in the history 
* replace DeltaPocS0Minus1 with DeltaPocS0

* make DeltaPocS0 a int32

* fix computation of DeltaPocS0 when InterRefPicSetPredictionFlag is false

* fix SPS parsing of QSV HEVC

* fix DTS extraction of QSV HEVC streams

* add additional fuzzing
  • Loading branch information
@aler9
aler9 authored May 18, 2024
1 parent b4e31e8 commit 1e6e29b
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 21 deletions.
17 changes: 9 additions & 8 deletions pkg/codecs/h265/dts_extractor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ func getPTSDTSDiff(buf []byte, sps *SPS, pps *PPS) (uint32, error) {

if !shortTermRefPicSetSpsFlag {
rps = &SPS_ShortTermRefPicSet{}
err = rps.unmarshal(buf, &pos, uint32(len(sps.ShortTermRefPicSets)), uint32(len(sps.ShortTermRefPicSets)), nil)
err = rps.unmarshal(buf, &pos, uint32(len(sps.ShortTermRefPicSets)),
uint32(len(sps.ShortTermRefPicSets)), sps.ShortTermRefPicSets)
if err != nil {
return 0, err
}
Expand All @@ -115,18 +116,18 @@ func getPTSDTSDiff(buf []byte, sps *SPS, pps *PPS) (uint32, error) {

if sliceType == 0 { // B-frame
if typ == NALUType_TRAIL_N || typ == NALUType_RASL_N {
v = sps.MaxNumReorderPics[0] - uint32(len(rps.DeltaPocS1Minus1))
v = sps.MaxNumReorderPics[0] - uint32(len(rps.DeltaPocS1))
} else if typ == NALUType_TRAIL_R || typ == NALUType_RASL_R {
if len(rps.DeltaPocS0Minus1) == 0 {
return 0, fmt.Errorf("invalid delta_poc_s0_minus1")
if len(rps.DeltaPocS0) == 0 {
return 0, fmt.Errorf("invalid DeltaPocS0")
}
v = rps.DeltaPocS0Minus1[0] + sps.MaxNumReorderPics[0] - 1
v = uint32(-rps.DeltaPocS0[0]-1+int32(sps.MaxNumReorderPics[0])) - uint32(len(rps.DeltaPocS1))
}
} else { // I or P-frame
if len(rps.DeltaPocS0Minus1) == 0 {
return 0, fmt.Errorf("invalid delta_poc_s0_minus1")
if len(rps.DeltaPocS0) == 0 {
return 0, fmt.Errorf("invalid DeltaPocS0")
}
v = rps.DeltaPocS0Minus1[0] + sps.MaxNumReorderPics[0]
v = uint32(-rps.DeltaPocS0[0] - 1 + int32(sps.MaxNumReorderPics[0]))
}

return v, nil
Expand Down
99 changes: 99 additions & 0 deletions pkg/codecs/h265/dts_extractor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,105 @@ func TestDTSExtractor(t *testing.T) {
},
},
},
{
"qsv hevc b_frames=3",
[]sequenceSample{
{
[][]byte{
{ // VPS
0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40,
0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03,
0x00, 0x00, 0x03, 0x00, 0x7b, 0x11, 0xc0, 0xc0,
0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x0f,
0x14,
},
{ // SPS
0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03,
0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03,
0x00, 0x7b, 0xa0, 0x03, 0xc0, 0x80, 0x11, 0x07,
0xcb, 0xb1, 0x1e, 0xe4, 0x6c, 0x0a, 0x9f, 0xa6,
0xb9, 0x97, 0x92, 0xcf, 0x60, 0x2d, 0x40, 0x40,
0x40, 0x45, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00,
0x00, 0x03, 0x00, 0x3c, 0x60, 0x35, 0xef, 0x7e,
0x00, 0x02, 0x62, 0x58, 0x00, 0x26, 0x17, 0x20,
},
{ // PPS
0x44, 0x01, 0xc0, 0x3c, 0xf0, 0x1b, 0x64,
},
{
0x4e, 0x01, 0x00, 0x0a, 0x80, 0x00, 0x00, 0x03,
0x00, 0x55, 0xda, 0x80, 0x01, 0xe5, 0xd0, 0x01,
0x07, 0x04, 0x00, 0x00, 0xee, 0x00, 0x00, 0x05,
0x80,
},
{
0x26, 0x01, 0xae, 0x80, 0x8f, 0x4c, 0xdd, 0xfc,
0xee, 0x2f, 0x79, 0x7c, 0x9e, 0x21, 0x6b, 0x2a,
0xe7, 0x6a, 0x57, 0x56, 0x46, 0x6f, 0x32, 0x5a,
0x7c, 0xbc, 0x47, 0xe8, 0xce, 0x5c, 0x5e, 0xfa,
0x1e, 0xd0, 0x94, 0x08, 0x4c, 0x98, 0x9d, 0xbb,
0x5d, 0x4c, 0x54, 0xa1, 0xd9, 0x5b, 0x1b, 0xba,
},
},
2033333333 * time.Nanosecond,
2000000000 * time.Nanosecond,
},
{
[][]byte{
{
0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x03,
0x00, 0x00, 0x03, 0x00, 0x0b, 0x80,
},
{
0x02, 0x01, 0xe2, 0x0a, 0x4f, 0xdd, 0x1e, 0xb7,
0xb7, 0xa1, 0x80, 0xad, 0xc7, 0x3c, 0x2e, 0x33,
0x3b, 0xde, 0xcc, 0x77, 0x13, 0x9c, 0x5b, 0xe3,
0x2c, 0xaa, 0xd4, 0x2e, 0xb0, 0x2b, 0x9e, 0x20,
0xdd, 0xc9, 0x1b, 0x39, 0xd9, 0x75, 0x06, 0xf5,
0xa8, 0x1f, 0x66, 0x62, 0x5b, 0xfe, 0x1f, 0xf9,
},
},
2099999999 * time.Nanosecond,
2016666666 * time.Nanosecond,
},
{
[][]byte{
{
0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x03,
0x02, 0x00, 0x00, 0x05, 0x80,
},
{
0x02, 0x01, 0xe1, 0x32, 0x27, 0xe3, 0xa0, 0x51,
0xcd, 0xff, 0x1a, 0x0b, 0x37, 0xaf, 0xe3, 0xe6,
0x9e, 0xaa, 0x27, 0x82, 0xcd, 0x28, 0xa3, 0xce,
0x57, 0x8b, 0x02, 0x3e, 0x62, 0x1f, 0x66, 0x5b,
0xbd, 0x67, 0x6b, 0xb1, 0x47, 0x9d, 0x1b, 0x26,
0xb7, 0x2a, 0x04, 0xac, 0x2e, 0x94, 0x1e, 0x22,
},
},
2066666666 * time.Nanosecond,
2033333333 * time.Nanosecond,
},
{
[][]byte{
{
0x4e, 0x01, 0x01, 0x07, 0x04, 0x00, 0x00, 0x04,
0x00, 0x00, 0x03, 0x01, 0x80,
},
{
0x00, 0x01, 0xe0, 0xcf, 0x8e, 0x80, 0x1e, 0x96,
0xa6, 0x88, 0xcb, 0x98, 0xf3, 0xd9, 0x2a, 0x4b,
0xa7, 0xa0, 0xf8, 0xa0, 0x4d, 0x21, 0x89, 0x76,
0x54, 0xe3, 0x8f, 0x46, 0xf6, 0x93, 0xde, 0x84,
0x33, 0x26, 0x3e, 0xe8, 0x20, 0x23, 0xef, 0x39,
0x03, 0x3c, 0x92, 0x11, 0x50, 0x98, 0xd6, 0x13,
},
},
2049999999 * time.Nanosecond,
2049999999 * time.Nanosecond,
},
},
},
} {
t.Run(ca.name, func(t *testing.T) {
ex := NewDTSExtractor()
Expand Down
102 changes: 91 additions & 11 deletions pkg/codecs/h265/sps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,9 +405,9 @@ type SPS_ShortTermRefPicSet struct { //nolint:revive
AbsDeltaRpsMinus1 uint32
NumNegativePics uint32
NumPositivePics uint32
DeltaPocS0Minus1 []uint32
DeltaPocS0 []int32
UsedByCurrPicS0Flag []bool
DeltaPocS1Minus1 []uint32
DeltaPocS1 []int32
UsedByCurrPicS1Flag []bool
}

Expand Down Expand Up @@ -441,23 +441,91 @@ func (r *SPS_ShortTermRefPicSet) unmarshal(buf []byte, pos *int, stRpsIdx uint32
return err
}

var s int32
if r.DeltaRpsSign {
s = 1
}
deltaRps := (1 - 2*s) * (int32(r.AbsDeltaRpsMinus1) + 1)

refRpsIdx := stRpsIdx - (r.DeltaIdxMinus1 + 1)
numDeltaPocs := shortTermRefPicSets[refRpsIdx].NumNegativePics + shortTermRefPicSets[refRpsIdx].NumPositivePics
refRPS := shortTermRefPicSets[refRpsIdx]
numDeltaPocs := refRPS.NumNegativePics + refRPS.NumPositivePics
usedByCurrPicFlag := make([]bool, numDeltaPocs+1)

useDeltaFlag := make([]bool, numDeltaPocs+1)
for i := range useDeltaFlag {
useDeltaFlag[i] = true
}

for j := uint32(0); j <= numDeltaPocs; j++ {
var usedByCurrPicFlag bool
usedByCurrPicFlag, err = bits.ReadFlag(buf, pos)
usedByCurrPicFlag[j], err = bits.ReadFlag(buf, pos)
if err != nil {
return err
}

if usedByCurrPicFlag {
_, err = bits.ReadGolombUnsigned(buf, pos) // use_delta_flag
if !usedByCurrPicFlag[j] {
useDeltaFlag[j], err = bits.ReadFlag(buf, pos)
if err != nil {
return err
}
}
}

i := uint32(0)

for j := (int32(refRPS.NumPositivePics) - 1); j >= 0; j-- {
dPoc := refRPS.DeltaPocS1[j] + deltaRps
if dPoc < 0 && useDeltaFlag[refRPS.NumNegativePics+uint32(j)] {
r.DeltaPocS0 = append(r.DeltaPocS0, dPoc)
r.UsedByCurrPicS0Flag = append(r.UsedByCurrPicS0Flag, usedByCurrPicFlag[refRPS.NumNegativePics+uint32(j)])
i++
}
}

if deltaRps < 0 && useDeltaFlag[numDeltaPocs] {
r.DeltaPocS0 = append(r.DeltaPocS0, deltaRps)
r.UsedByCurrPicS0Flag = append(r.UsedByCurrPicS0Flag, usedByCurrPicFlag[numDeltaPocs])
i++
}

for j := uint32(0); j < refRPS.NumNegativePics; j++ {
dPoc := refRPS.DeltaPocS0[j] + deltaRps
if dPoc < 0 && useDeltaFlag[j] {
r.DeltaPocS0 = append(r.DeltaPocS0, dPoc)
r.UsedByCurrPicS0Flag = append(r.UsedByCurrPicS0Flag, usedByCurrPicFlag[j])
i++
}
}

r.NumNegativePics = i

i = uint32(0)

for j := (int32(refRPS.NumNegativePics) - 1); j >= 0; j-- {
dPoc := refRPS.DeltaPocS0[j] + deltaRps
if dPoc > 0 && useDeltaFlag[j] {
r.DeltaPocS1 = append(r.DeltaPocS1, dPoc)
r.UsedByCurrPicS1Flag = append(r.UsedByCurrPicS1Flag, usedByCurrPicFlag[j])
i++
}
}

if deltaRps > 0 && useDeltaFlag[numDeltaPocs] {
r.DeltaPocS1 = append(r.DeltaPocS1, deltaRps)
r.UsedByCurrPicS1Flag = append(r.UsedByCurrPicS1Flag, usedByCurrPicFlag[numDeltaPocs])
i++
}

for j := uint32(0); j < refRPS.NumPositivePics; j++ {
dPoc := refRPS.DeltaPocS1[j] + deltaRps
if dPoc > 0 && useDeltaFlag[refRPS.NumNegativePics+j] {
r.DeltaPocS1 = append(r.DeltaPocS1, dPoc)
r.UsedByCurrPicS1Flag = append(r.UsedByCurrPicS1Flag, usedByCurrPicFlag[refRPS.NumNegativePics+j])
i++
}
}

r.NumPositivePics = i
} else {
r.NumNegativePics, err = bits.ReadGolombUnsigned(buf, pos)
if err != nil {
Expand All @@ -474,15 +542,21 @@ func (r *SPS_ShortTermRefPicSet) unmarshal(buf []byte, pos *int, stRpsIdx uint32
return fmt.Errorf("num_negative_pics exceeds %d", maxNegativePics)
}

r.DeltaPocS0Minus1 = make([]uint32, r.NumNegativePics)
r.DeltaPocS0 = make([]int32, r.NumNegativePics)
r.UsedByCurrPicS0Flag = make([]bool, r.NumNegativePics)

for i := uint32(0); i < r.NumNegativePics; i++ {
r.DeltaPocS0Minus1[i], err = bits.ReadGolombUnsigned(buf, pos)
deltaPocS0Minus1, err := bits.ReadGolombUnsigned(buf, pos)
if err != nil {
return err
}

if i == 0 {
r.DeltaPocS0[i] = -int32(deltaPocS0Minus1 + 1)
} else {
r.DeltaPocS0[i] = r.DeltaPocS0[i-1] - (int32(deltaPocS0Minus1) + 1)
}

r.UsedByCurrPicS0Flag[i], err = bits.ReadFlag(buf, pos)
if err != nil {
return err
Expand All @@ -495,15 +569,21 @@ func (r *SPS_ShortTermRefPicSet) unmarshal(buf []byte, pos *int, stRpsIdx uint32
return fmt.Errorf("num_positive_pics exceeds %d", maxPositivePics)
}

r.DeltaPocS1Minus1 = make([]uint32, r.NumPositivePics)
r.DeltaPocS1 = make([]int32, r.NumPositivePics)
r.UsedByCurrPicS1Flag = make([]bool, r.NumPositivePics)

for i := uint32(0); i < r.NumPositivePics; i++ {
r.DeltaPocS1Minus1[i], err = bits.ReadGolombUnsigned(buf, pos)
deltaPocS1Minus1, err := bits.ReadGolombUnsigned(buf, pos)
if err != nil {
return err
}

if i == 0 {
r.DeltaPocS1[i] = int32(deltaPocS1Minus1) + 1
} else {
r.DeltaPocS1[i] = r.DeltaPocS1[i-1] + int32(deltaPocS1Minus1) + 1
}

r.UsedByCurrPicS1Flag[i], err = bits.ReadFlag(buf, pos)
if err != nil {
return err
Expand Down
4 changes: 2 additions & 2 deletions pkg/codecs/h265/sps_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ var casesSPS = []struct {
SampleAdaptiveOffsetEnabledFlag: true,
ShortTermRefPicSets: []*SPS_ShortTermRefPicSet{{
NumNegativePics: 1,
DeltaPocS0Minus1: []uint32{0},
DeltaPocS0: []int32{-1},
UsedByCurrPicS0Flag: []bool{true},
}},
VUI: &SPS_VUI{
Expand Down Expand Up @@ -344,7 +344,7 @@ var casesSPS = []struct {
ShortTermRefPicSets: []*SPS_ShortTermRefPicSet{
{
NumNegativePics: 1,
DeltaPocS0Minus1: []uint32{0},
DeltaPocS0: []int32{-1},
UsedByCurrPicS0Flag: []bool{true},
},
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/codecs/h265/testdata/fuzz/FuzzSPSUnmarshal/0d39f7f9c8ac7afe
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("B000000000000000000002\xff0000\xfe0Z7")
2 changes: 2 additions & 0 deletions pkg/codecs/h265/testdata/fuzz/FuzzSPSUnmarshal/3a45cb6822dafb16
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("B0000000000000077y1$00A22200100")

0 comments on commit 1e6e29b

Please sign in to comment.