From 2ad2b196b83880b45f009cd52759509a42b74217 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 4 Oct 2019 11:43:27 +0000 Subject: [PATCH] Bug 1538433 [wpt PR 15991] - MSE: Fix mediasource-changetype-play to work with MseBufferByPts, a=testonly Automatic update from web-platform-tests MSE: Fix mediasource-changetype-play to work with MseBufferByPts This web-platform-test exercises changeType as it splice-overlaps pairs of audio or video media streams at varying offsets in the presentation timeline. Splice-overlapping an out-of-order decode stream (such as the test AVC MP4 media) at arbitrary times can, per spec, drop significant decode dependencies from a partially-overlapped GOP such that a buffered range gap could result. This change is more careful about where it performs splice-overlaps when the overlapped media is out-of-order-decode, adjusting the splice point to be at or very near to the next overlapped keyframe. This prevents removing out-of-order non-keyframes and their dependents from the overlapped media such that no buffered range gap nor playback stall should result. Note that Chromium is sensitive to such out-of-order buffering overlaps with the new, compliant, MseBufferByPts behavior. Fixing https://github.com/w3c/media-source/issues/160 could greatly simplify this problem by allowing apps to explicitly control how the user agent behaves at these small gaps. BUG=807793 Change-Id: I020e244c230756eaa1804f81b58a577124a6a28b Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1428601 Commit-Queue: Matthew Wolenetz Reviewed-by: Dan Sanders Cr-Commit-Position: refs/heads/master{#643473} -- wpt-commits: 27ad6759d421b95b4572f20cabaeb750b3eb9799 wpt-pr: 15991 UltraBlame original commit: c679de39ad4dbaafeee76985263d5c11c07de772 --- .../mediasource-changetype-play.html | 5 +- .../mediasource-changetype-util.js | 1104 +++++++++++++++-- 2 files changed, 1017 insertions(+), 92 deletions(-) diff --git a/testing/web-platform/tests/media-source/mediasource-changetype-play.html b/testing/web-platform/tests/media-source/mediasource-changetype-play.html index ebfca33e08649..835bde5c1db9e 100644 --- a/testing/web-platform/tests/media-source/mediasource-changetype-play.html +++ b/testing/web-platform/tests/media-source/mediasource-changetype-play.html @@ -162,10 +162,11 @@ { async_test ( -function ( test ) += +> { assert_true ( @@ -189,8 +190,6 @@ test media " -+ -audio_types ) ; assert_true diff --git a/testing/web-platform/tests/media-source/mediasource-changetype-util.js b/testing/web-platform/tests/media-source/mediasource-changetype-util.js index 6fe7ca056fd54..76b25391f456e 100644 --- a/testing/web-platform/tests/media-source/mediasource-changetype-util.js +++ b/testing/web-platform/tests/media-source/mediasource-changetype-util.js @@ -26,7 +26,7 @@ findSupportedChangeTypeTestTypes cb ) { -var +let CHANGE_TYPE_MEDIA_LIST = [ @@ -66,6 +66,41 @@ v . webm ' +start_time +: +0 +. +0 +/ +/ +keyframe_interval +: +N +/ +A +since +DTS += += +PTS +so +overlap +- +removal +of +/ +/ +non +- +keyframe +should +not +produce +a +buffered +range +gap +. } { type @@ -95,6 +130,41 @@ vp9 . webm ' +start_time +: +0 +. +0 +/ +/ +keyframe_interval +: +N +/ +A +since +DTS += += +PTS +so +overlap +- +removal +of +/ +/ +non +- +keyframe +should +not +produce +a +buffered +range +gap +. } { type @@ -134,6 +204,16 @@ v . mp4 ' +start_time +: +0 +. +083333 +keyframe_interval +: +0 +. +333333 } { type @@ -169,6 +249,59 @@ a . webm ' +start_time +: +0 +. +0 +/ +/ +keyframe_interval +: +N +/ +A +since +DTS += += +PTS +so +overlap +- +removal +of +/ +/ +non +- +keyframe +should +not +produce +a +buffered +range +gap +. +Also +all +frames +/ +/ +in +this +media +are +key +- +frames +( +it +is +audio +) +. } { type @@ -208,6 +341,59 @@ a . mp4 ' +start_time +: +0 +. +0 +/ +/ +keyframe_interval +: +N +/ +A +since +DTS += += +PTS +so +overlap +- +removal +of +/ +/ +non +- +keyframe +should +not +produce +a +buffered +range +gap +. +Also +all +frames +/ +/ +in +this +media +are +key +- +frames +( +it +is +audio +) +. } { type @@ -229,16 +415,69 @@ sound_5 . mp3 ' +start_time +: +0 +. +0 +/ +/ +keyframe_interval +: +N +/ +A +since +DTS += += +PTS +so +overlap +- +removal +of +/ +/ +non +- +keyframe +should +not +produce +a +buffered +range +gap +. +Also +all +frames +/ +/ +in +this +media +are +key +- +frames +( +it +is +audio +) +. } ] ; -var +let audio_result = [ ] ; -var +let video_result = [ @@ -246,7 +485,7 @@ video_result ; for ( -var +let i = 0 @@ -262,7 +501,7 @@ length i ) { -var +let media = CHANGE_TYPE_MEDIA_LIST @@ -454,118 +693,654 @@ runChangeTypeTest test mediaElement mediaSource -typeA +metadataA dataA -typeB +metadataB dataB ) { -var -sourceBuffer -= -mediaSource -. -addSourceBuffer -( -typeA -) -; -appendBuffer -( -test -sourceBuffer -dataA -) -; / / -changeType -A +Some +streams +like +the +MP4 +video +stream +contain +presentation +times +for +/ +/ +frames +out +of +order +versus +their +decode +times +. +If +we +overlap - -> -B -and append -B -starting -at -0 +the +/ +/ +latter +part +of +such +a +stream +' +s +GOP +presentation +interval +a +significant +/ +/ +portion +of +decode +- +dependent +non +- +keyframes +with +earlier +presentation +/ +/ +intervals +could +be +removed +and +a +presentation +time +buffered +range +gap +could +/ +/ +be +introduced . -5 -seconds +Therefore +we +test +overlap +appends +with +the +overlaps +/ +/ +occurring +very +near +to +a +keyframe +' +s +presentation +time +to +reduce +the +/ +/ +possibility +of +such +a +gap . +None +of +the test +media +is +SAP +- +Type +- +2 +so +we +/ +/ +don +' +t +take +any +extra +care +to +avoid +gaps +that +may +occur +when +/ +/ +splice +- +overlapping +such +GOP +sequences +that +aren +' +t +SAP +- +Type +- +1 . -waitForExpectedEvents +/ +/ +TODO ( +wolenetz +) +: +https +: +/ +/ +github +. +com +/ +w3c +/ +media +- +source +/ +issues +/ +160 +could +/ +/ +greatly +simplify +this +problem +by +allowing +us +play +through +these +small +gaps +. function +findSafeOffset ( +targetTime +overlappedMediaMetadata +overlappedStartTime +overlappingMediaMetadata ) { -sourceBuffer -. -changeType +assert_greater_than_equal ( -typeB +targetTime +overlappedStartTime ) ; -sourceBuffer -. -timestampOffset +let +offset = -0 -. -5 +targetTime ; -appendBuffer +if ( -test -sourceBuffer -dataB +" +start_time +" +in +overlappingMediaMetadata ) +{ +offset +- += +overlappingMediaMetadata +[ +" +start_time +" +] ; } -) -; / / -changeType -B +If +the +media +being +overlapped +is +not +out - -> -B -and -append -B -starting -at -1 -. -0 -seconds +of +- +order +decode +then +we +can +/ +/ +safely +use +the +supplied +times +. +if +( +! +( +" +keyframe_interval +" +in +overlappedMediaMetadata +) +) +{ +return +{ +" +offset +" +: +offset +" +adjustedTime +" +: +targetTime +} +; +} +/ +/ +Otherwise +we +' +re +overlapping +media +that +needs +care +to +prevent +introducing +/ +/ +a +gap +. +Adjust +offset +and +adjustedTime +to +make +the +overlapping +media +start +/ +/ +at +the +next +overlapped +media +keyframe +at +or +after +targetTime +. +let +gopsToRetain += +Math +. +ceil +( +( +targetTime +- +overlappedStartTime +) +/ +overlappedMediaMetadata +[ +" +keyframe_interval +" +] +) +; +let +adjustedTime += +overlappedStartTime ++ +gopsToRetain +* +overlappedMediaMetadata +[ +" +keyframe_interval +" +] +; +assert_greater_than_equal +( +adjustedTime +targetTime +) +; +offset ++ += +adjustedTime +- +targetTime +; +return +{ +" +offset +" +: +offset +" +adjustedTime +" +: +adjustedTime +} +; +} +let +sourceBuffer += +mediaSource +. +addSourceBuffer +( +metadataA +. +type +) +; +appendBuffer +( +test +sourceBuffer +dataA +) +; +let +lastStart += +metadataA +[ +" +start_time +" +] +; +if +( +lastStart += += +null +) +{ +lastStart += +0 +. +0 +; +} +/ +/ +changeType +A +- +> +B +and +append +the +first +media +of +B +effectively +at +0 +. +5 +seconds +/ +/ +( +or +at +the +first +keyframe +in +A +at +or +after +0 +. +5 +seconds +if +it +has +/ +/ +keyframe_interval +defined +) . test . waitForExpectedEvents ( -function ( ) += +> { +let +safeOffset += +findSafeOffset +( +0 +. +5 +metadataA +lastStart +metadataB +) +; +lastStart += +safeOffset +[ +" +adjustedTime +" +] +; sourceBuffer . changeType ( -typeB +metadataB +. +type ) ; sourceBuffer . timestampOffset = +safeOffset +[ +" +offset +" +] +; +appendBuffer +( +test +sourceBuffer +dataB +) +; +} +) +; +/ +/ +changeType +B +- +> +B +and +append +B +starting +at +1 +. +0 +seconds +( +or +at +the +first +/ +/ +keyframe +in +B +at +or +after +1 +. +0 +seconds +if +it +has +keyframe_interval +defined +) +. +test +. +waitForExpectedEvents +( +( +) += +> +{ +assert_less_than +( +lastStart +1 +. +0 +) +; +let +safeOffset += +findSafeOffset +( 1 . 0 +metadataB +lastStart +metadataB +) +; +lastStart += +safeOffset +[ +" +adjustedTime +" +] +; +sourceBuffer +. +changeType +( +metadataB +. +type +) +; +sourceBuffer +. +timestampOffset += +safeOffset +[ +" +offset +" +] ; appendBuffer ( @@ -593,29 +1368,108 @@ at . 5 seconds +( +or +at +the +first +/ +/ +keyframe +in +B +at +or +after +1 +. +5 +seconds +if +it +has +keyframe_interval +defined +) . test . waitForExpectedEvents ( -function ( ) += +> { +assert_less_than +( +lastStart +1 +. +5 +) +; +let +safeOffset += +findSafeOffset +( +1 +. +5 +metadataB +lastStart +metadataA +) +; +/ +/ +Retain +the +previous +lastStart +because +the +next +block +will +append +data +/ +/ +which +begins +between +that +start +time +and +this +block +' +s +start +time +. sourceBuffer . changeType ( -typeA +metadataA +. +type ) ; sourceBuffer . timestampOffset = -1 -. -5 +safeOffset +[ +" +offset +" +] ; appendBuffer ( @@ -629,7 +1483,7 @@ dataA ; / / -changeTypoe +changeType A - > @@ -643,29 +1497,98 @@ at . 3 seconds +( +or +at +the +first +/ +/ +keyframe +in +B +at +or +after +1 +. +3 +seconds +if +it +has +keyframe_interval +defined +) . test . waitForExpectedEvents ( -function ( ) += +> { +assert_less_than +( +lastStart +1 +. +3 +) +; +/ +/ +Our +next +append +will +begin +by +overlapping +some +of +metadataB +then +some +of +/ +/ +metadataA +. +let +safeOffset += +findSafeOffset +( +1 +. +3 +metadataB +lastStart +metadataA +) +; sourceBuffer . changeType ( -typeA +metadataA +. +type ) ; sourceBuffer . timestampOffset = -1 -. -3 +safeOffset +[ +" +offset +" +] ; appendBuffer ( @@ -694,9 +1617,10 @@ test . waitForExpectedEvents ( -function ( ) += +> { trimBuffered ( @@ -716,9 +1640,10 @@ test . waitForExpectedEvents ( -function ( ) += +> { trimDuration ( @@ -735,9 +1660,10 @@ test . waitForExpectedEvents ( -function ( ) += +> { assert_equals ( @@ -796,9 +1722,10 @@ test . waitForExpectedEvents ( -function ( ) += +> { test . @@ -820,12 +1747,13 @@ description { mediasource_test ( -function ( test mediaElement mediaSource ) += +> { mediaElement . @@ -862,10 +1790,11 @@ test metadataA . url -function ( dataA ) += +> { MediaSourceUtil . @@ -875,10 +1804,11 @@ test metadataB . url -function ( dataB ) += +> { runChangeTypeTest ( @@ -886,12 +1816,8 @@ test mediaElement mediaSource metadataA -. -type dataA metadataB -. -type dataB ) ;