From 31da2753b751a2cde94145b195f97644ffd64ed9 Mon Sep 17 00:00:00 2001
From: Matt Wolenetz <wolenetz@chromium.org>
Date: Thu, 21 Mar 2019 17:29:37 -0700
Subject: [PATCH] MSE: Fix mediasource-changetype-play to work with
 MseBufferByPts

This web-platform-test exercises changeType as it splice-overlaps
pairs of audio or video media streams at varying offsets in the
presentation timeline. Splice-overlapping an out-of-order decode stream
(such as the test AVC MP4 media) at arbitrary times can, per spec, drop
significant decode dependencies from a partially-overlapped GOP such
that a buffered range gap could result.

This change is more careful about where it performs splice-overlaps when
the overlapped media is out-of-order-decode, adjusting the splice point
to be at or very near to the next overlapped keyframe. This prevents
removing out-of-order non-keyframes and their dependents from the
overlapped media such that no buffered range gap nor playback stall
should result.

Note that Chromium is sensitive to such out-of-order buffering overlaps
with the new, compliant, MseBufferByPts behavior. Fixing
https://github.com/w3c/media-source/issues/160 could greatly simplify
this problem by allowing apps to explicitly control how the
user agent behaves at these small gaps.

BUG=807793

Change-Id: I020e244c230756eaa1804f81b58a577124a6a28b
---
 media-source/mediasource-changetype-play.html |  33 +-
 media-source/mediasource-changetype-util.js   | 340 +++++++++++-------
 2 files changed, 217 insertions(+), 156 deletions(-)

diff --git a/media-source/mediasource-changetype-play.html b/media-source/mediasource-changetype-play.html
index c4527149d0d4b8..ae6bd9b1b901ff 100644
--- a/media-source/mediasource-changetype-play.html
+++ b/media-source/mediasource-changetype-play.html
@@ -12,27 +12,26 @@
         <div id="log"></div>
         <script>
 
-function generateChangeTypeTests(audio_types, video_types)
-{
-    async_test(function(test) {
-        assert_true(audio_types.length > 1, "Browser doesn't support at least 2 types of audio test media" + audio_types);
-        assert_true(video_types.length > 1, "Browser doesn't support at least 2 types of video test media");
-        test.done();
-    }, "Check if browser supports enough test media types");
+function generateChangeTypeTests(audio_types, video_types) {
+  async_test((test) => {
+    assert_true(audio_types.length > 1, "Browser doesn't support at least 2 types of audio test media");
+    assert_true(video_types.length > 1, "Browser doesn't support at least 2 types of video test media");
+    test.done();
+  }, "Check if browser supports enough test media types");
 
-    // Generate audio-only changeType tests
-    for (let audio1 of audio_types) {
-        for (let audio2 of audio_types) {
-            mediaSourceChangeTypeTest(audio1, audio2, "Test audio-only changeType for " + audio1.type + " <-> " + audio2.type);
-        }
+  // Generate audio-only changeType tests
+  for (let audio1 of audio_types) {
+    for (let audio2 of audio_types) {
+      mediaSourceChangeTypeTest(audio1, audio2, "Test audio-only changeType for " + audio1.type + " <-> " + audio2.type);
     }
+  }
 
-    // Generate video-only changeType tests
-    for (let video1 of video_types) {
-        for (let video2 of video_types) {
-            mediaSourceChangeTypeTest(video1, video2, "Test video-only changeType for " + video1.type + " <-> " + video2.type);
-        }
+  // Generate video-only changeType tests
+  for (let video1 of video_types) {
+    for (let video2 of video_types) {
+      mediaSourceChangeTypeTest(video1, video2, "Test video-only changeType for " + video1.type + " <-> " + video2.type);
     }
+  }
 }
 
 findSupportedChangeTypeTestTypes(generateChangeTypeTests);
diff --git a/media-source/mediasource-changetype-util.js b/media-source/mediasource-changetype-util.js
index 4e24223e0439f6..f183bf653e5d78 100644
--- a/media-source/mediasource-changetype-util.js
+++ b/media-source/mediasource-changetype-util.js
@@ -1,156 +1,218 @@
 // Copyright © 2018 Chromium authors and World Wide Web Consortium, (Massachusetts Institute of Technology, ERCIM, Keio University, Beihang).
 
-function findSupportedChangeTypeTestTypes(cb)
-{
-    var CHANGE_TYPE_MEDIA_LIST = [
-        {
-            type: 'video/webm; codecs="vp8"',
-            is_video: true,
-            url: 'webm/test-v-128k-320x240-24fps-8kfr.webm'
-        },
-        {
-            type: 'video/webm; codecs="vp9"',
-            is_video: true,
-            url: 'webm/test-vp9.webm'
-        },
-        {
-            type: 'video/mp4; codecs="avc1.4D4001"',
-            is_video: true,
-            url: 'mp4/test-v-128k-320x240-24fps-8kfr.mp4'
-        },
-        {
-            type: 'audio/webm; codecs="vorbis"',
-            is_video: false,
-            url: 'webm/test-a-128k-44100Hz-1ch.webm'
-        },
-        {
-            type: 'audio/mp4; codecs="mp4a.40.2"',
-            is_video: false,
-            url: 'mp4/test-a-128k-44100Hz-1ch.mp4'
-        },
-        {
-            type: 'audio/mpeg',
-            is_video: false,
-            url: 'mp3/sound_5.mp3'
-        }
-    ];
-
-    var audio_result = [];
-    var video_result = [];
-
-    for (var i = 0; i < CHANGE_TYPE_MEDIA_LIST.length; ++i) {
-        var media = CHANGE_TYPE_MEDIA_LIST[i];
-        if (window.MediaSource && MediaSource.isTypeSupported(media.type)) {
-            if (media.is_video === true) {
-                video_result.push(media);
-            } else {
-                audio_result.push(media);
-            }
-        }
+function findSupportedChangeTypeTestTypes(cb) {
+  let CHANGE_TYPE_MEDIA_LIST = [
+    {
+      type: 'video/webm; codecs="vp8"',
+      is_video: true,
+      url: 'webm/test-v-128k-320x240-24fps-8kfr.webm',
+      start_time: 0.0
+      // keyframe_interval: N/A since DTS==PTS so overlap-removal of
+      // non-keyframe should not produce a buffered range gap.
+    },
+    {
+      type: 'video/webm; codecs="vp9"',
+      is_video: true,
+      url: 'webm/test-vp9.webm',
+      start_time: 0.0
+      // keyframe_interval: N/A since DTS==PTS so overlap-removal of
+      // non-keyframe should not produce a buffered range gap.
+    },
+    {
+      type: 'video/mp4; codecs="avc1.4D4001"',
+      is_video: true,
+      url: 'mp4/test-v-128k-320x240-24fps-8kfr.mp4',
+      start_time: 0.083333,
+      keyframe_interval: 0.333333
+    },
+    {
+      type: 'audio/webm; codecs="vorbis"',
+      is_video: false,
+      url: 'webm/test-a-128k-44100Hz-1ch.webm',
+      start_time: 0.0
+      // keyframe_interval: N/A since DTS==PTS so overlap-removal of
+      // non-keyframe should not produce a buffered range gap. Also, all frames
+      // in this media are key-frames (it is audio).
+    },
+    {
+      type: 'audio/mp4; codecs="mp4a.40.2"',
+      is_video: false,
+      url: 'mp4/test-a-128k-44100Hz-1ch.mp4',
+      start_time: 0.0
+      // keyframe_interval: N/A since DTS==PTS so overlap-removal of
+      // non-keyframe should not produce a buffered range gap. Also, all frames
+      // in this media are key-frames (it is audio).
+    },
+    {
+      type: 'audio/mpeg',
+      is_video: false,
+      url: 'mp3/sound_5.mp3',
+      start_time: 0.0
+      // keyframe_interval: N/A since DTS==PTS so overlap-removal of
+      // non-keyframe should not produce a buffered range gap. Also, all frames
+      // in this media are key-frames (it is audio).
     }
+  ];
+
+  let audio_result = [];
+  let video_result = [];
+
+  for (let i = 0; i < CHANGE_TYPE_MEDIA_LIST.length; ++i) {
+    let media = CHANGE_TYPE_MEDIA_LIST[i];
+    if (window.MediaSource && MediaSource.isTypeSupported(media.type)) {
+      if (media.is_video === true) {
+        video_result.push(media);
+      } else {
+        audio_result.push(media);
+      }
+    }
+  }
 
-
-    cb(audio_result, video_result);
+  cb(audio_result, video_result);
 }
 
-function appendBuffer(test, sourceBuffer, data)
-{
-    test.expectEvent(sourceBuffer, "update");
-    test.expectEvent(sourceBuffer, "updateend");
-    sourceBuffer.appendBuffer(data);
+function appendBuffer(test, sourceBuffer, data) {
+  test.expectEvent(sourceBuffer, "update");
+  test.expectEvent(sourceBuffer, "updateend");
+  sourceBuffer.appendBuffer(data);
 }
 
-function trimBuffered(test, mediaElement, sourceBuffer, minimumPreviousDuration, newDuration)
-{
-    assert_less_than(newDuration, minimumPreviousDuration);
-    assert_less_than(minimumPreviousDuration, mediaElement.duration);
-    test.expectEvent(sourceBuffer, "update");
-    test.expectEvent(sourceBuffer, "updateend");
-    sourceBuffer.remove(newDuration, Infinity);
+function trimBuffered(test, mediaElement, sourceBuffer, minimumPreviousDuration, newDuration) {
+  assert_less_than(newDuration, minimumPreviousDuration);
+  assert_less_than(minimumPreviousDuration, mediaElement.duration);
+  test.expectEvent(sourceBuffer, "update");
+  test.expectEvent(sourceBuffer, "updateend");
+  sourceBuffer.remove(newDuration, Infinity);
 }
 
-function trimDuration(test, mediaElement, mediaSource, newDuration)
-{
-    assert_less_than(newDuration, mediaElement.duration);
-    test.expectEvent(mediaElement, "durationchange");
-    mediaSource.duration = newDuration;
+function trimDuration(test, mediaElement, mediaSource, newDuration) {
+  assert_less_than(newDuration, mediaElement.duration);
+  test.expectEvent(mediaElement, "durationchange");
+  mediaSource.duration = newDuration;
 }
 
-function runChangeTypeTest(test, mediaElement, mediaSource, typeA, dataA, typeB, dataB)
-{
-    var sourceBuffer = mediaSource.addSourceBuffer(typeA);
+function runChangeTypeTest(test, mediaElement, mediaSource, metadataA, dataA, metadataB, dataB) {
+  // Some streams, like the MP4 video stream, contain presentation times for
+  // frames out of order versus their decode times. If we overlap-append the
+  // latter part of such a stream's GOP presentation interval, a significant
+  // portion of decode-dependent non-keyframes with earlier presentation
+  // intervals could be removed and a presentation time buffered range gap could
+  // be introduced. Therefore, we test overlap appends with the overlaps
+  // occurring very near to a keyframe's presentation time to reduce the
+  // possibility of such a gap. None of the test media is SAP-Type-2, so we
+  // don't take any extra care to avoid gaps that may occur when
+  // splice-overlapping such GOP sequences that aren't SAP-Type-1.
+  // TODO(wolenetz): https://github.com/w3c/media-source/issues/160 could
+  // greatly simplify this problem by allowing us play through these small gaps.
+
+  function findSafeOffset(targetTime, overlappedMediaMetadata, overlappedStartTime, overlappingMediaMetadata) {
+    assert_greater_than_equal(targetTime, overlappedStartTime);
+
+    let offset = targetTime;
+    if ("start_time" in overlappingMediaMetadata)
+      offset -= overlappingMediaMetadata["start_time"];
+
+    // If the media being overlapped is not out-of-order decode, then we can
+    // safely use the supplied times.
+    if (!("keyframe_interval" in overlappedMediaMetadata)) {
+      return { "offset": offset, "adjustedTime": targetTime };
+    }
 
+    // Otherwise, we're overlapping media that needs care to prevent introducing
+    // a gap. Adjust offset and adjustedTime to make the overlapping media start
+    // at the next overlapped media keyframe at or after targetTime.
+    let gopsToRetain = Math.ceil((targetTime - overlappedStartTime) / overlappedMediaMetadata["keyframe_interval"]);
+    let adjustedTime = overlappedStartTime + gopsToRetain * overlappedMediaMetadata["keyframe_interval"];
+
+    assert_greater_than_equal(adjustedTime, targetTime);
+    offset += adjustedTime - targetTime;
+    return { "offset": offset, "adjustedTime": adjustedTime };
+  }
+
+  let sourceBuffer = mediaSource.addSourceBuffer(metadataA.type);
+
+  appendBuffer(test, sourceBuffer, dataA);
+  let lastStart = metadataA["start_time"];
+  if (lastStart == null)
+    lastStart = 0.0;
+
+  // changeType A->B and append the first media of B effectively at 0.5 seconds
+  // (or at the first keyframe in A at or after 0.5 seconds if it has
+  // keyframe_interval defined).
+  test.waitForExpectedEvents(() => {
+    let safeOffset = findSafeOffset(0.5, metadataA, lastStart, metadataB);
+    lastStart = safeOffset["adjustedTime"];
+    sourceBuffer.changeType(metadataB.type);
+    sourceBuffer.timestampOffset = safeOffset["offset"];
+    appendBuffer(test, sourceBuffer, dataB);
+  });
+
+  // changeType B->B and append B starting at 1.0 seconds (or at the first
+  // keyframe in B at or after 1.0 seconds if it has keyframe_interval defined).
+  test.waitForExpectedEvents(() => {
+    assert_less_than(lastStart, 1.0);
+    let safeOffset = findSafeOffset(1.0, metadataB, lastStart, metadataB);
+    lastStart = safeOffset["adjustedTime"];
+    sourceBuffer.changeType(metadataB.type);
+    sourceBuffer.timestampOffset = safeOffset["offset"];
+    appendBuffer(test, sourceBuffer, dataB);
+  });
+
+  // changeType B->A and append A starting at 1.5 seconds (or at the first
+  // keyframe in B at or after 1.5 seconds if it has keyframe_interval defined).
+  test.waitForExpectedEvents(() => {
+    assert_less_than(lastStart, 1.5);
+    let safeOffset = findSafeOffset(1.5, metadataB, lastStart, metadataA);
+    // Retain the previous lastStart because the next block will append data
+    // which begins between that start time and this block's start time.
+    sourceBuffer.changeType(metadataA.type);
+    sourceBuffer.timestampOffset = safeOffset["offset"];
     appendBuffer(test, sourceBuffer, dataA);
-
-    // changeType A->B and append B starting at 0.5 seconds.
-    test.waitForExpectedEvents(function()
-    {
-        sourceBuffer.changeType(typeB);
-        sourceBuffer.timestampOffset = 0.5;
-        appendBuffer(test, sourceBuffer, dataB);
-    });
-
-    // changeType B->B and append B starting at 1.0 seconds.
-    test.waitForExpectedEvents(function()
-    {
-        sourceBuffer.changeType(typeB);
-        sourceBuffer.timestampOffset = 1.0;
-        appendBuffer(test, sourceBuffer, dataB);
-    });
-
-    // changeType B->A and append A starting at 1.5 seconds.
-    test.waitForExpectedEvents(function()
-    {
-        sourceBuffer.changeType(typeA);
-        sourceBuffer.timestampOffset = 1.5;
-        appendBuffer(test, sourceBuffer, dataA);
-    });
-
-    // changeTypoe A->A and append A starting at 1.3 seconds.
-    test.waitForExpectedEvents(function()
-    {
-        sourceBuffer.changeType(typeA);
-        sourceBuffer.timestampOffset = 1.3;
-        appendBuffer(test, sourceBuffer, dataA);
-    });
-
-    // Trim duration to 2 seconds, then play through to end.
-    test.waitForExpectedEvents(function()
-    {
-        trimBuffered(test, mediaElement, sourceBuffer, 2.1, 2);
-    });
-
-    test.waitForExpectedEvents(function()
-    {
-        trimDuration(test, mediaElement, mediaSource, 2);
-    });
-
-    test.waitForExpectedEvents(function()
-    {
-        assert_equals(mediaElement.currentTime, 0);
-        test.expectEvent(mediaSource, "sourceended");
-        test.expectEvent(mediaElement, "play");
-        test.expectEvent(mediaElement, "ended");
-        mediaSource.endOfStream();
-        mediaElement.play();
-    });
-
-    test.waitForExpectedEvents(function() {
-        test.done();
-    });
+  });
+
+  // changeType A->A and append A starting at 1.3 seconds (or at the first
+  // keyframe in B at or after 1.3 seconds if it has keyframe_interval defined).
+  test.waitForExpectedEvents(() => {
+    assert_less_than(lastStart, 1.3);
+    // Our next append will begin by overlapping some of metadataB, then some of
+    // metadataA.
+    let safeOffset = findSafeOffset(1.3, metadataB, lastStart, metadataA);
+    sourceBuffer.changeType(metadataA.type);
+    sourceBuffer.timestampOffset = safeOffset["offset"];
+    appendBuffer(test, sourceBuffer, dataA);
+  });
+
+  // Trim duration to 2 seconds, then play through to end.
+  test.waitForExpectedEvents(() => {
+    trimBuffered(test, mediaElement, sourceBuffer, 2.1, 2);
+  });
+
+  test.waitForExpectedEvents(() => {
+    trimDuration(test, mediaElement, mediaSource, 2);
+  });
+
+  test.waitForExpectedEvents(() => {
+    assert_equals(mediaElement.currentTime, 0);
+    test.expectEvent(mediaSource, "sourceended");
+    test.expectEvent(mediaElement, "play");
+    test.expectEvent(mediaElement, "ended");
+    mediaSource.endOfStream();
+    mediaElement.play();
+  });
+
+  test.waitForExpectedEvents(() => {
+    test.done();
+  });
 }
 
-function mediaSourceChangeTypeTest(metadataA, metadataB, description)
-{
-    mediasource_test(function(test, mediaElement, mediaSource)
-    {
-        mediaElement.pause();
-        mediaElement.addEventListener('error', test.unreached_func("Unexpected event 'error'"));
-        MediaSourceUtil.loadBinaryData(test, metadataA.url, function(dataA) {
-            MediaSourceUtil.loadBinaryData(test, metadataB.url, function(dataB) {
-                runChangeTypeTest(test, mediaElement, mediaSource, metadataA.type, dataA, metadataB.type, dataB);
-            });
-        });
-    }, description);
+function mediaSourceChangeTypeTest(metadataA, metadataB, description) {
+  mediasource_test((test, mediaElement, mediaSource) => {
+    mediaElement.pause();
+    mediaElement.addEventListener('error', test.unreached_func("Unexpected event 'error'"));
+    MediaSourceUtil.loadBinaryData(test, metadataA.url, (dataA) => {
+      MediaSourceUtil.loadBinaryData(test, metadataB.url, (dataB) => {
+        runChangeTypeTest(test, mediaElement, mediaSource, metadataA, dataA, metadataB, dataB);
+      });
+    });
+  }, description);
 }
-