fix: Fix green screen issue on Edge with mixed content (#6719)

On Edge, to properly play mixed content, we need to insert init segment twice for clear part - once as encrypted, and immediately again as clear. Otherwise we may encounter green screen and errors from video decoder. Co-authored-by: Joey Parrish <joeyparrish@users.noreply.github.com> Co-authored-by: Nick Michael <nick-michael@users.noreply.github.com> Co-authored-by: Álvaro Velad Galván <ladvan91@hotmail.com> Co-authored-by: Casey Occhialini <1508707+littlespex@users.noreply.github.com>
shaka-project · Jun 7, 2024 · d5b1863 · d5b1863
1 parent 60e6847
commit d5b1863
Show file tree

Hide file tree

Showing 15 changed files with 219 additions and 98 deletions.
diff --git a/build/types/core b/build/types/core
@@ -79,6 +79,7 @@
 +../../lib/util/delayed_tick.js
 +../../lib/util/destroyer.js
 +../../lib/util/dom_utils.js
++../../lib/util/drm_utils.js
 +../../lib/util/ebml_parser.js
 +../../lib/util/error.js
 +../../lib/util/event_manager.js

diff --git a/lib/media/content_workarounds.js b/lib/media/content_workarounds.js
@@ -36,7 +36,8 @@ shaka.media.ContentWorkarounds = class {
    */
   static fakeEncryption(initSegmentBuffer, uri) {
     const ContentWorkarounds = shaka.media.ContentWorkarounds;
-    let initSegment = shaka.util.BufferUtils.toUint8(initSegmentBuffer);
+    const initSegment = shaka.util.BufferUtils.toUint8(initSegmentBuffer);
+    let modifiedInitSegment = initSegment;
     let isEncrypted = false;
     /** @type {shaka.extern.ParsedBox} */
     let stsdBox;
@@ -166,11 +167,24 @@ shaka.media.ContentWorkarounds = class {
       const insertedBoxType =
           shaka.util.Mp4Parser.typeToString(workItem.newType);
       shaka.log.debug(`Inserting "${insertedBoxType}" box into init segment.`);
-      initSegment = ContentWorkarounds.insertEncryptionMetadata_(
-          initSegment, stsdBox, workItem.box, ancestorBoxes, workItem.newType);
+      modifiedInitSegment = ContentWorkarounds.insertEncryptionMetadata_(
+          modifiedInitSegment, stsdBox, workItem.box, ancestorBoxes,
+          workItem.newType);
     }
 
-    return initSegment;
+    // Edge Windows needs the unmodified init segment to be appended after the
+    // patched one, otherwise video element throws following error:
+    // CHUNK_DEMUXER_ERROR_APPEND_FAILED: Sample encryption info is not
+    // available.
+    if (shaka.util.Platform.isEdge() && shaka.util.Platform.isWindows()) {
+      const doubleInitSegment = new Uint8Array(initSegment.byteLength +
+        modifiedInitSegment.byteLength);
+      doubleInitSegment.set(modifiedInitSegment);
+      doubleInitSegment.set(initSegment, modifiedInitSegment.byteLength);
+      return doubleInitSegment;
+    }
+
+    return modifiedInitSegment;
   }
 
   /**
@@ -197,11 +211,12 @@ shaka.media.ContentWorkarounds = class {
     const newInitSegment =
         new Uint8Array(initSegment.byteLength + metadataBoxArray.byteLength);
 
-    // For Xbox One, we cut and insert at the start of the source box.  For
-    // other platforms, we cut and insert at the end of the source box.  It's
+    // For Xbox One & Edge, we cut and insert at the start of the source box.
+    // For other platforms, we cut and insert at the end of the source box. It's
     // not clear why this is necessary on Xbox One, but it seems to be evidence
     // of another bug in the firmware implementation of MediaSource & EME.
-    const cutPoint = shaka.util.Platform.isXboxOne() ?
+    const cutPoint =
+      (shaka.util.Platform.isXboxOne() || shaka.util.Platform.isEdge()) ?
         sourceBox.start :
         sourceBox.start + sourceBox.size;
 

diff --git a/lib/media/drm_engine.js b/lib/media/drm_engine.js
@@ -12,6 +12,7 @@ goog.require('shaka.net.NetworkingEngine');
 goog.require('shaka.transmuxer.TransmuxerEngine');
 goog.require('shaka.util.BufferUtils');
 goog.require('shaka.util.Destroyer');
+goog.require('shaka.util.DrmUtils');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.EventManager');
 goog.require('shaka.util.FakeEvent');
@@ -762,28 +763,6 @@ shaka.media.DrmEngine = class {
     return drmInfo ? drmInfo.keySystem : '';
   }
 
-  /**
-   * @param {?string} keySystem
-   * @return {boolean} */
-  static isPlayReadyKeySystem(keySystem) {
-    if (keySystem) {
-      return !!keySystem.match(/^com\.(microsoft|chromecast)\.playready/);
-    }
-
-    return false;
-  }
-
-  /**
-   * @param {?string} keySystem
-   * @return {boolean} */
-  static isFairPlayKeySystem(keySystem) {
-    if (keySystem) {
-      return !!keySystem.match(/^com\.apple\.fps/);
-    }
-
-    return false;
-  }
-
   /**
    * Check if DrmEngine (as initialized) will likely be able to support the
    * given content type.
@@ -1502,7 +1481,7 @@ shaka.media.DrmEngine = class {
     }
     // NOTE: allowCrossSiteCredentials can be set in a request filter.
 
-    if (shaka.media.DrmEngine.isPlayReadyKeySystem(
+    if (shaka.util.DrmUtils.isPlayReadyKeySystem(
         this.currentDrmInfo_.keySystem)) {
       this.unpackPlayReadyRequest_(request);
     }
@@ -1686,7 +1665,7 @@ shaka.media.DrmEngine = class {
       // NOTE that we skip this if byteLength != 16.  This is used for Edge
       // which uses single-byte dummy key IDs.
       // However, unlike Edge and Chromecast, Tizen doesn't have this problem.
-      if (shaka.media.DrmEngine.isPlayReadyKeySystem(
+      if (shaka.util.DrmUtils.isPlayReadyKeySystem(
           this.currentDrmInfo_.keySystem) &&
           keyId.byteLength == 16 &&
           (shaka.util.Platform.isEdge() || shaka.util.Platform.isPS4())) {

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -53,15 +53,17 @@ shaka.media.MediaSourceEngine = class {
    *    The text displayer that will be used with the text engine.
    *    MediaSourceEngine takes ownership of the displayer. When
    *    MediaSourceEngine is destroyed, it will destroy the displayer.
-   * @param {!function(!Array.<shaka.extern.ID3Metadata>, number, ?number)=}
-   *    onMetadata
+   * @param {!shaka.media.MediaSourceEngine.PlayerInterface} playerInterface
+   *   Interface for common player methods.
    * @param {?shaka.lcevc.Dec} [lcevcDec] Optional -  LCEVC Decoder Object
-   *
    */
-  constructor(video, textDisplayer, onMetadata, lcevcDec) {
+  constructor(video, textDisplayer, playerInterface, lcevcDec) {
     /** @private {HTMLMediaElement} */
     this.video_ = video;
 
+    /** @private {?shaka.media.MediaSourceEngine.PlayerInterface} */
+    this.playerInterface_ = playerInterface;
+
     /** @private {?shaka.extern.MediaSourceConfiguration} */
     this.config_ = null;
 
@@ -87,12 +89,6 @@ shaka.media.MediaSourceEngine = class {
     /** @private {boolean} */
     this.segmentRelativeVttTiming_ = false;
 
-    const onMetadataNoOp = (metadata, timestampOffset, segmentEnd) => {};
-
-    /** @private {!function(!Array.<shaka.extern.ID3Metadata>,
-                    number, ?number)} */
-    this.onMetadata_ = onMetadata || onMetadataNoOp;
-
     /** @private {?shaka.lcevc.Dec} */
     this.lcevcDec_ = lcevcDec || null;
 
@@ -438,6 +434,7 @@ shaka.media.MediaSourceEngine = class {
     this.lcevcDec_ = null;
 
     this.tsParser_ = null;
+    this.playerInterface_ = null;
   }
 
   /**
@@ -813,7 +810,8 @@ shaka.media.MediaSourceEngine = class {
           dts: reference.startTime,
           pts: reference.startTime,
         };
-        this.onMetadata_([metadata], /* offset= */ 0, reference.endTime);
+        this.playerInterface_.onMetadata(
+            [metadata], /* offset= */ 0, reference.endTime);
       }
     } else if (mimeType.includes('/mp4') &&
         reference && reference.timestampOffset == 0 &&
@@ -855,7 +853,7 @@ shaka.media.MediaSourceEngine = class {
       }
       const metadata = tsParser.getMetadata();
       if (metadata.length) {
-        this.onMetadata_(metadata, timestampOffset,
+        this.playerInterface_.onMetadata(metadata, timestampOffset,
             reference ? reference.endTime : null);
       }
     }
@@ -1719,8 +1717,11 @@ shaka.media.MediaSourceEngine = class {
    * @private
    */
   workAroundBrokenPlatforms_(segment, startTime, contentType, uri) {
+    const Platform = shaka.util.Platform;
+
     const isInitSegment = startTime == null;
     const encryptionExpected = this.expectedEncryption_[contentType];
+    const keySystem = this.playerInterface_.getKeySystem();
 
     // If:
     //   1. the configuration tells to insert fake encryption,
@@ -1735,7 +1736,7 @@ shaka.media.MediaSourceEngine = class {
     if (this.config_.insertFakeEncryptionInInit &&
         isInitSegment &&
         encryptionExpected &&
-        shaka.util.Platform.requiresEncryptionInfoInAllInitSegments() &&
+        Platform.requiresEncryptionInfoInAllInitSegments(keySystem) &&
         shaka.util.MimeUtils.getContainerType(
             this.sourceBufferTypes_[contentType]) == 'mp4') {
       shaka.log.debug('Forcing fake encryption information in init segment.');
@@ -2151,3 +2152,19 @@ shaka.media.MediaSourceEngine.SourceBufferMode_ = {
   SEQUENCE: 'sequence',
   SEGMENTS: 'segments',
 };
+
+
+/**
+ * @typedef {{
+ *   getKeySystem: function():?string,
+ *   onMetadata: function(!Array<shaka.extern.ID3Metadata>, number, ?number)
+ * }}
+ *
+ * @summary Player interface
+ * @property {function():?string} getKeySystem
+ *   Gets currently used key system or null if not used.
+ * @property {function(
+ *     !Array<shaka.extern.ID3Metadata>, number, ?number)} onMetadata
+ *   Callback to use when metadata arrives.
+ */
+shaka.media.MediaSourceEngine.PlayerInterface;
diff --git a/lib/player.js b/lib/player.js
@@ -2305,8 +2305,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     const mediaSourceEngine = this.createMediaSourceEngine(
         this.video_,
         textDisplayer,
-        (metadata, offset, endTime) => {
-          this.processTimedMetadataMediaSrc_(metadata, offset, endTime);
+        {
+          getKeySystem: () => this.keySystem(),
+          onMetadata: (metadata, offset, endTime) => {
+            this.processTimedMetadataMediaSrc_(metadata, offset, endTime);
+          },
         },
         this.lcevcDec_);
     mediaSourceEngine.configure(this.config_.mediaSource);
@@ -3440,17 +3443,17 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    *
    * @param {!HTMLMediaElement} mediaElement
    * @param {!shaka.extern.TextDisplayer} textDisplayer
-   * @param {!function(!Array.<shaka.extern.ID3Metadata>, number, ?number)}
-   *  onMetadata
+   * @param {!shaka.media.MediaSourceEngine.PlayerInterface} playerInterface
    * @param {shaka.lcevc.Dec} lcevcDec
    *
    * @return {!shaka.media.MediaSourceEngine}
    */
-  createMediaSourceEngine(mediaElement, textDisplayer, onMetadata, lcevcDec) {
+  createMediaSourceEngine(mediaElement, textDisplayer, playerInterface,
+      lcevcDec) {
     return new shaka.media.MediaSourceEngine(
         mediaElement,
         textDisplayer,
-        onMetadata,
+        playerInterface,
         lcevcDec);
   }
 

diff --git a/lib/util/drm_utils.js b/lib/util/drm_utils.js
@@ -0,0 +1,34 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.util.DrmUtils');
+
+
+shaka.util.DrmUtils = class {
+  /**
+   * @param {?string} keySystem
+   * @return {boolean}
+   */
+  static isPlayReadyKeySystem(keySystem) {
+    if (keySystem) {
+      return !!keySystem.match(/^com\.(microsoft|chromecast)\.playready/);
+    }
+
+    return false;
+  }
+
+  /**
+   * @param {?string} keySystem
+   * @return {boolean}
+   */
+  static isFairPlayKeySystem(keySystem) {
+    if (keySystem) {
+      return !!keySystem.match(/^com\.apple\.fps/);
+    }
+
+    return false;
+  }
+};
diff --git a/lib/util/platform.js b/lib/util/platform.js
@@ -7,6 +7,7 @@
 goog.provide('shaka.util.Platform');
 
 goog.require('shaka.log');
+goog.require('shaka.util.DrmUtils');
 goog.require('shaka.util.Timer');
 
 
@@ -590,12 +591,15 @@ shaka.util.Platform = class {
    * around a lack of such info by inserting fake encryption information into
    * initialization segments.
    *
+   * @param {?string} keySystem
    * @return {boolean}
    * @see https://github.com/shaka-project/shaka-player/issues/2759
    */
-  static requiresEncryptionInfoInAllInitSegments() {
+  static requiresEncryptionInfoInAllInitSegments(keySystem) {
     const Platform = shaka.util.Platform;
-    return Platform.isTizen() || Platform.isXboxOne() || Platform.isOrange();
+    const isPlayReady = shaka.util.DrmUtils.isPlayReadyKeySystem(keySystem);
+    return Platform.isTizen() || Platform.isXboxOne() || Platform.isOrange() ||
+      (Platform.isEdge() && Platform.isWindows() && isPlayReady);
   }
 
   /**

diff --git a/test/cast/cast_utils_unit.js b/test/cast/cast_utils_unit.js
@@ -221,7 +221,11 @@ describe('CastUtils', () => {
 
         mediaSourceEngine = new shaka.media.MediaSourceEngine(
             video,
-            new shaka.test.FakeTextDisplayer());
+            new shaka.test.FakeTextDisplayer(),
+            {
+              getKeySystem: () => null,
+              onMetadata: () => {},
+            });
         const config =
             shaka.util.PlayerConfiguration.createDefault().mediaSource;
         mediaSourceEngine.configure(config);

diff --git a/test/media/content_workarounds_unit.js b/test/media/content_workarounds_unit.js
@@ -5,6 +5,8 @@
  */
 
 describe('ContentWorkarounds', () => {
+  const Util = shaka.test.Util;
+
   const encryptionBoxes = {
     'encv': ['hev1', 'hvc1', 'avc1', 'avc3'],
     'enca': ['ac-3', 'ec-3', 'ac-4', 'mp4a'],
@@ -34,10 +36,42 @@ describe('ContentWorkarounds', () => {
         const spy = jasmine.createSpy('boxCallback');
         new shaka.util.Mp4Parser()
             .fullBox('stsd', shaka.util.Mp4Parser.sampleDescription)
-            .box(encryptionBox, /** @type {!Function} */ (spy))
+            .box(encryptionBox, Util.spyFunc(spy))
             .parse(faked);
         expect(spy).toHaveBeenCalled();
       });
     }
   }
+
+  it('faked encryption on Edge returns two init segments', () => {
+    spyOn(shaka.util.Platform, 'isEdge').and.returnValue(true);
+    spyOn(shaka.util.Platform, 'isWindows').and.returnValue(true);
+
+    const unencrypted = new Uint8Array([
+      0x00, 0x00, 0x00, 0x20, // size
+      0x73, 0x74, 0x73, 0x64, // stsd
+      0x01,                   // version
+      0x12, 0x34, 0x56,       // flags
+      0x00, 0x00, 0x00, 0x01, // count
+      0x00, 0x00, 0x00, 0x10, // size
+      0x61, 0x76, 0x63, 0x31, // avc1
+      0x01,                   // version
+      0x12, 0x34, 0x56,       // flags
+      0x00, 0x11, 0x22, 0x33, // payload
+    ]);
+
+    const faked =
+        shaka.media.ContentWorkarounds.fakeEncryption(unencrypted, null);
+    const stsdSpy = jasmine.createSpy('stsdCallback').and
+        .callFake(shaka.util.Mp4Parser.sampleDescription);
+    const encvSpy = jasmine.createSpy('encvCallback');
+    new shaka.util.Mp4Parser()
+        .fullBox('stsd', Util.spyFunc(stsdSpy))
+        .box('encv', Util.spyFunc(encvSpy))
+        .parse(faked);
+    // 2 init segments
+    expect(stsdSpy).toHaveBeenCalledTimes(2);
+    // but only one encrypted
+    expect(encvSpy).toHaveBeenCalledTimes(1);
+  });
 });
diff --git a/test/media/drm_engine_integration.js b/test/media/drm_engine_integration.js
@@ -114,7 +114,11 @@ describe('DrmEngine', () => {
 
     mediaSourceEngine = new shaka.media.MediaSourceEngine(
         video,
-        new shaka.test.FakeTextDisplayer());
+        new shaka.test.FakeTextDisplayer(),
+        {
+          getKeySystem: () => null,
+          onMetadata: () => {},
+        });
     const mediaSourceConfig =
         shaka.util.PlayerConfiguration.createDefault().mediaSource;
     mediaSourceEngine.configure(mediaSourceConfig);