MSE-in-Workers: srcObject part 4: media element use of handle

Adds ability to attach a dedicated-worker-owned MediaSource to an
HTMLMediaElement using a MediaSourceHandle for that MediaSource instance
that the app has posted over to the main thread and assigned to the
srcObject attribute on the media element. Previously, only a MediaStream
could be set as a srcObject in Chrome. This change adds the relevant
MediaProvider IDL union type and makes it the type of that srcObject
attribute. For historical reasons, the srcObject attribute of an
HTMLMediaElement is implemented in modules as a partial interface; this
change updates that partial interface as well as the media element
implementation to enable the scenario without regressing existing
ability to use MediaStream as media element's srcObject.

The attachment of the worker MediaSource using the handle does not
involve lookup into a URL registry; rather, an internal blob URL is
built into the handle when it was retrieved from the MediaSource and
that URL is used to satisfy existing media element load safety checks as
well as provide the underlying WebMediaPlayer a URL that it can use when
logging to devtools.

Later changes will add further test updates and remove the previous
ability to attach a worker MediaSource using a registered object URL.

References:
Full prototype CL:
    https://chromium-review.googlesource.com/c/chromium/src/+/3515334
MSE spec issue:
    w3c/media-source#175
MSE spec feature updates switching from worker MSE attachment via
  object URL to attachment via srcObject MediaSourceHandle:
  * w3c/media-source#305
  * further clarifications in discussion at
    w3c/media-source#306 (comment)

crbug.com/506273 is somewhat related, but not fixed by this change (it
refers to directly setting MediaSource on Window context to the media
element's srcObject attribute.) This change sequence is specific to
enabling MSE-in-Worker attachment via srcObject using a handle object.

BUG=878133,506273

Change-Id: I86cddd87bafae1c7cbae9e94ea4614418067012f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3688740
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Reviewed-by: Elad Alon <eladalon@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Commit-Queue: Matthew Wolenetz <wolenetz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1012202}
NOKEYCHECK=True
GitOrigin-RevId: 756af69b67c9e40e0a0c92cb6c0a911c96ea6835

blink/renderer/bindings/generated_in_modules.gni

@@ -2659,6 +2659,8 @@ generated_union_sources_in_modules = [
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_idbcursor_idbindex_idbobjectstore.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_idbindex_idbobjectstore.cc",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_idbindex_idbobjectstore.h",
+  "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_mediasourcehandle_mediastream.cc",
+  "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_mediasourcehandle_mediastream.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_mediastreamtrack_string.cc",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_mediastreamtrack_string.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_union_path2d_string.cc",

blink/renderer/core/html/media/html_media_element.cc

@@ -87,6 +87,7 @@
 #include "third_party/blink/renderer/core/html/media/media_error.h"
 #include "third_party/blink/renderer/core/html/media/media_fragment_uri_parser.h"
 #include "third_party/blink/renderer/core/html/media/media_source_attachment.h"
+#include "third_party/blink/renderer/core/html/media/media_source_handle.h"
 #include "third_party/blink/renderer/core/html/media/media_source_tracer.h"
 #include "third_party/blink/renderer/core/html/time_ranges.h"
 #include "third_party/blink/renderer/core/html/track/audio_track.h"
@@ -813,7 +814,8 @@ Node::InsertionNotificationRequest HTMLMediaElement::InsertedInto(
   HTMLElement::InsertedInto(insertion_point);
   if (insertion_point.isConnected()) {
     UseCounter::Count(GetDocument(), WebFeature::kHTMLMediaElementInDocument);
-    if ((!FastGetAttribute(html_names::kSrcAttr).IsEmpty() || src_object_) &&
+    if ((!FastGetAttribute(html_names::kSrcAttr).IsEmpty() ||
+         src_object_stream_descriptor_ || src_object_media_source_handle_) &&
         network_state_ == kNetworkEmpty) {
       ignore_preload_none_ = false;
       InvokeLoadAlgorithm();
@@ -898,12 +900,40 @@ void HTMLMediaElement::SetSrc(const AtomicString& url) {
   setAttribute(html_names::kSrcAttr, url);
 }
 
-void HTMLMediaElement::SetSrcObject(MediaStreamDescriptor* src_object) {
-  DVLOG(1) << "setSrcObject(" << *this << ")";
-  src_object_ = src_object;
+void HTMLMediaElement::SetSrcObjectVariant(
+    SrcObjectVariant src_object_variant) {
+  DVLOG(1) << __func__ << "(" << *this << ")";
+  src_object_stream_descriptor_ = nullptr;
+  src_object_media_source_handle_ = nullptr;
+  if (auto** desc = absl::get_if<MediaStreamDescriptor*>(&src_object_variant)) {
+    src_object_stream_descriptor_ = *desc;
+  } else if (auto** handle =
+                 absl::get_if<MediaSourceHandle*>(&src_object_variant)) {
+    src_object_media_source_handle_ = *handle;
+  }
+
+  DVLOG(2) << __func__
+           << ": stream_descriptor=" << src_object_stream_descriptor_
+           << ", media_source_handle=" << src_object_media_source_handle_;
+
   InvokeLoadAlgorithm();
 }
 
+HTMLMediaElement::SrcObjectVariant HTMLMediaElement::GetSrcObjectVariant()
+    const {
+  DVLOG(1) << __func__ << "(" << *this << ")"
+           << ": stream_descriptor=" << src_object_stream_descriptor_
+           << ", media_source_handle=" << src_object_media_source_handle_;
+
+  // At most one is set.
+  DCHECK(!(src_object_stream_descriptor_ && src_object_media_source_handle_));
+
+  if (src_object_media_source_handle_)
+    return SrcObjectVariant(src_object_media_source_handle_.Get());
+
+  return SrcObjectVariant(src_object_stream_descriptor_.Get());
+}
+
 HTMLMediaElement::NetworkState HTMLMediaElement::getNetworkState() const {
   return network_state_;
 }
@@ -1141,7 +1171,7 @@ void HTMLMediaElement::SelectMediaResource() {
 
   // 6 - If the media element has an assigned media provider object, then let
   //     mode be object.
-  if (src_object_) {
+  if (src_object_stream_descriptor_ || src_object_media_source_handle_) {
     mode = kObject;
   } else if (FastHasAttribute(html_names::kSrcAttr)) {
     // Otherwise, if the media element has no assigned media provider object
@@ -1209,12 +1239,40 @@ void HTMLMediaElement::SelectMediaResource() {
 }
 
 void HTMLMediaElement::LoadSourceFromObject() {
-  DCHECK(src_object_);
+  DCHECK(src_object_stream_descriptor_ || src_object_media_source_handle_);
   load_state_ = kLoadingFromSrcObject;
 
+  if (src_object_media_source_handle_) {
+    DCHECK(!src_object_stream_descriptor_);
+
+    // Retrieve the internal blob URL from the handle that was created in the
+    // context where the referenced MediaSource is owned, for the purposes of
+    // using existing security and logging logic for loading media from a
+    // MediaSource with a blob URL.
+    const String media_source_handle_url_ =
+        src_object_media_source_handle_->GetInternalBlobURL();
+    DCHECK(!media_source_handle_url_.IsEmpty());
+
+    KURL media_url = GetDocument().CompleteURL(media_source_handle_url_);
+    if (!IsSafeToLoadURL(media_url, kComplain)) {
+      MediaLoadingFailed(
+          WebMediaPlayer::kNetworkStateFormatError,
+          BuildElementErrorMessage(
+              "Media load from MediaSourceHandle rejected by safety check"));
+      return;
+    }
+
+    // No type is available when loading from a MediaSourceHandle, via
+    // srcObject, even with an internal MediaSource blob URL.
+    LoadResource(WebMediaPlayerSource(WebURL(media_url)), String());
+    return;
+  }
+
   // No type is available when the resource comes from the 'srcObject'
   // attribute.
-  LoadResource(WebMediaPlayerSource(WebMediaStream(src_object_)), String());
+  LoadResource(
+      WebMediaPlayerSource(WebMediaStream(src_object_stream_descriptor_)),
+      String());
 }
 
 void HTMLMediaElement::LoadSourceFromAttribute() {
@@ -1280,24 +1338,39 @@ void HTMLMediaElement::LoadResource(const WebMediaPlayerSource& source,
   // The resource fetch algorithm
   SetNetworkState(kNetworkLoading);
 
-  // Set current_src_ *before* changing to the cache url, the fact that we are
+  // Set |current_src_| *before* changing to the cache url, the fact that we are
   // loading from the app cache is an internal detail not exposed through the
-  // media element API.
-  current_src_ = url;
+  // media element API. If loading from an internal MediaSourceHandle object
+  // URL, then do not expose that URL to app, but instead hold it for use later
+  // in StartPlayerLoad and elsewhere (for origin, security etc checks normally
+  // done on |current_src_|.)
+  if (src_object_media_source_handle_) {
+    DCHECK(!url.IsEmpty());
+    current_src_.SetSource(url,
+                           SourceMetadata::SourceVisibility::kInvisibleToApp);
+  } else {
+    current_src_.SetSource(url,
+                           SourceMetadata::SourceVisibility::kVisibleToApp);
+  }
 
   // Default this to empty, so that we use |current_src_| unless the player
   // provides one later.
   current_src_after_redirects_ = KURL();
 
   if (audio_source_node_)
-    audio_source_node_->OnCurrentSrcChanged(current_src_);
+    audio_source_node_->OnCurrentSrcChanged(current_src_.GetSourceIfVisible());
 
   // Update remote playback client with the new src and consider it incompatible
   // until proved otherwise.
-  RemotePlaybackCompatibilityChanged(current_src_, false);
+  RemotePlaybackCompatibilityChanged(current_src_.GetSourceIfVisible(), false);
 
-  DVLOG(3) << "loadResource(" << *this << ") - current_src_ -> "
-           << UrlForLoggingMedia(current_src_);
+  DVLOG(3) << "loadResource(" << *this << ") - current src if visible="
+           << UrlForLoggingMedia(current_src_.GetSourceIfVisible())
+           << ", current src =" << UrlForLoggingMedia(current_src_.GetSource())
+           << ", src_object_media_source_handle_="
+           << src_object_media_source_handle_
+           << ", src_object_stream_descriptor_="
+           << src_object_stream_descriptor_;
 
   StartProgressEventTimer();
 
@@ -1309,8 +1382,13 @@ void HTMLMediaElement::LoadResource(const WebMediaPlayerSource& source,
 
   bool attempt_load = true;
 
-  media_source_attachment_ =
-      MediaSourceAttachment::LookupMediaSource(url.GetString());
+  if (src_object_media_source_handle_) {
+    media_source_attachment_ = src_object_media_source_handle_->GetAttachment();
+    DCHECK(media_source_attachment_);
+  } else {
+    media_source_attachment_ =
+        MediaSourceAttachment::LookupMediaSource(url.GetString());
+  }
   if (media_source_attachment_) {
     bool start_result = false;
     media_source_tracer_ =
@@ -1322,9 +1400,12 @@ void HTMLMediaElement::LoadResource(const WebMediaPlayerSource& source,
       // attachment. This can help reduce memory bloat later if the app does not
       // revoke the object URL explicitly and the object URL was the only
       // remaining strong reference to an attached HTMLMediaElement+MediaSource
-      // cycle of objects that could otherwise be garbage-collectable.
+      // cycle of objects that could otherwise be garbage-collectable. Don't
+      // auto-revoke the internal, unregistered, object URL used to attach via
+      // srcObject with a MediaSourceHandle, though.
       if (base::FeatureList::IsEnabled(
-              media::kRevokeMediaSourceObjectURLOnAttach)) {
+              media::kRevokeMediaSourceObjectURLOnAttach) &&
+          !src_object_media_source_handle_) {
         URLFileAPI::revokeObjectURL(GetExecutionContext(), url.GetString());
       }
     } else {
@@ -1370,8 +1451,15 @@ void HTMLMediaElement::StartPlayerLoad() {
   DCHECK(!web_media_player_);
 
   WebMediaPlayerSource source;
-  if (src_object_) {
-    source = WebMediaPlayerSource(WebMediaStream(src_object_));
+  if (src_object_stream_descriptor_) {
+    source =
+        WebMediaPlayerSource(WebMediaStream(src_object_stream_descriptor_));
+  } else if (src_object_media_source_handle_) {
+    DCHECK(current_src_.GetSourceIfVisible().IsEmpty());
+    const KURL& internal_url = current_src_.GetSource();
+    DCHECK(!internal_url.IsEmpty());
+
+    source = WebMediaPlayerSource(WebURL(internal_url));
   } else {
     // Filter out user:pass as those two URL components aren't
     // considered for media resource fetches (including for the CORS
@@ -1386,7 +1474,7 @@ void HTMLMediaElement::StartPlayerLoad() {
     // 'authentication flag' to control how user:pass embedded in a
     // media resource URL should be treated, then update the handling
     // here to match.
-    KURL request_url = current_src_;
+    KURL request_url = current_src_.GetSourceIfVisible();
     if (!request_url.User().IsEmpty())
       request_url.SetUser(String());
     if (!request_url.Pass().IsEmpty())
@@ -1545,9 +1633,9 @@ void HTMLMediaElement::DeferredLoadTimerFired(TimerBase*) {
 
 WebMediaPlayer::LoadType HTMLMediaElement::GetLoadType() const {
   if (media_source_attachment_)
-    return WebMediaPlayer::kLoadTypeMediaSource;
+    return WebMediaPlayer::kLoadTypeMediaSource;  // Either via src or srcObject
 
-  if (src_object_)
+  if (src_object_stream_descriptor_)
     return WebMediaPlayer::kLoadTypeMediaStream;
 
   return WebMediaPlayer::kLoadTypeURL;
@@ -1845,7 +1933,9 @@ void HTMLMediaElement::MediaLoadingFailed(WebMediaPlayer::NetworkState error,
         MakeGarbageCollected<MediaError>(MediaError::kMediaErrDecode, message));
   } else if ((error == WebMediaPlayer::kNetworkStateFormatError ||
               error == WebMediaPlayer::kNetworkStateNetworkError) &&
-             load_state_ == kLoadingFromSrcAttr) {
+             (load_state_ == kLoadingFromSrcAttr ||
+              (load_state_ == kLoadingFromSrcObject &&
+               src_object_media_source_handle_))) {
     if (message.IsEmpty()) {
       // Generate a more meaningful error message to differentiate the two types
       // of MEDIA_SRC_ERR_NOT_SUPPORTED.
@@ -1959,20 +2049,21 @@ void HTMLMediaElement::SetReadyState(ReadyState state) {
     // skips notification of insecure content. Ensure we always notify the
     // MixedContentChecker of what happened, even if the load was skipped.
     if (LocalFrame* frame = GetDocument().GetFrame()) {
+      const KURL& current_src_for_check = current_src_.GetSource();
       // We don't care about the return value here. The MixedContentChecker will
       // internally notify for insecure content if it needs to regardless of
       // what the return value ends up being for this call.
       MixedContentChecker::ShouldBlockFetch(
           frame,
           HasVideo() ? mojom::blink::RequestContextType::VIDEO
                      : mojom::blink::RequestContextType::AUDIO,
-          current_src_,
+          current_src_for_check,
           // Strictly speaking, this check is an approximation; a request could
           // have have redirected back to its original URL, for example.
           // However, the redirect status is only used to prevent leaking
           // information cross-origin via CSP reports, so comparing URLs is
           // sufficient for that purpose.
-          current_src_after_redirects_ == current_src_
+          current_src_after_redirects_ == current_src_for_check
               ? ResourceRequest::RedirectStatus::kNoRedirect
               : ResourceRequest::RedirectStatus::kFollowedRedirect,
           current_src_after_redirects_, /* devtools_id= */ absl::nullopt,
@@ -2022,7 +2113,7 @@ void HTMLMediaElement::SetReadyState(ReadyState state) {
   if (ready_state_ >= kHaveMetadata && old_state < kHaveMetadata) {
     CreatePlaceholderTracksIfNecessary();
 
-    MediaFragmentURIParser fragment_parser(current_src_);
+    MediaFragmentURIParser fragment_parser(current_src_.GetSource());
     fragment_end_time_ = fragment_parser.EndTime();
 
     // Set the current playback position and the official playback position to
@@ -4304,7 +4395,8 @@ void HTMLMediaElement::Trace(Visitor* visitor) const {
   visitor->Trace(play_promise_resolve_list_);
   visitor->Trace(play_promise_reject_list_);
   visitor->Trace(audio_source_provider_);
-  visitor->Trace(src_object_);
+  visitor->Trace(src_object_stream_descriptor_);
+  visitor->Trace(src_object_media_source_handle_);
   visitor->Trace(autoplay_policy_);
   visitor->Trace(media_controls_);
   visitor->Trace(controls_list_);

blink/renderer/core/html/media/html_media_element.h

@@ -33,6 +33,7 @@
 #include "base/timer/elapsed_timer.h"
 #include "media/mojo/mojom/media_player.mojom-blink.h"
 #include "third_party/abseil-cpp/absl/types/optional.h"
+#include "third_party/abseil-cpp/absl/types/variant.h"
 #include "third_party/blink/public/common/media/display_type.h"
 #include "third_party/blink/public/platform/web_media_player_client.h"
 #include "third_party/blink/public/platform/webaudiosourceprovider_impl.h"
@@ -85,6 +86,7 @@ class HTMLSourceElement;
 class HTMLTrackElement;
 class MediaError;
 class MediaSourceAttachment;
+class MediaSourceHandle;
 class MediaSourceTracer;
 class MediaStreamDescriptor;
 class ScriptPromiseResolver;
@@ -177,7 +179,7 @@ class CORE_EXPORT HTMLMediaElement
 
   // network state
   void SetSrc(const AtomicString&);
-  const KURL& currentSrc() const { return current_src_; }
+  const KURL& currentSrc() const { return current_src_.GetSourceIfVisible(); }
 
   // Return the URL to be used for downloading the media.
   const KURL& downloadURL() const {
@@ -189,8 +191,10 @@ class CORE_EXPORT HTMLMediaElement
     return current_src_after_redirects_;
   }
 
-  void SetSrcObject(MediaStreamDescriptor*);
-  MediaStreamDescriptor* GetSrcObject() const { return src_object_.Get(); }
+  using SrcObjectVariant =
+      absl::variant<MediaStreamDescriptor*, MediaSourceHandle*>;
+  void SetSrcObjectVariant(SrcObjectVariant src_object_variant);
+  SrcObjectVariant GetSrcObjectVariant() const;
 
   enum NetworkState {
     kNetworkEmpty,
@@ -452,6 +456,30 @@ class CORE_EXPORT HTMLMediaElement
   friend class PictureInPictureControllerTest;
   friend class VideoWakeLockTest;
 
+  class SourceMetadata {
+    DISALLOW_NEW();
+
+   public:
+    enum class SourceVisibility { kVisibleToApp, kInvisibleToApp };
+    SourceMetadata() = default;
+    void SetSource(const KURL& src, SourceVisibility visibility) {
+      src_ = src;
+      invisible_to_app_ = visibility == SourceVisibility::kInvisibleToApp;
+    }
+    const KURL& GetSourceIfVisible() const {
+      return invisible_to_app_ ? NullURL() : src_;
+    }
+    const KURL& GetSource() const { return src_; }
+
+   private:
+    KURL src_;
+
+    // If true, then |current_src| is used only for internal loading and safety
+    // checks, and for logging that is not visible to apps, either. For example,
+    // when loading from a MediaSourceHandle as srcObject, this would be true.
+    bool invisible_to_app_ = false;
+  };
+
   bool HasPendingActivityInternal() const;
 
   void ResetMediaPlayerAndMediaSource();
@@ -700,9 +728,12 @@ class CORE_EXPORT HTMLMediaElement
   NetworkState network_state_;
   ReadyState ready_state_;
   ReadyState ready_state_maximum_;
-  KURL current_src_;
+
+  SourceMetadata current_src_;
   KURL current_src_after_redirects_;
-  Member<MediaStreamDescriptor> src_object_;
+
+  Member<MediaStreamDescriptor> src_object_stream_descriptor_;
+  Member<MediaSourceHandle> src_object_media_source_handle_;
 
   // To prevent potential regression when extended by the MSE API, do not set
   // |error_| outside of constructor and SetError().

blink/renderer/core/html/media/html_media_element_test.cc

@@ -296,7 +296,8 @@ class HTMLMediaElementTest : public testing::TestWithParam<MediaTestParam> {
   HTMLMediaElement* Media() const { return media_.Get(); }
   void SetCurrentSrc(const AtomicString& src) {
     KURL url(src);
-    Media()->current_src_ = url;
+    Media()->current_src_.SetSource(
+        url, HTMLMediaElement::SourceMetadata::SourceVisibility::kVisibleToApp);
   }
 
   MockWebMediaPlayer* MockMediaPlayer() { return media_player_; }