Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MacOS] Add support for creating and presenting platform views. #22905

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -1094,6 +1094,13 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterMouse
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterOpenGLRenderer.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterOpenGLRenderer.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterOpenGLRendererTest.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewControllerTest.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController_Internal.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewMock.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewMock.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViews.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterRenderer.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterResizableBackingStoreProvider.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterResizableBackingStoreProvider.mm
Expand Down
6 changes: 6 additions & 0 deletions shell/platform/darwin/macos/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ source_set("flutter_framework_source") {
"framework/Source/FlutterMouseCursorPlugin.mm",
"framework/Source/FlutterOpenGLRenderer.h",
"framework/Source/FlutterOpenGLRenderer.mm",
"framework/Source/FlutterPlatformViewController.mm",
"framework/Source/FlutterPlatformViewController_Internal.h",
"framework/Source/FlutterPlatformViews.h",
"framework/Source/FlutterRenderer.h",
"framework/Source/FlutterResizableBackingStoreProvider.h",
"framework/Source/FlutterResizableBackingStoreProvider.mm",
Expand Down Expand Up @@ -147,6 +150,9 @@ executable("flutter_desktop_darwin_unittests") {
sources = [
"framework/Source/FlutterEngineTest.mm",
"framework/Source/FlutterGLCompositorUnittests.mm",
"framework/Source/FlutterPlatformViewControllerTest.mm",
"framework/Source/FlutterPlatformViewMock.h",
"framework/Source/FlutterPlatformViewMock.mm",
"framework/Source/FlutterViewControllerTest.mm",
"framework/Source/FlutterViewControllerTestUtils.h",
"framework/Source/FlutterViewControllerTestUtils.mm",
Expand Down
94 changes: 90 additions & 4 deletions shell/platform/darwin/macos/framework/Source/FlutterEngine.mm
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterOpenGLRenderer.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController_Internal.h"
#import "flutter/shell/platform/embedder/embedder.h"

/**
Expand Down Expand Up @@ -57,6 +58,16 @@ - (void)postMainThreadTask:(FlutterTask)task targetTimeInNanoseconds:(uint64_t)t
*/
- (void)loadAOTData:(NSString*)assetsDir;

/**
* Creates and returns a FlutterCompositor* to be used by the embedder.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: s/used by the embedder/provided to the embedder API/. The embedder is the runner (in our case), not the flutter engine.

*/
- (FlutterCompositor*)createFlutterCompositor;

/**
* Create a platform view channel and setup a method call handler.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: Creates [...] and sets up [...]

*/
- (void)setupPlatformViewChannel;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: setUpPlatformViewChannel. "Setup" is a noun; the verb is "set up".


@end

#pragma mark -
Expand Down Expand Up @@ -141,6 +152,13 @@ @implementation FlutterEngine {

// FlutterCompositor is copied and used in embedder.cc.
FlutterCompositor _compositor;

// A method channel for platform view functionality.
FlutterMethodChannel* _platformViewsChannel;

// Used to support creation and deletion of platform views and
// registering platform view factories.
FlutterPlatformViewController* _platformViewController;
}

- (instancetype)initWithName:(NSString*)labelPrefix project:(FlutterDartProject*)project {
Expand Down Expand Up @@ -248,6 +266,9 @@ - (BOOL)runWithEntrypoint:(NSString*)entrypoint {
flutterArguments.aot_data = _aotData;
}

[self setupPlatformViewChannel];
_platformViewController = [[FlutterPlatformViewController alloc] init];

flutterArguments.compositor = [self createFlutterCompositor];

FlutterRendererConfig rendererConfig = [_renderer createRendererConfig];
Expand Down Expand Up @@ -428,6 +449,10 @@ - (void)sendPointerEvent:(const FlutterPointerEvent&)event {
_embedderAPI.SendPointerEvent(_engine, &event, 1);
}

- (FlutterPlatformViewController*)platformViewController {
return _platformViewController;
}

#pragma mark - Private methods

- (void)sendUserLocales {
Expand Down Expand Up @@ -504,6 +529,67 @@ - (void)shutDownEngine {
_engine = nullptr;
}

- (FlutterCompositor*)createFlutterCompositor {
RichardJCai marked this conversation as resolved.
Show resolved Hide resolved
// TODO(richardjcai): Add support for creating a FlutterGLCompositor
// with a nil _viewController for headless engines.
// https://github.com/flutter/flutter/issues/71606
if (_viewController == nullptr) {
return nullptr;
}

[_mainOpenGLContext makeCurrentContext];

_macOSGLCompositor =
std::make_unique<flutter::FlutterGLCompositor>(_viewController, _platformViewController);

_compositor = {};
_compositor.struct_size = sizeof(FlutterCompositor);
_compositor.user_data = _macOSGLCompositor.get();

_compositor.create_backing_store_callback = [](const FlutterBackingStoreConfig* config, //
FlutterBackingStore* backing_store_out, //
void* user_data //
) {
return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->CreateBackingStore(
config, backing_store_out);
};

_compositor.collect_backing_store_callback = [](const FlutterBackingStore* backing_store, //
void* user_data //
) {
return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->CollectBackingStore(
backing_store);
};

_compositor.present_layers_callback = [](const FlutterLayer** layers, //
size_t layers_count, //
void* user_data //
) {
return reinterpret_cast<flutter::FlutterGLCompositor*>(user_data)->Present(layers,
layers_count);
};

__weak FlutterEngine* weakSelf = self;
_macOSGLCompositor->SetPresentCallback(
[weakSelf]() { return [weakSelf engineCallbackOnPresent]; });

_compositor.avoid_backing_store_cache = true;

return &_compositor;
}

- (void)setupPlatformViewChannel {
_platformViewsChannel =
[FlutterMethodChannel methodChannelWithName:@"flutter/platform_views"
binaryMessenger:self.binaryMessenger
codec:[FlutterStandardMethodCodec sharedInstance]];

__weak FlutterEngine* weakSelf = self;
[_platformViewsChannel setMethodCallHandler:^(FlutterMethodCall* call, FlutterResult result) {
[[weakSelf platformViewController] handleMethodCall:call result:result];
}];
}

#pragma mark - FlutterBinaryMessenger

- (void)sendOnChannel:(nonnull NSString*)channel message:(nullable NSData*)message {
Expand Down Expand Up @@ -610,11 +696,11 @@ - (BOOL)unregisterTextureWithID:(int64_t)textureID {
- (void)postMainThreadTask:(FlutterTask)task targetTimeInNanoseconds:(uint64_t)targetTime {
const auto engine_time = _embedderAPI.GetCurrentTime();

__weak FlutterEngine* weak_self = self;
__weak FlutterEngine* weakSelf = self;
auto worker = ^{
FlutterEngine* strong_self = weak_self;
if (strong_self && strong_self->_engine) {
auto result = _embedderAPI.RunTask(strong_self->_engine, &task);
FlutterEngine* strongSelf = weakSelf;
if (strongSelf && strongSelf->_engine) {
auto result = _embedderAPI.RunTask(strongSelf->_engine, &task);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reference to _embedderAPI here actually breaks the whole weak/strong dance; this looks like a bug I probably introduced while converting to the embedder API via find/replace. Since you're touching this code already, could you extract this whole block body to a helper method to avoid that class of issues in the first place? (As a method you don't need the strongSelf part, or to be careful about ivar access; you can just call [weakSelf whateverHelperMethod] and then code as normal inside it due to the retain during call).

if (result != kSuccess) {
NSLog(@"Could not post a task to the Flutter engine.");
}
Expand Down
12 changes: 12 additions & 0 deletions shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include <map>

#include "flutter/fml/macros.h"
#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.h"
#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController_Internal.h"
#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterSurfaceManager.h"
#include "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h"
#include "flutter/shell/platform/embedder/embedder.h"
Expand Down Expand Up @@ -50,6 +52,7 @@ class FlutterGLCompositor {

private:
const FlutterViewController* view_controller_;
const FlutterPlatformViewController* platform_view_controller_;
const NSOpenGLContext* open_gl_context_;
PresentCallback present_callback_;

Expand All @@ -67,6 +70,15 @@ class FlutterGLCompositor {
// created for the frame.
bool frame_started_ = false;

// Update the backing CALayer using the backing store's specifications.
void PresentBackingStoreContent(
FlutterBackingStoreData* flutter_backing_store_data,
size_t layer_position);

// Add the Platform View's content to the FlutterView at depth
// layer_position.
void PresentPlatformView(const FlutterLayer* layer, size_t layer_position);

// Set frame_started_ to true and reset all layer state.
void StartFrame();

Expand Down
68 changes: 50 additions & 18 deletions shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.mm
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@
NSOpenGLContext* opengl_context)
: open_gl_context_(opengl_context) {
FML_CHECK(view_controller != nullptr) << "FlutterViewController* cannot be nullptr";
FML_CHECK(platform_view_controller != nullptr)
<< "FlutterPlatformViewController* cannot be nullptr";

view_controller_ = view_controller;
platform_view_controller_ = platform_view_controller;
}

bool FlutterGLCompositor::CreateBackingStore(const FlutterBackingStoreConfig* config,
Expand Down Expand Up @@ -75,35 +79,21 @@
}

bool FlutterGLCompositor::Present(const FlutterLayer** layers, size_t layers_count) {
[platform_view_controller_ disposePlatformViews];
for (size_t i = 0; i < layers_count; ++i) {
const auto* layer = layers[i];
FlutterBackingStore* backing_store = const_cast<FlutterBackingStore*>(layer->backing_store);
switch (layer->type) {
case kFlutterLayerContentTypeBackingStore: {
if (backing_store->open_gl.framebuffer.user_data) {
FlutterBackingStoreData* backing_store_data =
FlutterBackingStoreData* flutter_backing_store_data =
(__bridge FlutterBackingStoreData*)backing_store->open_gl.framebuffer.user_data;

FlutterIOSurfaceHolder* io_surface_holder = [backing_store_data ioSurfaceHolder];
size_t layer_id = [backing_store_data layerId];

CALayer* content_layer = ca_layer_map_[layer_id];

FML_CHECK(content_layer) << "Unable to find a content layer with layer id " << layer_id;

content_layer.frame = content_layer.superlayer.bounds;

// The surface is an OpenGL texture, which means it has origin in bottom left corner
// and needs to be flipped vertically
content_layer.transform = CATransform3DMakeScale(1, -1, 1);
IOSurfaceRef io_surface_contents = [io_surface_holder ioSurface];
[content_layer setContents:(__bridge id)io_surface_contents];
PresentBackingStoreContent(flutter_backing_store_data, i);
}
break;
}
case kFlutterLayerContentTypePlatformView:
RichardJCai marked this conversation as resolved.
Show resolved Hide resolved
// Add functionality in follow up PR.
FML_LOG(WARNING) << "Presenting PlatformViews not yet supported";
PresentPlatformView(layer, i);
break;
};
}
Expand All @@ -113,6 +103,48 @@
return present_callback_();
}

void FlutterGLCompositor::PresentBackingStoreContent(
FlutterBackingStoreData* flutter_backing_store_data,
size_t layer_position) {
FML_DCHECK([[NSThread currentThread] isMainThread])
<< "Must be on the main thread to update CALayer contents";

FlutterIOSurfaceHolder* io_surface_holder = [flutter_backing_store_data ioSurfaceHolder];
size_t layer_id = [flutter_backing_store_data layerId];

CALayer* content_layer = ca_layer_map_[layer_id];

FML_DCHECK(content_layer) << "Unable to find a content layer with layer id " << layer_id;

content_layer.frame = content_layer.superlayer.bounds;
content_layer.zPosition = layer_position;

// The surface is an OpenGL texture, which means it has origin in bottom left corner
// and needs to be flipped vertically
content_layer.transform = CATransform3DMakeScale(1, -1, 1);
IOSurfaceRef io_surface_contents = [io_surface_holder ioSurface];
[content_layer setContents:(__bridge id)io_surface_contents];
}

void FlutterGLCompositor::PresentPlatformView(const FlutterLayer* layer, size_t layer_position) {
FML_DCHECK([[NSThread currentThread] isMainThread])
<< "Must be on the main thread to handle presenting platform views";

FML_DCHECK(platform_view_controller_.platformViews.count(layer->platform_view->identifier))
<< "Platform view not found for id: " << layer->platform_view->identifier;

NSView* platform_view = platform_view_controller_.platformViews[layer->platform_view->identifier];

CGFloat scale = [[NSScreen mainScreen] backingScaleFactor];
platform_view.frame = CGRectMake(layer->offset.x / scale, layer->offset.y / scale,
layer->size.width / scale, layer->size.height / scale);
if (platform_view.superview == nil) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: we generally use !foo rather than foo == nil in the macOS embedding.

[view_controller_.flutterView addSubview:platform_view];
} else {
platform_view.layer.zPosition = layer_position;
}
}

void FlutterGLCompositor::SetPresentCallback(
const FlutterGLCompositor::PresentCallback& present_callback) {
present_callback_ = present_callback;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,19 @@
#import <Foundation/Foundation.h>

#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterGLCompositor.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterPlatformViewController_Internal.h"
#import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.h"
#import "flutter/testing/testing.h"

namespace flutter::testing {

TEST(FlutterGLCompositorTest, TestPresent) {
id mockViewController = CreateMockViewController(nil);
FlutterPlatformViewController* platformViewController =
[[FlutterPlatformViewController alloc] init];

std::unique_ptr<flutter::FlutterGLCompositor> macos_compositor =
std::make_unique<FlutterGLCompositor>(mockViewController, nullptr);
std::make_unique<FlutterGLCompositor>(mockViewController, platformViewController);

bool flag = false;
macos_compositor->SetPresentCallback([f = &flag]() {
Expand Down
Loading