From afbf2e70cb9abe8f07e8dd3cb6d71d55d742ef41 Mon Sep 17 00:00:00 2001 From: Loic Sharma Date: Thu, 22 Feb 2024 17:25:51 -0800 Subject: [PATCH] [Windows] Introduce an accessibility plugin --- ci/licenses_golden/licenses_flutter | 4 + shell/platform/windows/BUILD.gn | 2 + .../platform/windows/accessibility_plugin.cc | 93 +++++++++++++++++++ shell/platform/windows/accessibility_plugin.h | 41 ++++++++ shell/platform/windows/fixtures/main.dart | 85 +++++++++++------ .../windows/flutter_windows_engine.cc | 38 ++------ .../platform/windows/flutter_windows_engine.h | 7 +- .../flutter_windows_engine_unittests.cc | 48 ++++++++++ 8 files changed, 257 insertions(+), 61 deletions(-) create mode 100644 shell/platform/windows/accessibility_plugin.cc create mode 100644 shell/platform/windows/accessibility_plugin.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index e051154ac0e64..29809ce3d2ccc 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -29964,6 +29964,8 @@ ORIGIN: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_view.h + . ORIGIN: ../../../flutter/shell/platform/linux/public/flutter_linux/flutter_linux.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/windows/accessibility_plugin.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/shell/platform/windows/accessibility_plugin.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/flutter_engine.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/flutter_view_controller.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/include/flutter/dart_project.h + ../../../flutter/LICENSE @@ -32825,6 +32827,8 @@ FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_view.h FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/flutter_linux.h FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.cc FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.h +FILE: ../../../flutter/shell/platform/windows/accessibility_plugin.cc +FILE: ../../../flutter/shell/platform/windows/accessibility_plugin.h FILE: ../../../flutter/shell/platform/windows/client_wrapper/flutter_engine.cc FILE: ../../../flutter/shell/platform/windows/client_wrapper/flutter_view_controller.cc FILE: ../../../flutter/shell/platform/windows/client_wrapper/include/flutter/dart_project.h diff --git a/shell/platform/windows/BUILD.gn b/shell/platform/windows/BUILD.gn index 0f0815cb261e8..50998f5840393 100644 --- a/shell/platform/windows/BUILD.gn +++ b/shell/platform/windows/BUILD.gn @@ -40,6 +40,8 @@ source_set("flutter_windows_source") { sources = [ "accessibility_bridge_windows.cc", "accessibility_bridge_windows.h", + "accessibility_plugin.cc", + "accessibility_plugin.h", "compositor.h", "compositor_opengl.cc", "compositor_opengl.h", diff --git a/shell/platform/windows/accessibility_plugin.cc b/shell/platform/windows/accessibility_plugin.cc new file mode 100644 index 0000000000000..1b67df4fe1dcf --- /dev/null +++ b/shell/platform/windows/accessibility_plugin.cc @@ -0,0 +1,93 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/windows/accessibility_plugin.h" + +#include + +#include "flutter/fml/platform/win/wstring_conversion.h" +#include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h" +#include "flutter/shell/platform/windows/flutter_windows_engine.h" +#include "flutter/shell/platform/windows/flutter_windows_view.h" + +namespace flutter { + +namespace { + +static constexpr char kAccessibilityChannelName[] = "flutter/accessibility"; +static constexpr char kTypeKey[] = "type"; +static constexpr char kDataKey[] = "data"; +static constexpr char kMessageKey[] = "message"; +static constexpr char kAnnounceValue[] = "announce"; + +// Handles messages like: +// {"type": "announce", "data": {"message": "Hello"}} +void HandleMessage(AccessibilityPlugin* plugin, const EncodableValue& message) { + const auto* map = std::get_if(&message); + if (!map) { + return; + } + const auto& type_itr = map->find(EncodableValue{kTypeKey}); + const auto& data_itr = map->find(EncodableValue{kDataKey}); + if (type_itr == map->end() || data_itr == map->end()) { + return; + } + const auto* type = std::get_if(&type_itr->second); + const auto* data = std::get_if(&data_itr->second); + if (!type || !data) { + return; + } + + if (type->compare(kAnnounceValue) == 0) { + const auto& message_itr = data->find(EncodableValue{kMessageKey}); + if (message_itr == data->end()) { + return; + } + const auto* message = std::get_if(&message_itr->second); + if (!message) { + return; + } + + plugin->Announce(*message); + } +} + +} // namespace + +AccessibilityPlugin::AccessibilityPlugin(FlutterWindowsEngine* engine) + : engine_(engine) {} + +void AccessibilityPlugin::SetUp(BinaryMessenger* binary_messenger, + AccessibilityPlugin* plugin) { + BasicMessageChannel<> channel{binary_messenger, kAccessibilityChannelName, + &StandardMessageCodec::GetInstance()}; + + channel.SetMessageHandler( + [plugin](const EncodableValue& message, + const MessageReply& reply) { + HandleMessage(plugin, message); + + // The accessibility channel does not support error handling. + // Always return an empty response even on failure. + reply(EncodableValue{std::monostate{}}); + }); +} + +void AccessibilityPlugin::Announce(const std::string_view message) { + if (!engine_->semantics_enabled()) { + return; + } + + // TODO(loicsharma): Remove implicit view assumption. + // https://github.com/flutter/flutter/issues/142845 + auto view = engine_->view(kImplicitViewId); + if (!view) { + return; + } + + std::wstring wide_text = fml::Utf8ToWideString(message); + view->AnnounceAlert(wide_text); +} + +} // namespace flutter diff --git a/shell/platform/windows/accessibility_plugin.h b/shell/platform/windows/accessibility_plugin.h new file mode 100644 index 0000000000000..cd0c24ead8645 --- /dev/null +++ b/shell/platform/windows/accessibility_plugin.h @@ -0,0 +1,41 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_ +#define FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_ + +#include + +#include "flutter/fml/macros.h" +#include "flutter/shell/platform/common/client_wrapper/include/flutter/binary_messenger.h" + +namespace flutter { + +class FlutterWindowsEngine; + +// Handles messages on the flutter/accessibility channel. +// +// See: +// https://api.flutter.dev/flutter/semantics/SemanticsService-class.html +class AccessibilityPlugin { + public: + explicit AccessibilityPlugin(FlutterWindowsEngine* engine); + + // Begin handling accessibility messages on the `binary_messenger`. + static void SetUp(BinaryMessenger* binary_messenger, + AccessibilityPlugin* plugin); + + // Announce a message through the assistive technology. + virtual void Announce(const std::string_view message); + + private: + // The engine that owns this plugin. + FlutterWindowsEngine* engine_ = nullptr; + + FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityPlugin); +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_ diff --git a/shell/platform/windows/fixtures/main.dart b/shell/platform/windows/fixtures/main.dart index 25ac37fcc4c0b..71dcd1679d408 100644 --- a/shell/platform/windows/fixtures/main.dart +++ b/shell/platform/windows/fixtures/main.dart @@ -59,36 +59,65 @@ void sendAccessibilityAnnouncement() async { await semanticsChanged; } - // Serializers for data types are in the framework, so this will be hardcoded. + // Standard message codec magic number identifiers. + // See: https://github.com/flutter/flutter/blob/ee94fe262b63b0761e8e1f889ae52322fef068d2/packages/flutter/lib/src/services/message_codecs.dart#L262 const int valueMap = 13, valueString = 7; - // Corresponds to: - // Map data = - // {"type": "announce", "data": {"message": ""}}; + + // Corresponds to: {"type": "announce", "data": {"message": "hello"}} + // See: https://github.com/flutter/flutter/blob/b781da9b5822de1461a769c3b245075359f5464d/packages/flutter/lib/src/semantics/semantics_event.dart#L86 + final Uint8List data = Uint8List.fromList([ + // Map with 2 entries + valueMap, 2, + // Map key: "type" + valueString, 'type'.length, ...'type'.codeUnits, + // Map value: "announce" + valueString, 'announce'.length, ...'announce'.codeUnits, + // Map key: "data" + valueString, 'data'.length, ...'data'.codeUnits, + // Map value: map with 1 entry + valueMap, 1, + // Map key: "message" + valueString, 'message'.length, ...'message'.codeUnits, + // Map value: "hello" + valueString, 'hello'.length, ...'hello'.codeUnits, + ]); + final ByteData byteData = data.buffer.asByteData(); + + ui.PlatformDispatcher.instance.sendPlatformMessage( + 'flutter/accessibility', + byteData, + (ByteData? _) => signal(), + ); +} + +@pragma('vm:entry-point') +void sendAccessibilityTooltipEvent() async { + // Wait until semantics are enabled. + if (!ui.PlatformDispatcher.instance.semanticsEnabled) { + await semanticsChanged; + } + + // Standard message codec magic number identifiers. + // See: https://github.com/flutter/flutter/blob/ee94fe262b63b0761e8e1f889ae52322fef068d2/packages/flutter/lib/src/services/message_codecs.dart#L262 + const int valueMap = 13, valueString = 7; + + // Corresponds to: {"type": "tooltip", "data": {"message": "hello"}} + // See: https://github.com/flutter/flutter/blob/b781da9b5822de1461a769c3b245075359f5464d/packages/flutter/lib/src/semantics/semantics_event.dart#L120 final Uint8List data = Uint8List.fromList([ - valueMap, // _valueMap - 2, // Size - // key: "type" - valueString, - 'type'.length, - ...'type'.codeUnits, - // value: "announce" - valueString, - 'announce'.length, - ...'announce'.codeUnits, - // key: "data" - valueString, - 'data'.length, - ...'data'.codeUnits, - // value: map - valueMap, // _valueMap - 1, // Size - // key: "message" - valueString, - 'message'.length, - ...'message'.codeUnits, - // value: "" - valueString, - 0, // Length of empty string == 0. + // Map with 2 entries + valueMap, 2, + // Map key: "type" + valueString, 'type'.length, ...'type'.codeUnits, + // Map value: "tooltip" + valueString, 'tooltip'.length, ...'tooltip'.codeUnits, + // Map key: "data" + valueString, 'data'.length, ...'data'.codeUnits, + // Map value: map with 1 entry + valueMap, 1, + // Map key: "message" + valueString, 'message'.length, ...'message'.codeUnits, + // Map value: "hello" + valueString, 'hello'.length, ...'hello'.codeUnits, ]); final ByteData byteData = data.buffer.asByteData(); diff --git a/shell/platform/windows/flutter_windows_engine.cc b/shell/platform/windows/flutter_windows_engine.cc index 7ccb66037874b..25c8b859ccea0 100644 --- a/shell/platform/windows/flutter_windows_engine.cc +++ b/shell/platform/windows/flutter_windows_engine.cc @@ -183,14 +183,6 @@ FlutterWindowsEngine::FlutterWindowsEngine( std::make_unique(messenger_->ToRef()); message_dispatcher_ = std::make_unique(messenger_->ToRef()); - message_dispatcher_->SetMessageCallback( - kAccessibilityChannelName, - [](FlutterDesktopMessengerRef messenger, - const FlutterDesktopMessage* message, void* data) { - FlutterWindowsEngine* engine = static_cast(data); - engine->HandleAccessibilityMessage(messenger, message); - }, - static_cast(this)); texture_registrar_ = std::make_unique(this, gl_); @@ -219,6 +211,11 @@ FlutterWindowsEngine::FlutterWindowsEngine( // https://github.com/flutter/flutter/issues/71099 internal_plugin_registrar_ = std::make_unique(plugin_registrar_.get()); + + accessibility_plugin_ = std::make_unique(this); + AccessibilityPlugin::SetUp(messenger_wrapper_.get(), + accessibility_plugin_.get()); + cursor_handler_ = std::make_unique(messenger_wrapper_.get(), this); platform_handler_ = @@ -759,7 +756,9 @@ void FlutterWindowsEngine::UpdateSemanticsEnabled(bool enabled) { if (engine_ && semantics_enabled_ != enabled) { semantics_enabled_ = enabled; embedder_api_.UpdateSemanticsEnabled(engine_, enabled); - view_->UpdateSemanticsEnabled(enabled); + if (view_) { + view_->UpdateSemanticsEnabled(enabled); + } } } @@ -807,27 +806,6 @@ void FlutterWindowsEngine::SendAccessibilityFeatures() { engine_, static_cast(flags)); } -void FlutterWindowsEngine::HandleAccessibilityMessage( - FlutterDesktopMessengerRef messenger, - const FlutterDesktopMessage* message) { - const auto& codec = StandardMessageCodec::GetInstance(); - auto data = codec.DecodeMessage(message->message, message->message_size); - EncodableMap map = std::get(*data); - std::string type = std::get(map.at(EncodableValue("type"))); - if (type.compare("announce") == 0) { - if (semantics_enabled_) { - EncodableMap data_map = - std::get(map.at(EncodableValue("data"))); - std::string text = - std::get(data_map.at(EncodableValue("message"))); - std::wstring wide_text = fml::Utf8ToWideString(text); - view_->AnnounceAlert(wide_text); - } - } - SendPlatformMessageResponse(message->response_handle, - reinterpret_cast(""), 0); -} - void FlutterWindowsEngine::RequestApplicationQuit(HWND hwnd, WPARAM wparam, LPARAM lparam, diff --git a/shell/platform/windows/flutter_windows_engine.h b/shell/platform/windows/flutter_windows_engine.h index 53710b141e839..c33e6d9cd8229 100644 --- a/shell/platform/windows/flutter_windows_engine.h +++ b/shell/platform/windows/flutter_windows_engine.h @@ -22,6 +22,7 @@ #include "flutter/shell/platform/common/incoming_message_dispatcher.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/windows/accessibility_bridge_windows.h" +#include "flutter/shell/platform/windows/accessibility_plugin.h" #include "flutter/shell/platform/windows/compositor.h" #include "flutter/shell/platform/windows/cursor_handler.h" #include "flutter/shell/platform/windows/egl/manager.h" @@ -335,9 +336,6 @@ class FlutterWindowsEngine { // Send the currently enabled accessibility features to the engine. void SendAccessibilityFeatures(); - void HandleAccessibilityMessage(FlutterDesktopMessengerRef messenger, - const FlutterDesktopMessage* message); - // The handle to the embedder.h engine instance. FLUTTER_API_SYMBOL(FlutterEngine) engine_ = nullptr; @@ -381,6 +379,9 @@ class FlutterWindowsEngine { // The plugin registrar managing internal plugins. std::unique_ptr internal_plugin_registrar_; + // Handler for accessibility events. + std::unique_ptr accessibility_plugin_; + // Handler for cursor events. std::unique_ptr cursor_handler_; diff --git a/shell/platform/windows/flutter_windows_engine_unittests.cc b/shell/platform/windows/flutter_windows_engine_unittests.cc index 901a8b772a0ec..6c55e995a3510 100644 --- a/shell/platform/windows/flutter_windows_engine_unittests.cc +++ b/shell/platform/windows/flutter_windows_engine_unittests.cc @@ -676,6 +676,54 @@ TEST_F(FlutterWindowsEngineTest, AccessibilityAnnouncement) { } } +// Verify the app can send accessibility announcements while in headless mode. +TEST_F(FlutterWindowsEngineTest, AccessibilityAnnouncementHeadless) { + auto& context = GetContext(); + WindowsConfigBuilder builder{context}; + builder.SetDartEntrypoint("sendAccessibilityAnnouncement"); + + bool done = false; + auto native_entry = + CREATE_NATIVE_ENTRY([&](Dart_NativeArguments args) { done = true; }); + context.AddNativeFunction("Signal", native_entry); + + EnginePtr engine{builder.RunHeadless()}; + ASSERT_NE(engine, nullptr); + + auto windows_engine = reinterpret_cast(engine.get()); + windows_engine->UpdateSemanticsEnabled(true); + + // Rely on timeout mechanism in CI. + while (!done) { + windows_engine->task_runner()->ProcessTasks(); + } +} + +// Verify the engine does not crash if it receives an accessibility event +// it does not support yet. +TEST_F(FlutterWindowsEngineTest, AccessibilityTooltip) { + auto& context = GetContext(); + WindowsConfigBuilder builder{context}; + builder.SetDartEntrypoint("sendAccessibilityTooltipEvent"); + + bool done = false; + auto native_entry = + CREATE_NATIVE_ENTRY([&](Dart_NativeArguments args) { done = true; }); + context.AddNativeFunction("Signal", native_entry); + + ViewControllerPtr controller{builder.Run()}; + ASSERT_NE(controller, nullptr); + + auto engine = FlutterDesktopViewControllerGetEngine(controller.get()); + auto windows_engine = reinterpret_cast(engine); + windows_engine->UpdateSemanticsEnabled(true); + + // Rely on timeout mechanism in CI. + while (!done) { + windows_engine->task_runner()->ProcessTasks(); + } +} + class MockWindowsLifecycleManager : public WindowsLifecycleManager { public: MockWindowsLifecycleManager(FlutterWindowsEngine* engine)