Skip to content

Commit

Permalink
[Windows] Introduce an accessibility plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
loic-sharma committed Feb 26, 2024
1 parent 3c036c0 commit afbf2e7
Show file tree
Hide file tree
Showing 8 changed files with 257 additions and 61 deletions.
4 changes: 4 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -29964,6 +29964,8 @@ ORIGIN: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_view.h + .
ORIGIN: ../../../flutter/shell/platform/linux/public/flutter_linux/flutter_linux.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/accessibility_plugin.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/accessibility_plugin.h + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/flutter_engine.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/flutter_view_controller.cc + ../../../flutter/LICENSE
ORIGIN: ../../../flutter/shell/platform/windows/client_wrapper/include/flutter/dart_project.h + ../../../flutter/LICENSE
Expand Down Expand Up @@ -32825,6 +32827,8 @@ FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_view.h
FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/flutter_linux.h
FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.cc
FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_windows.h
FILE: ../../../flutter/shell/platform/windows/accessibility_plugin.cc
FILE: ../../../flutter/shell/platform/windows/accessibility_plugin.h
FILE: ../../../flutter/shell/platform/windows/client_wrapper/flutter_engine.cc
FILE: ../../../flutter/shell/platform/windows/client_wrapper/flutter_view_controller.cc
FILE: ../../../flutter/shell/platform/windows/client_wrapper/include/flutter/dart_project.h
Expand Down
2 changes: 2 additions & 0 deletions shell/platform/windows/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ source_set("flutter_windows_source") {
sources = [
"accessibility_bridge_windows.cc",
"accessibility_bridge_windows.h",
"accessibility_plugin.cc",
"accessibility_plugin.h",
"compositor.h",
"compositor_opengl.cc",
"compositor_opengl.h",
Expand Down
93 changes: 93 additions & 0 deletions shell/platform/windows/accessibility_plugin.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "flutter/shell/platform/windows/accessibility_plugin.h"

#include <variant>

#include "flutter/fml/platform/win/wstring_conversion.h"
#include "flutter/shell/platform/common/client_wrapper/include/flutter/standard_message_codec.h"
#include "flutter/shell/platform/windows/flutter_windows_engine.h"
#include "flutter/shell/platform/windows/flutter_windows_view.h"

namespace flutter {

namespace {

static constexpr char kAccessibilityChannelName[] = "flutter/accessibility";
static constexpr char kTypeKey[] = "type";
static constexpr char kDataKey[] = "data";
static constexpr char kMessageKey[] = "message";
static constexpr char kAnnounceValue[] = "announce";

// Handles messages like:
// {"type": "announce", "data": {"message": "Hello"}}
void HandleMessage(AccessibilityPlugin* plugin, const EncodableValue& message) {
const auto* map = std::get_if<EncodableMap>(&message);
if (!map) {
return;
}
const auto& type_itr = map->find(EncodableValue{kTypeKey});
const auto& data_itr = map->find(EncodableValue{kDataKey});
if (type_itr == map->end() || data_itr == map->end()) {
return;
}
const auto* type = std::get_if<std::string>(&type_itr->second);
const auto* data = std::get_if<EncodableMap>(&data_itr->second);
if (!type || !data) {
return;
}

if (type->compare(kAnnounceValue) == 0) {
const auto& message_itr = data->find(EncodableValue{kMessageKey});
if (message_itr == data->end()) {
return;
}
const auto* message = std::get_if<std::string>(&message_itr->second);
if (!message) {
return;
}

plugin->Announce(*message);
}
}

} // namespace

AccessibilityPlugin::AccessibilityPlugin(FlutterWindowsEngine* engine)
: engine_(engine) {}

void AccessibilityPlugin::SetUp(BinaryMessenger* binary_messenger,
AccessibilityPlugin* plugin) {
BasicMessageChannel<> channel{binary_messenger, kAccessibilityChannelName,
&StandardMessageCodec::GetInstance()};

channel.SetMessageHandler(
[plugin](const EncodableValue& message,
const MessageReply<EncodableValue>& reply) {
HandleMessage(plugin, message);

// The accessibility channel does not support error handling.
// Always return an empty response even on failure.
reply(EncodableValue{std::monostate{}});
});
}

void AccessibilityPlugin::Announce(const std::string_view message) {
if (!engine_->semantics_enabled()) {
return;
}

// TODO(loicsharma): Remove implicit view assumption.
// https://github.com/flutter/flutter/issues/142845
auto view = engine_->view(kImplicitViewId);
if (!view) {
return;
}

std::wstring wide_text = fml::Utf8ToWideString(message);
view->AnnounceAlert(wide_text);
}

} // namespace flutter
41 changes: 41 additions & 0 deletions shell/platform/windows/accessibility_plugin.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_
#define FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_

#include <string_view>

#include "flutter/fml/macros.h"
#include "flutter/shell/platform/common/client_wrapper/include/flutter/binary_messenger.h"

namespace flutter {

class FlutterWindowsEngine;

// Handles messages on the flutter/accessibility channel.
//
// See:
// https://api.flutter.dev/flutter/semantics/SemanticsService-class.html
class AccessibilityPlugin {
public:
explicit AccessibilityPlugin(FlutterWindowsEngine* engine);

// Begin handling accessibility messages on the `binary_messenger`.
static void SetUp(BinaryMessenger* binary_messenger,
AccessibilityPlugin* plugin);

// Announce a message through the assistive technology.
virtual void Announce(const std::string_view message);

private:
// The engine that owns this plugin.
FlutterWindowsEngine* engine_ = nullptr;

FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityPlugin);
};

} // namespace flutter

#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_PLUGIN_H_
85 changes: 57 additions & 28 deletions shell/platform/windows/fixtures/main.dart
Original file line number Diff line number Diff line change
Expand Up @@ -59,36 +59,65 @@ void sendAccessibilityAnnouncement() async {
await semanticsChanged;
}

// Serializers for data types are in the framework, so this will be hardcoded.
// Standard message codec magic number identifiers.
// See: https://github.com/flutter/flutter/blob/ee94fe262b63b0761e8e1f889ae52322fef068d2/packages/flutter/lib/src/services/message_codecs.dart#L262
const int valueMap = 13, valueString = 7;
// Corresponds to:
// Map<String, Object> data =
// {"type": "announce", "data": {"message": ""}};

// Corresponds to: {"type": "announce", "data": {"message": "hello"}}
// See: https://github.com/flutter/flutter/blob/b781da9b5822de1461a769c3b245075359f5464d/packages/flutter/lib/src/semantics/semantics_event.dart#L86
final Uint8List data = Uint8List.fromList([
// Map with 2 entries
valueMap, 2,
// Map key: "type"
valueString, 'type'.length, ...'type'.codeUnits,
// Map value: "announce"
valueString, 'announce'.length, ...'announce'.codeUnits,
// Map key: "data"
valueString, 'data'.length, ...'data'.codeUnits,
// Map value: map with 1 entry
valueMap, 1,
// Map key: "message"
valueString, 'message'.length, ...'message'.codeUnits,
// Map value: "hello"
valueString, 'hello'.length, ...'hello'.codeUnits,
]);
final ByteData byteData = data.buffer.asByteData();

ui.PlatformDispatcher.instance.sendPlatformMessage(
'flutter/accessibility',
byteData,
(ByteData? _) => signal(),
);
}

@pragma('vm:entry-point')
void sendAccessibilityTooltipEvent() async {
// Wait until semantics are enabled.
if (!ui.PlatformDispatcher.instance.semanticsEnabled) {
await semanticsChanged;
}

// Standard message codec magic number identifiers.
// See: https://github.com/flutter/flutter/blob/ee94fe262b63b0761e8e1f889ae52322fef068d2/packages/flutter/lib/src/services/message_codecs.dart#L262
const int valueMap = 13, valueString = 7;

// Corresponds to: {"type": "tooltip", "data": {"message": "hello"}}
// See: https://github.com/flutter/flutter/blob/b781da9b5822de1461a769c3b245075359f5464d/packages/flutter/lib/src/semantics/semantics_event.dart#L120
final Uint8List data = Uint8List.fromList([
valueMap, // _valueMap
2, // Size
// key: "type"
valueString,
'type'.length,
...'type'.codeUnits,
// value: "announce"
valueString,
'announce'.length,
...'announce'.codeUnits,
// key: "data"
valueString,
'data'.length,
...'data'.codeUnits,
// value: map
valueMap, // _valueMap
1, // Size
// key: "message"
valueString,
'message'.length,
...'message'.codeUnits,
// value: ""
valueString,
0, // Length of empty string == 0.
// Map with 2 entries
valueMap, 2,
// Map key: "type"
valueString, 'type'.length, ...'type'.codeUnits,
// Map value: "tooltip"
valueString, 'tooltip'.length, ...'tooltip'.codeUnits,
// Map key: "data"
valueString, 'data'.length, ...'data'.codeUnits,
// Map value: map with 1 entry
valueMap, 1,
// Map key: "message"
valueString, 'message'.length, ...'message'.codeUnits,
// Map value: "hello"
valueString, 'hello'.length, ...'hello'.codeUnits,
]);
final ByteData byteData = data.buffer.asByteData();

Expand Down
38 changes: 8 additions & 30 deletions shell/platform/windows/flutter_windows_engine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,6 @@ FlutterWindowsEngine::FlutterWindowsEngine(
std::make_unique<BinaryMessengerImpl>(messenger_->ToRef());
message_dispatcher_ =
std::make_unique<IncomingMessageDispatcher>(messenger_->ToRef());
message_dispatcher_->SetMessageCallback(
kAccessibilityChannelName,
[](FlutterDesktopMessengerRef messenger,
const FlutterDesktopMessage* message, void* data) {
FlutterWindowsEngine* engine = static_cast<FlutterWindowsEngine*>(data);
engine->HandleAccessibilityMessage(messenger, message);
},
static_cast<void*>(this));

texture_registrar_ =
std::make_unique<FlutterWindowsTextureRegistrar>(this, gl_);
Expand Down Expand Up @@ -219,6 +211,11 @@ FlutterWindowsEngine::FlutterWindowsEngine(
// https://github.com/flutter/flutter/issues/71099
internal_plugin_registrar_ =
std::make_unique<PluginRegistrar>(plugin_registrar_.get());

accessibility_plugin_ = std::make_unique<AccessibilityPlugin>(this);
AccessibilityPlugin::SetUp(messenger_wrapper_.get(),
accessibility_plugin_.get());

cursor_handler_ =
std::make_unique<CursorHandler>(messenger_wrapper_.get(), this);
platform_handler_ =
Expand Down Expand Up @@ -759,7 +756,9 @@ void FlutterWindowsEngine::UpdateSemanticsEnabled(bool enabled) {
if (engine_ && semantics_enabled_ != enabled) {
semantics_enabled_ = enabled;
embedder_api_.UpdateSemanticsEnabled(engine_, enabled);
view_->UpdateSemanticsEnabled(enabled);
if (view_) {
view_->UpdateSemanticsEnabled(enabled);
}
}
}

Expand Down Expand Up @@ -807,27 +806,6 @@ void FlutterWindowsEngine::SendAccessibilityFeatures() {
engine_, static_cast<FlutterAccessibilityFeature>(flags));
}

void FlutterWindowsEngine::HandleAccessibilityMessage(
FlutterDesktopMessengerRef messenger,
const FlutterDesktopMessage* message) {
const auto& codec = StandardMessageCodec::GetInstance();
auto data = codec.DecodeMessage(message->message, message->message_size);
EncodableMap map = std::get<EncodableMap>(*data);
std::string type = std::get<std::string>(map.at(EncodableValue("type")));
if (type.compare("announce") == 0) {
if (semantics_enabled_) {
EncodableMap data_map =
std::get<EncodableMap>(map.at(EncodableValue("data")));
std::string text =
std::get<std::string>(data_map.at(EncodableValue("message")));
std::wstring wide_text = fml::Utf8ToWideString(text);
view_->AnnounceAlert(wide_text);
}
}
SendPlatformMessageResponse(message->response_handle,
reinterpret_cast<const uint8_t*>(""), 0);
}

void FlutterWindowsEngine::RequestApplicationQuit(HWND hwnd,
WPARAM wparam,
LPARAM lparam,
Expand Down
7 changes: 4 additions & 3 deletions shell/platform/windows/flutter_windows_engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "flutter/shell/platform/common/incoming_message_dispatcher.h"
#include "flutter/shell/platform/embedder/embedder.h"
#include "flutter/shell/platform/windows/accessibility_bridge_windows.h"
#include "flutter/shell/platform/windows/accessibility_plugin.h"
#include "flutter/shell/platform/windows/compositor.h"
#include "flutter/shell/platform/windows/cursor_handler.h"
#include "flutter/shell/platform/windows/egl/manager.h"
Expand Down Expand Up @@ -335,9 +336,6 @@ class FlutterWindowsEngine {
// Send the currently enabled accessibility features to the engine.
void SendAccessibilityFeatures();

void HandleAccessibilityMessage(FlutterDesktopMessengerRef messenger,
const FlutterDesktopMessage* message);

// The handle to the embedder.h engine instance.
FLUTTER_API_SYMBOL(FlutterEngine) engine_ = nullptr;

Expand Down Expand Up @@ -381,6 +379,9 @@ class FlutterWindowsEngine {
// The plugin registrar managing internal plugins.
std::unique_ptr<PluginRegistrar> internal_plugin_registrar_;

// Handler for accessibility events.
std::unique_ptr<AccessibilityPlugin> accessibility_plugin_;

// Handler for cursor events.
std::unique_ptr<CursorHandler> cursor_handler_;

Expand Down
Loading

0 comments on commit afbf2e7

Please sign in to comment.