Multi-view pointer event (#46213)

This PR adds a new field `view_id` to embedder API's `FlutterPointerEvent`, allowing platforms to specify the source view of pointer events.

flutter/flutter#112205

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

examples/glfw/FlutterEmbedderGLFW.cc

@@ -13,6 +13,7 @@
 static double g_pixelRatio = 1.0;
 static const size_t kInitialWindowWidth = 800;
 static const size_t kInitialWindowHeight = 600;
+static constexpr FlutterViewId kImplicitViewId = 0;
 
 static_assert(FLUTTER_ENGINE_VERSION == 1,
               "This Flutter Embedder was authored against the stable Flutter "
@@ -33,6 +34,9 @@ void GLFWcursorPositionCallbackAtPhase(GLFWwindow* window,
       std::chrono::duration_cast<std::chrono::microseconds>(
           std::chrono::high_resolution_clock::now().time_since_epoch())
           .count();
+  // This example only supports a single window, therefore we assume the pointer
+  // event occurred in the only view, the implicit view.
+  event.view_id = kImplicitViewId;
   FlutterEngineSendPointerEvent(
       reinterpret_cast<FlutterEngine>(glfwGetWindowUserPointer(window)), &event,
       1);

examples/glfw_drm/FlutterEmbedderGLFW.cc

@@ -26,6 +26,7 @@ static const size_t kInitialWindowHeight = 600;
 // Maximum damage history - for triple buffering we need to store damage for
 // last two frames; Some Android devices (Pixel 4) use quad buffering.
 static const int kMaxHistorySize = 10;
+static constexpr FlutterViewId kImplicitViewId = 0;
 
 // Keeps track of the most recent frame damages so that existing damage can
 // be easily computed.
@@ -56,6 +57,9 @@ void GLFWcursorPositionCallbackAtPhase(GLFWwindow* window,
       std::chrono::duration_cast<std::chrono::microseconds>(
           std::chrono::high_resolution_clock::now().time_since_epoch())
           .count();
+  // This example only supports a single window, therefore we assume the pointer
+  // event occurred in the only view, the implicit view.
+  event.view_id = kImplicitViewId;
   FlutterEngineSendPointerEvent(
       reinterpret_cast<FlutterEngine>(glfwGetWindowUserPointer(window)), &event,
       1);

lib/ui/platform_dispatcher.dart

@@ -414,12 +414,9 @@ class PlatformDispatcher {
     }
   }
 
-  // If this value changes, update the encoding code in the following files:
-  //
-  //  * pointer_data.cc
-  //  * pointer.dart
-  //  * AndroidTouchProcessor.java
-  static const int _kPointerDataFieldCount = 35;
+  // This value must match kPointerDataFieldCount in pointer_data.cc. (The
+  // pointer_data.cc also lists other locations that must be kept consistent.)
+  static const int _kPointerDataFieldCount = 36;
 
   static PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
     const int kStride = Int64List.bytesPerElement;
@@ -430,7 +427,7 @@ class PlatformDispatcher {
     for (int i = 0; i < length; ++i) {
       int offset = i * _kPointerDataFieldCount;
       data.add(PointerData(
-        // TODO(goderbauer): Wire up viewId.
+        // The unpacking code must match the struct in pointer_data.h.
         embedderId: packet.getInt64(kStride * offset++, _kFakeHostEndian),
         timeStamp: Duration(microseconds: packet.getInt64(kStride * offset++, _kFakeHostEndian)),
         change: PointerChange.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],
@@ -466,6 +463,7 @@ class PlatformDispatcher {
         panDeltaY: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
         scale: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
         rotation: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        viewId: packet.getInt64(kStride * offset++, _kFakeHostEndian),
       ));
       assert(offset == (i + 1) * _kPointerDataFieldCount);
     }

lib/ui/pointer.dart

@@ -419,7 +419,8 @@ class PointerData {
              'panDeltaX: $panDeltaX, '
              'panDeltaY: $panDeltaY, '
              'scale: $scale, '
-             'rotation: $rotation'
+             'rotation: $rotation, '
+             'viewId: $viewId'
            ')';
   }
 }

lib/ui/window/pointer_data.cc

@@ -8,6 +8,17 @@
 
 namespace flutter {
 
+// The number of fields of PointerData.
+//
+// If kPointerDataFieldCount changes, update the corresponding values to:
+//
+//  * _kPointerDataFieldCount in platform_dispatcher.dart
+//  * POINTER_DATA_FIELD_COUNT in AndroidTouchProcessor.java
+//
+// (This is a centralized list of all locations that should be kept up-to-date.)
+static constexpr int kPointerDataFieldCount = 36;
+static constexpr int kBytesPerField = sizeof(int64_t);
+
 static_assert(sizeof(PointerData) == kBytesPerField * kPointerDataFieldCount,
               "PointerData has the wrong size");
 

lib/ui/window/pointer_data.h

@@ -9,10 +9,6 @@
 
 namespace flutter {
 
-// If this value changes, update the pointer data unpacking code in
-// platform_dispatcher.dart.
-static constexpr int kPointerDataFieldCount = 35;
-static constexpr int kBytesPerField = sizeof(int64_t);
 // Must match the button constants in events.dart.
 enum PointerButtonMouse : int64_t {
   kPointerButtonMousePrimary = 1 << 0,
@@ -32,7 +28,13 @@ enum PointerButtonStylus : int64_t {
   kPointerButtonStylusSecondary = 1 << 2,
 };
 
-// This structure is unpacked by hooks.dart.
+// This structure is unpacked by platform_dispatcher.dart.
+//
+// If this struct changes, update:
+//  * kPointerDataFieldCount in pointer_data.cc. (The pointer_data.cc also
+//    lists out other locations that must be kept consistent.)
+//  * The functions to create simulated data in
+//    pointer_data_packet_converter_unittests.cc.
 struct alignas(8) PointerData {
   // Must match the PointerChange enum in pointer.dart.
   enum class Change : int64_t {
@@ -100,6 +102,7 @@ struct alignas(8) PointerData {
   double pan_delta_y;
   double scale;
   double rotation;
+  int64_t view_id;
 
   void Clear();
 };

lib/ui/window/pointer_data_packet_converter_unittests.cc

@@ -45,6 +45,7 @@ void CreateSimulatedPointerData(PointerData& data,  // NOLINT
   data.platformData = 0;
   data.scroll_delta_x = 0.0;
   data.scroll_delta_y = 0.0;
+  data.view_id = 0;
 }
 
 void CreateSimulatedMousePointerData(PointerData& data,  // NOLINT
@@ -84,6 +85,7 @@ void CreateSimulatedMousePointerData(PointerData& data,  // NOLINT
   data.platformData = 0;
   data.scroll_delta_x = scroll_delta_x;
   data.scroll_delta_y = scroll_delta_y;
+  data.view_id = 0;
 }
 
 void CreateSimulatedTrackpadGestureData(PointerData& data,  // NOLINT
@@ -129,6 +131,7 @@ void CreateSimulatedTrackpadGestureData(PointerData& data,  // NOLINT
   data.pan_delta_y = 0.0;
   data.scale = scale;
   data.rotation = rotation;
+  data.view_id = 0;
 }
 
 void UnpackPointerPacket(std::vector<PointerData>& output,  // NOLINT
@@ -713,5 +716,25 @@ TEST(PointerDataPacketConverterTest, CanConvertTrackpadGesture) {
   ASSERT_EQ(result[3].synthesized, 0);
 }
 
+TEST(PointerDataPacketConverterTest, CanConvertViewId) {
+  PointerDataPacketConverter converter;
+  auto packet = std::make_unique<PointerDataPacket>(2);
+  PointerData data;
+  CreateSimulatedPointerData(data, PointerData::Change::kAdd, 0, 0.0, 0.0, 0);
+  data.view_id = 100;
+  packet->SetPointerData(0, data);
+  CreateSimulatedPointerData(data, PointerData::Change::kHover, 0, 1.0, 0.0, 0);
+  data.view_id = 200;
+  packet->SetPointerData(1, data);
+  auto converted_packet = converter.Convert(std::move(packet));
+
+  std::vector<PointerData> result;
+  UnpackPointerPacket(result, std::move(converted_packet));
+
+  ASSERT_EQ(result.size(), (size_t)2);
+  ASSERT_EQ(result[0].view_id, 100);
+  ASSERT_EQ(result[1].view_id, 200);
+}
+
 }  // namespace testing
 }  // namespace flutter

lib/web_ui/lib/pointer.dart

@@ -147,7 +147,8 @@ class PointerData {
            'panDeltaX: $panDeltaX, '
            'panDeltaY: $panDeltaY, '
            'scale: $scale, '
-           'rotation: $rotation'
+           'rotation: $rotation, '
+           'viewId: $viewId'
            ')';
   }
 }

shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java

@@ -80,8 +80,9 @@ public class AndroidTouchProcessor {
     int UNKNOWN = 4;
   }
 
-  // Must match the unpacking code in hooks.dart.
-  private static final int POINTER_DATA_FIELD_COUNT = 35;
+  // This value must match kPointerDataFieldCount in pointer_data.cc. (The
+  // pointer_data.cc also lists other locations that must be kept consistent.)
+  private static final int POINTER_DATA_FIELD_COUNT = 36;
   @VisibleForTesting static final int BYTES_PER_FIELD = 8;
 
   // Default if context is null, chosen to ensure reasonable speed scrolling.
@@ -92,6 +93,9 @@ public class AndroidTouchProcessor {
   // This flag indicates whether the original Android pointer events were batched together.
   private static final int POINTER_DATA_FLAG_BATCHED = 1;
 
+  // The view ID for the only view in a single-view Flutter app.
+  private static final int IMPLICIT_VIEW_ID = 0;
+
   @NonNull private final FlutterRenderer renderer;
   @NonNull private final MotionEventTracker motionEventTracker;
 
@@ -134,6 +138,8 @@ public boolean onTouchEvent(@NonNull MotionEvent event) {
   public boolean onTouchEvent(@NonNull MotionEvent event, @NonNull Matrix transformMatrix) {
     int pointerCount = event.getPointerCount();
 
+    // The following packing code must match the struct in pointer_data.h.
+
     // Prepare a data packet of the appropriate size and order.
     ByteBuffer packet =
         ByteBuffer.allocateDirect(pointerCount * POINTER_DATA_FIELD_COUNT * BYTES_PER_FIELD);
@@ -253,6 +259,10 @@ private void addPointerForIndex(
     if (pointerChange == -1) {
       return;
     }
+    // TODO(dkwingsmt): Use the correct source view ID once Android supports
+    // multiple views.
+    // https://github.com/flutter/flutter/issues/134405
+    final int viewId = IMPLICIT_VIEW_ID;
     final int pointerId = event.getPointerId(pointerIndex);
 
     int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex));
@@ -411,6 +421,7 @@ private void addPointerForIndex(
     packet.putDouble(0.0); // pan_delta_y
     packet.putDouble(1.0); // scale
     packet.putDouble(0.0); // rotation
+    packet.putLong(viewId); // view_id
 
     if (isTrackpadPan && (panZoomType == PointerChange.PAN_ZOOM_END)) {
       ongoingPans.remove(pointerId);

shell/platform/darwin/ios/framework/Source/FlutterViewController.mm

@@ -9,6 +9,7 @@
 #import <os/log.h>
 #include <memory>
 
+#include "flutter/common/constants.h"
 #include "flutter/fml/memory/weak_ptr.h"
 #include "flutter/fml/message_loop.h"
 #include "flutter/fml/platform/darwin/platform_version.h"
@@ -58,6 +59,11 @@
 // change. Unfortunately unless you have Werror turned on, incompatible pointers as arguments are
 // just a warning.
 @interface FlutterViewController () <FlutterBinaryMessenger, UIScrollViewDelegate>
+// TODO(dkwingsmt): Make the view ID property public once the iOS shell
+// supports multiple views.
+// https://github.com/flutter/flutter/issues/138168
+@property(nonatomic, readonly) int64_t viewIdentifier;
+
 @property(nonatomic, readwrite, getter=isDisplayingFlutterUI) BOOL displayingFlutterUI;
 @property(nonatomic, assign) BOOL isHomeIndicatorHidden;
 @property(nonatomic, assign) BOOL isPresentingViewControllerAnimating;
@@ -148,6 +154,7 @@ @implementation FlutterViewController {
 
 @synthesize displayingFlutterUI = _displayingFlutterUI;
 @synthesize prefersStatusBarHidden = _flutterPrefersStatusBarHidden;
+@dynamic viewIdentifier;
 
 #pragma mark - Manage and override all designated initializers
 
@@ -642,6 +649,12 @@ - (void)installFirstFrameCallback {
 
 #pragma mark - Properties
 
+- (int64_t)viewIdentifier {
+  // TODO(dkwingsmt): Fill the view ID property with the correct value once the
+  // iOS shell supports multiple views.
+  return flutter::kFlutterImplicitViewId;
+}
+
 - (UIView*)splashScreenView {
   if (!_splashScreenView) {
     return nil;
@@ -928,6 +941,7 @@ - (void)flushOngoingTouches {
       pointer_data.change = flutter::PointerData::Change::kCancel;
       pointer_data.device = device.longLongValue;
       pointer_data.pointer_identifier = 0;
+      pointer_data.view_id = self.viewIdentifier;
 
       // Anything we put here will be arbitrary since there are no touches.
       pointer_data.physical_x = 0;
@@ -1176,6 +1190,8 @@ - (void)dispatchTouches:(NSSet*)touches
 
     pointer_data.device = reinterpret_cast<int64_t>(touch);
 
+    pointer_data.view_id = self.viewIdentifier;
+
     // Pointer will be generated in pointer_data_packet_converter.cc.
     pointer_data.pointer_identifier = 0;
 
@@ -2393,6 +2409,7 @@ - (BOOL)gestureRecognizer:(UIGestureRecognizer*)gestureRecognizer
     pointer_data.device = reinterpret_cast<int64_t>(_continuousScrollingPanGestureRecognizer);
     pointer_data.kind = flutter::PointerData::DeviceKind::kTrackpad;
     pointer_data.signal_kind = flutter::PointerData::SignalKind::kScrollInertiaCancel;
+    pointer_data.view_id = self.viewIdentifier;
 
     if (event.timestamp < _scrollInertiaEventAppKitDeadline) {
       // Only send the event if it occured before the expected natural end of gesture momentum.
@@ -2416,6 +2433,7 @@ - (void)hoverEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4))
   flutter::PointerData pointer_data = [self generatePointerDataAtLastMouseLocation];
   pointer_data.device = reinterpret_cast<int64_t>(recognizer);
   pointer_data.kind = flutter::PointerData::DeviceKind::kMouse;
+  pointer_data.view_id = self.viewIdentifier;
 
   switch (_hoverGestureRecognizer.state) {
     case UIGestureRecognizerStateBegan:
@@ -2454,6 +2472,7 @@ - (void)hoverEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4))
     inertia_cancel.device = reinterpret_cast<int64_t>(_continuousScrollingPanGestureRecognizer);
     inertia_cancel.kind = flutter::PointerData::DeviceKind::kTrackpad;
     inertia_cancel.signal_kind = flutter::PointerData::SignalKind::kScrollInertiaCancel;
+    inertia_cancel.view_id = self.viewIdentifier;
     packet->SetPointerData(/*i=*/1, inertia_cancel);
     [_engine.get() dispatchPointerDataPacket:std::move(packet)];
     _scrollInertiaEventStartline = DBL_MAX;
@@ -2477,6 +2496,7 @@ - (void)discreteScrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(io
   pointer_data.signal_kind = flutter::PointerData::SignalKind::kScroll;
   pointer_data.scroll_delta_x = (translation.x - _mouseState.last_translation.x);
   pointer_data.scroll_delta_y = -(translation.y - _mouseState.last_translation.y);
+  pointer_data.view_id = self.viewIdentifier;
 
   // The translation reported by UIPanGestureRecognizer is the total translation
   // generated by the pan gesture since the gesture began. We need to be able
@@ -2500,6 +2520,7 @@ - (void)continuousScrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(
   flutter::PointerData pointer_data = [self generatePointerDataAtLastMouseLocation];
   pointer_data.device = reinterpret_cast<int64_t>(recognizer);
   pointer_data.kind = flutter::PointerData::DeviceKind::kTrackpad;
+  pointer_data.view_id = self.viewIdentifier;
   switch (recognizer.state) {
     case UIGestureRecognizerStateBegan:
       pointer_data.change = flutter::PointerData::Change::kPanZoomStart;
@@ -2548,6 +2569,7 @@ - (void)pinchEvent:(UIPinchGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)
   flutter::PointerData pointer_data = [self generatePointerDataAtLastMouseLocation];
   pointer_data.device = reinterpret_cast<int64_t>(recognizer);
   pointer_data.kind = flutter::PointerData::DeviceKind::kTrackpad;
+  pointer_data.view_id = self.viewIdentifier;
   switch (recognizer.state) {
     case UIGestureRecognizerStateBegan:
       pointer_data.change = flutter::PointerData::Change::kPanZoomStart;

shell/platform/darwin/macos/framework/Source/FlutterViewController.mm

@@ -743,6 +743,7 @@ - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase {
       .device_kind = deviceKind,
       // If a click triggered a synthesized kAdd, don't pass the buttons in that event.
       .buttons = phase == kAdd ? 0 : _mouseState.buttons,
+      .view_id = static_cast<FlutterViewId>(_viewId),
   };
 
   if (phase == kPanZoomUpdate) {
@@ -1049,6 +1050,7 @@ - (void)touchesBeganWithEvent:(NSEvent*)event {
           .device = kPointerPanZoomDeviceId,
           .signal_kind = kFlutterPointerSignalKindScrollInertiaCancel,
           .device_kind = kFlutterPointerDeviceKindTrackpad,
+          .view_id = static_cast<FlutterViewId>(_viewId),
       };
 
       [_engine sendPointerEvent:flutterEvent];

shell/platform/embedder/embedder.cc

@@ -102,7 +102,7 @@ extern const intptr_t kPlatformStrongDillSize;
 const int32_t kFlutterSemanticsNodeIdBatchEnd = -1;
 const int32_t kFlutterSemanticsCustomActionIdBatchEnd = -1;
 
-static constexpr int64_t kFlutterImplicitViewId = 0;
+static constexpr FlutterViewId kFlutterImplicitViewId = 0;
 
 // A message channel to send platform-independent FlutterKeyData to the
 // framework.
@@ -2326,6 +2326,8 @@ FlutterEngineResult FlutterEngineSendPointerEvent(
     pointer_data.pan_delta_y = 0.0;
     pointer_data.scale = SAFE_ACCESS(current, scale, 0.0);
     pointer_data.rotation = SAFE_ACCESS(current, rotation, 0.0);
+    pointer_data.view_id =
+        SAFE_ACCESS(current, view_id, kFlutterImplicitViewId);
     packet->SetPointerData(i, pointer_data);
     current = reinterpret_cast<const FlutterPointerEvent*>(
         reinterpret_cast<const uint8_t*>(current) + current->struct_size);