From 01c8ecbb74c1b6cbe22953f65c6fb6863353388a Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Mon, 6 Jul 2020 22:22:37 -0700 Subject: [PATCH] Track motion events for reuse post gesture disambiguation (#19484) This change makes it so that we track all the motion events encountered by `FlutterView` and all of its subviews in the `MotionEventTracker` class, indexed by a unique `MotionEventId`. This identifier is then passed to the Flutter framework as seen in https://github.com/flutter/flutter/pull/60930. Once the gestures take part in gesture disambiguation and are sent back to the engine, we look-up the original motion event using the `MotionEventId` and dispatch it to the platform. Bug: https://github.com/flutter/flutter/issues/58837 --- ci/licenses_golden/licenses_flutter | 1 + lib/ui/hooks.dart | 5 +- lib/ui/pointer.dart | 9 +++ lib/ui/window/pointer_data.h | 3 +- lib/web_ui/lib/src/ui/pointer.dart | 69 +++++++++------- shell/platform/android/BUILD.gn | 1 + .../android/AndroidTouchProcessor.java | 7 +- .../embedding/android/MotionEventTracker.java | 79 +++++++++++++++++++ .../systemchannels/PlatformViewsChannel.java | 9 ++- .../platform/PlatformViewsController.java | 18 ++++- shell/platform/embedder/embedder.cc | 2 + 11 files changed, 165 insertions(+), 38 deletions(-) create mode 100644 shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 206221ac40f53..057c045413f57 100755 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -697,6 +697,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/Flutt FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterSurfaceView.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterTextureView.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterView.java +FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/RenderMode.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreen.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreenProvider.java diff --git a/lib/ui/hooks.dart b/lib/ui/hooks.dart index 645e54e3879dd..ff2dcaa127391 100644 --- a/lib/ui/hooks.dart +++ b/lib/ui/hooks.dart @@ -287,9 +287,9 @@ void _invoke3(void callback(A1 a1, A2 a2, A3 a3)?, Zone zone, A1 arg // If this value changes, update the encoding code in the following files: // // * pointer_data.cc -// * pointers.dart +// * pointer.dart // * AndroidTouchProcessor.java -const int _kPointerDataFieldCount = 28; +const int _kPointerDataFieldCount = 29; PointerDataPacket _unpackPointerDataPacket(ByteData packet) { const int kStride = Int64List.bytesPerElement; @@ -300,6 +300,7 @@ PointerDataPacket _unpackPointerDataPacket(ByteData packet) { for (int i = 0; i < length; ++i) { int offset = i * _kPointerDataFieldCount; data.add(PointerData( + embedderId: packet.getInt64(kStride * offset++, _kFakeHostEndian), timeStamp: Duration(microseconds: packet.getInt64(kStride * offset++, _kFakeHostEndian)), change: PointerChange.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)], kind: PointerDeviceKind.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)], diff --git a/lib/ui/pointer.dart b/lib/ui/pointer.dart index bf9ba6496ad6f..3c12cb9718f63 100644 --- a/lib/ui/pointer.dart +++ b/lib/ui/pointer.dart @@ -72,6 +72,7 @@ enum PointerSignalKind { class PointerData { /// Creates an object that represents the state of a pointer. const PointerData({ + this.embedderId = 0, this.timeStamp = Duration.zero, this.change = PointerChange.cancel, this.kind = PointerDeviceKind.touch, @@ -102,6 +103,13 @@ class PointerData { this.scrollDeltaY = 0.0, }); + /// Unique identifier that ties the [PointerEvent] to embedder event created it. + /// + /// No two pointer events can have the same [embedderId]. This is different from + /// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to + /// identify the platform event. + final int embedderId; + /// Time of event dispatch, relative to an arbitrary timeline. final Duration timeStamp; @@ -263,6 +271,7 @@ class PointerData { /// Returns a complete textual description of the information in this object. String toStringFull() { return '$runtimeType(' + 'embedderId: $embedderId, ' 'timeStamp: $timeStamp, ' 'change: $change, ' 'kind: $kind, ' diff --git a/lib/ui/window/pointer_data.h b/lib/ui/window/pointer_data.h index 96a9e7d5dad91..124f35e94e216 100644 --- a/lib/ui/window/pointer_data.h +++ b/lib/ui/window/pointer_data.h @@ -10,7 +10,7 @@ namespace flutter { // If this value changes, update the pointer data unpacking code in hooks.dart. -static constexpr int kPointerDataFieldCount = 28; +static constexpr int kPointerDataFieldCount = 29; static constexpr int kBytesPerField = sizeof(int64_t); // Must match the button constants in events.dart. enum PointerButtonMouse : int64_t { @@ -58,6 +58,7 @@ struct alignas(8) PointerData { kScroll, }; + int64_t embedder_id; int64_t time_stamp; Change change; DeviceKind kind; diff --git a/lib/web_ui/lib/src/ui/pointer.dart b/lib/web_ui/lib/src/ui/pointer.dart index d541aba19f2d8..698badcc473ed 100644 --- a/lib/web_ui/lib/src/ui/pointer.dart +++ b/lib/web_ui/lib/src/ui/pointer.dart @@ -71,6 +71,7 @@ enum PointerSignalKind { class PointerData { /// Creates an object that represents the state of a pointer. const PointerData({ + this.embedderId = 0, this.timeStamp = Duration.zero, this.change = PointerChange.cancel, this.kind = PointerDeviceKind.touch, @@ -101,6 +102,13 @@ class PointerData { this.scrollDeltaY = 0.0, }); + /// Unique identifier that ties the [PointerEvent] to embedder event created it. + /// + /// No two pointer events can have the same [embedderId]. This is different from + /// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to + /// identify the platform event. + final int embedderId; + /// Time of event dispatch, relative to an arbitrary timeline. final Duration timeStamp; @@ -257,46 +265,47 @@ class PointerData { final double scrollDeltaY; @override - String toString() => '$runtimeType(x: $physicalX, y: $physicalY)'; + String toString() => 'PointerData(x: $physicalX, y: $physicalY)'; /// Returns a complete textual description of the information in this object. String toStringFull() { return '$runtimeType(' - 'timeStamp: $timeStamp, ' - 'change: $change, ' - 'kind: $kind, ' - 'signalKind: $signalKind, ' - 'device: $device, ' - 'pointerIdentifier: $pointerIdentifier, ' - 'physicalX: $physicalX, ' - 'physicalY: $physicalY, ' - 'physicalDeltaX: $physicalDeltaX, ' - 'physicalDeltaY: $physicalDeltaY, ' - 'buttons: $buttons, ' - 'synthesized: $synthesized, ' - 'pressure: $pressure, ' - 'pressureMin: $pressureMin, ' - 'pressureMax: $pressureMax, ' - 'distance: $distance, ' - 'distanceMax: $distanceMax, ' - 'size: $size, ' - 'radiusMajor: $radiusMajor, ' - 'radiusMinor: $radiusMinor, ' - 'radiusMin: $radiusMin, ' - 'radiusMax: $radiusMax, ' - 'orientation: $orientation, ' - 'tilt: $tilt, ' - 'platformData: $platformData, ' - 'scrollDeltaX: $scrollDeltaX, ' - 'scrollDeltaY: $scrollDeltaY' - ')'; + 'embedderId: $embedderId, ' + 'timeStamp: $timeStamp, ' + 'change: $change, ' + 'kind: $kind, ' + 'signalKind: $signalKind, ' + 'device: $device, ' + 'pointerIdentifier: $pointerIdentifier, ' + 'physicalX: $physicalX, ' + 'physicalY: $physicalY, ' + 'physicalDeltaX: $physicalDeltaX, ' + 'physicalDeltaY: $physicalDeltaY, ' + 'buttons: $buttons, ' + 'synthesized: $synthesized, ' + 'pressure: $pressure, ' + 'pressureMin: $pressureMin, ' + 'pressureMax: $pressureMax, ' + 'distance: $distance, ' + 'distanceMax: $distanceMax, ' + 'size: $size, ' + 'radiusMajor: $radiusMajor, ' + 'radiusMinor: $radiusMinor, ' + 'radiusMin: $radiusMin, ' + 'radiusMax: $radiusMax, ' + 'orientation: $orientation, ' + 'tilt: $tilt, ' + 'platformData: $platformData, ' + 'scrollDeltaX: $scrollDeltaX, ' + 'scrollDeltaY: $scrollDeltaY' + ')'; } } /// A sequence of reports about the state of pointers. class PointerDataPacket { /// Creates a packet of pointer data reports. - const PointerDataPacket({this.data = const []}) : assert(data != null); // ignore: unnecessary_null_comparison + const PointerDataPacket({ this.data = const [] }) : assert(data != null); // ignore: unnecessary_null_comparison /// Data about the individual pointers in this packet. /// diff --git a/shell/platform/android/BUILD.gn b/shell/platform/android/BUILD.gn index 70e77a9bc0584..da29ead5521d4 100644 --- a/shell/platform/android/BUILD.gn +++ b/shell/platform/android/BUILD.gn @@ -140,6 +140,7 @@ android_java_sources = [ "io/flutter/embedding/android/FlutterSurfaceView.java", "io/flutter/embedding/android/FlutterTextureView.java", "io/flutter/embedding/android/FlutterView.java", + "io/flutter/embedding/android/MotionEventTracker.java", "io/flutter/embedding/android/RenderMode.java", "io/flutter/embedding/android/SplashScreen.java", "io/flutter/embedding/android/SplashScreenProvider.java", diff --git a/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java b/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java index 027b13c53d274..a772d14a88c94 100644 --- a/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java +++ b/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java @@ -57,7 +57,7 @@ public class AndroidTouchProcessor { } // Must match the unpacking code in hooks.dart. - private static final int POINTER_DATA_FIELD_COUNT = 28; + private static final int POINTER_DATA_FIELD_COUNT = 29; private static final int BYTES_PER_FIELD = 8; // This value must match the value in framework's platform_view.dart. @@ -65,6 +65,7 @@ public class AndroidTouchProcessor { private static final int POINTER_DATA_FLAG_BATCHED = 1; @NonNull private final FlutterRenderer renderer; + @NonNull private final MotionEventTracker motionEventTracker; private static final int _POINTER_BUTTON_PRIMARY = 1; @@ -76,6 +77,7 @@ public class AndroidTouchProcessor { // FlutterRenderer public AndroidTouchProcessor(@NonNull FlutterRenderer renderer) { this.renderer = renderer; + this.motionEventTracker = MotionEventTracker.getInstance(); } /** Sends the given {@link MotionEvent} data to Flutter in a format that Flutter understands. */ @@ -174,6 +176,8 @@ private void addPointerForIndex( return; } + MotionEventTracker.MotionEventId motionEventId = motionEventTracker.track(event); + int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex)); int signalKind = @@ -183,6 +187,7 @@ private void addPointerForIndex( long timeStamp = event.getEventTime() * 1000; // Convert from milliseconds to microseconds. + packet.putLong(motionEventId.getId()); packet.putLong(timeStamp); // time_stamp packet.putLong(pointerChange); // change packet.putLong(pointerKind); // kind diff --git a/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java b/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java new file mode 100644 index 0000000000000..27d3dcff20622 --- /dev/null +++ b/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java @@ -0,0 +1,79 @@ +package io.flutter.embedding.android; + +import android.util.LongSparseArray; +import android.view.MotionEvent; +import androidx.annotation.Nullable; +import java.util.PriorityQueue; +import java.util.concurrent.atomic.AtomicLong; + +/** Tracks the motion events received by the FlutterView. */ +public final class MotionEventTracker { + + /** Represents a unique identifier corresponding to a motion event. */ + public static class MotionEventId { + private static final AtomicLong ID_COUNTER = new AtomicLong(0); + private final long id; + + private MotionEventId(long id) { + this.id = id; + } + + public static MotionEventId from(long id) { + return new MotionEventId(id); + } + + public static MotionEventId createUnique() { + return MotionEventId.from(ID_COUNTER.incrementAndGet()); + } + + public long getId() { + return id; + } + } + + private final LongSparseArray eventById; + private final PriorityQueue unusedEvents; + private static MotionEventTracker INSTANCE; + + public static MotionEventTracker getInstance() { + if (INSTANCE == null) { + INSTANCE = new MotionEventTracker(); + } + return INSTANCE; + } + + private MotionEventTracker() { + eventById = new LongSparseArray<>(); + unusedEvents = new PriorityQueue<>(); + } + + /** Tracks the event and returns a unique MotionEventId identifying the event. */ + public MotionEventId track(MotionEvent event) { + MotionEventId eventId = MotionEventId.createUnique(); + eventById.put(eventId.id, event); + unusedEvents.add(eventId.id); + return eventId; + } + + /** + * Returns the MotionEvent corresponding to the eventId while discarding all the motion events + * that occured prior to the event represented by the eventId. Returns null if this event was + * popped or discarded. + */ + @Nullable + public MotionEvent pop(MotionEventId eventId) { + // remove all the older events. + while (!unusedEvents.isEmpty() && unusedEvents.peek() < eventId.id) { + eventById.remove(unusedEvents.poll()); + } + + // remove the current event from the heap if it exists. + if (!unusedEvents.isEmpty() && unusedEvents.peek() == eventId.id) { + unusedEvents.poll(); + } + + MotionEvent event = eventById.get(eventId.id); + eventById.remove(eventId.id); + return event; + } +} diff --git a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java index f5b5e51fdf873..3f238c98d99a0 100644 --- a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java +++ b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java @@ -166,7 +166,8 @@ private void touch(@NonNull MethodCall call, @NonNull MethodChannel.Result resul (int) args.get(11), (int) args.get(12), (int) args.get(13), - (int) args.get(14)); + (int) args.get(14), + ((Number) args.get(15)).longValue()); try { handler.onTouch(touch); @@ -380,6 +381,8 @@ public static class PlatformViewTouch { public final int source; /** TODO(mattcarroll): javadoc */ public final int flags; + /** TODO(iskakaushik): javadoc */ + public final long motionEventId; PlatformViewTouch( int viewId, @@ -396,7 +399,8 @@ public static class PlatformViewTouch { int deviceId, int edgeFlags, int source, - int flags) { + int flags, + long motionEventId) { this.viewId = viewId; this.downTime = downTime; this.eventTime = eventTime; @@ -412,6 +416,7 @@ public static class PlatformViewTouch { this.edgeFlags = edgeFlags; this.source = source; this.flags = flags; + this.motionEventId = motionEventId; } } } diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index 61ceab005a147..3929698a8ef64 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -21,6 +21,7 @@ import androidx.annotation.VisibleForTesting; import io.flutter.embedding.android.FlutterImageView; import io.flutter.embedding.android.FlutterView; +import io.flutter.embedding.android.MotionEventTracker; import io.flutter.embedding.engine.FlutterOverlaySurface; import io.flutter.embedding.engine.dart.DartExecutor; import io.flutter.embedding.engine.mutatorsstack.*; @@ -93,6 +94,9 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega // Platform view IDs that were displayed since the start of the current frame. private HashSet currentFrameUsedPlatformViewIds; + // Used to acquire the original motion events using the motionEventIds. + private final MotionEventTracker motionEventTracker; + private final PlatformViewsChannel.PlatformViewsHandler channelHandler = new PlatformViewsChannel.PlatformViewsHandler() { @@ -301,8 +305,16 @@ private void ensureValidAndroidVersion(int minSdkVersion) { } }; - private static MotionEvent toMotionEvent( - float density, PlatformViewsChannel.PlatformViewTouch touch) { + private MotionEvent toMotionEvent(float density, PlatformViewsChannel.PlatformViewTouch touch) { + MotionEventTracker.MotionEventId motionEventId = + MotionEventTracker.MotionEventId.from(touch.motionEventId); + MotionEvent trackedEvent = motionEventTracker.pop(motionEventId); + if (trackedEvent != null) { + return trackedEvent; + } + + // TODO (kaushikiska) : warn that we are potentially using an untracked + // event in the platform views. PointerProperties[] pointerProperties = parsePointerPropertiesList(touch.rawPointerPropertiesList) .toArray(new PointerProperties[touch.pointerCount]); @@ -339,6 +351,8 @@ public PlatformViewsController() { platformViewRequests = new SparseArray<>(); platformViews = new SparseArray<>(); mutatorViews = new SparseArray<>(); + + motionEventTracker = MotionEventTracker.getInstance(); } /** diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index 6e2f5bcf60ca6..c9b3589182e1f 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -1225,6 +1225,8 @@ FlutterEngineResult FlutterEngineSendPointerEvent( for (size_t i = 0; i < events_count; ++i) { flutter::PointerData pointer_data; pointer_data.Clear(); + // this is currely in use only on android embedding. + pointer_data.embedder_id = 0; pointer_data.time_stamp = SAFE_ACCESS(current, timestamp, 0); pointer_data.change = ToPointerDataChange( SAFE_ACCESS(current, phase, FlutterPointerPhase::kCancel));