Skip to content

Commit

Permalink
Track motion events for reuse post gesture disambiguation (flutter#19484
Browse files Browse the repository at this point in the history
)

This change makes it so that we track all the motion events encountered by `FlutterView` and all of its subviews in the `MotionEventTracker` class, indexed by a unique `MotionEventId`. This identifier is then passed to the Flutter framework as seen in flutter/flutter#60930. Once the gestures take part in gesture disambiguation and are sent back to the engine, we look-up the original motion event using the `MotionEventId` and dispatch it to the platform.

Bug: flutter/flutter#58837
  • Loading branch information
iskakaushik authored and justinmc committed Jul 7, 2020
1 parent f1056b4 commit 01c8ecb
Show file tree
Hide file tree
Showing 11 changed files with 165 additions and 38 deletions.
1 change: 1 addition & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/Flutt
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterSurfaceView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterTextureView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/RenderMode.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreen.java
FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreenProvider.java
Expand Down
5 changes: 3 additions & 2 deletions lib/ui/hooks.dart
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,9 @@ void _invoke3<A1, A2, A3>(void callback(A1 a1, A2 a2, A3 a3)?, Zone zone, A1 arg
// If this value changes, update the encoding code in the following files:
//
// * pointer_data.cc
// * pointers.dart
// * pointer.dart
// * AndroidTouchProcessor.java
const int _kPointerDataFieldCount = 28;
const int _kPointerDataFieldCount = 29;

PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
const int kStride = Int64List.bytesPerElement;
Expand All @@ -300,6 +300,7 @@ PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
for (int i = 0; i < length; ++i) {
int offset = i * _kPointerDataFieldCount;
data.add(PointerData(
embedderId: packet.getInt64(kStride * offset++, _kFakeHostEndian),
timeStamp: Duration(microseconds: packet.getInt64(kStride * offset++, _kFakeHostEndian)),
change: PointerChange.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],
kind: PointerDeviceKind.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],
Expand Down
9 changes: 9 additions & 0 deletions lib/ui/pointer.dart
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ enum PointerSignalKind {
class PointerData {
/// Creates an object that represents the state of a pointer.
const PointerData({
this.embedderId = 0,
this.timeStamp = Duration.zero,
this.change = PointerChange.cancel,
this.kind = PointerDeviceKind.touch,
Expand Down Expand Up @@ -102,6 +103,13 @@ class PointerData {
this.scrollDeltaY = 0.0,
});

/// Unique identifier that ties the [PointerEvent] to embedder event created it.
///
/// No two pointer events can have the same [embedderId]. This is different from
/// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to
/// identify the platform event.
final int embedderId;

/// Time of event dispatch, relative to an arbitrary timeline.
final Duration timeStamp;

Expand Down Expand Up @@ -263,6 +271,7 @@ class PointerData {
/// Returns a complete textual description of the information in this object.
String toStringFull() {
return '$runtimeType('
'embedderId: $embedderId, '
'timeStamp: $timeStamp, '
'change: $change, '
'kind: $kind, '
Expand Down
3 changes: 2 additions & 1 deletion lib/ui/window/pointer_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
namespace flutter {

// If this value changes, update the pointer data unpacking code in hooks.dart.
static constexpr int kPointerDataFieldCount = 28;
static constexpr int kPointerDataFieldCount = 29;
static constexpr int kBytesPerField = sizeof(int64_t);
// Must match the button constants in events.dart.
enum PointerButtonMouse : int64_t {
Expand Down Expand Up @@ -58,6 +58,7 @@ struct alignas(8) PointerData {
kScroll,
};

int64_t embedder_id;
int64_t time_stamp;
Change change;
DeviceKind kind;
Expand Down
69 changes: 39 additions & 30 deletions lib/web_ui/lib/src/ui/pointer.dart
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ enum PointerSignalKind {
class PointerData {
/// Creates an object that represents the state of a pointer.
const PointerData({
this.embedderId = 0,
this.timeStamp = Duration.zero,
this.change = PointerChange.cancel,
this.kind = PointerDeviceKind.touch,
Expand Down Expand Up @@ -101,6 +102,13 @@ class PointerData {
this.scrollDeltaY = 0.0,
});

/// Unique identifier that ties the [PointerEvent] to embedder event created it.
///
/// No two pointer events can have the same [embedderId]. This is different from
/// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to
/// identify the platform event.
final int embedderId;

/// Time of event dispatch, relative to an arbitrary timeline.
final Duration timeStamp;

Expand Down Expand Up @@ -257,46 +265,47 @@ class PointerData {
final double scrollDeltaY;

@override
String toString() => '$runtimeType(x: $physicalX, y: $physicalY)';
String toString() => 'PointerData(x: $physicalX, y: $physicalY)';

/// Returns a complete textual description of the information in this object.
String toStringFull() {
return '$runtimeType('
'timeStamp: $timeStamp, '
'change: $change, '
'kind: $kind, '
'signalKind: $signalKind, '
'device: $device, '
'pointerIdentifier: $pointerIdentifier, '
'physicalX: $physicalX, '
'physicalY: $physicalY, '
'physicalDeltaX: $physicalDeltaX, '
'physicalDeltaY: $physicalDeltaY, '
'buttons: $buttons, '
'synthesized: $synthesized, '
'pressure: $pressure, '
'pressureMin: $pressureMin, '
'pressureMax: $pressureMax, '
'distance: $distance, '
'distanceMax: $distanceMax, '
'size: $size, '
'radiusMajor: $radiusMajor, '
'radiusMinor: $radiusMinor, '
'radiusMin: $radiusMin, '
'radiusMax: $radiusMax, '
'orientation: $orientation, '
'tilt: $tilt, '
'platformData: $platformData, '
'scrollDeltaX: $scrollDeltaX, '
'scrollDeltaY: $scrollDeltaY'
')';
'embedderId: $embedderId, '
'timeStamp: $timeStamp, '
'change: $change, '
'kind: $kind, '
'signalKind: $signalKind, '
'device: $device, '
'pointerIdentifier: $pointerIdentifier, '
'physicalX: $physicalX, '
'physicalY: $physicalY, '
'physicalDeltaX: $physicalDeltaX, '
'physicalDeltaY: $physicalDeltaY, '
'buttons: $buttons, '
'synthesized: $synthesized, '
'pressure: $pressure, '
'pressureMin: $pressureMin, '
'pressureMax: $pressureMax, '
'distance: $distance, '
'distanceMax: $distanceMax, '
'size: $size, '
'radiusMajor: $radiusMajor, '
'radiusMinor: $radiusMinor, '
'radiusMin: $radiusMin, '
'radiusMax: $radiusMax, '
'orientation: $orientation, '
'tilt: $tilt, '
'platformData: $platformData, '
'scrollDeltaX: $scrollDeltaX, '
'scrollDeltaY: $scrollDeltaY'
')';
}
}

/// A sequence of reports about the state of pointers.
class PointerDataPacket {
/// Creates a packet of pointer data reports.
const PointerDataPacket({this.data = const <PointerData>[]}) : assert(data != null); // ignore: unnecessary_null_comparison
const PointerDataPacket({ this.data = const <PointerData>[] }) : assert(data != null); // ignore: unnecessary_null_comparison

/// Data about the individual pointers in this packet.
///
Expand Down
1 change: 1 addition & 0 deletions shell/platform/android/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ android_java_sources = [
"io/flutter/embedding/android/FlutterSurfaceView.java",
"io/flutter/embedding/android/FlutterTextureView.java",
"io/flutter/embedding/android/FlutterView.java",
"io/flutter/embedding/android/MotionEventTracker.java",
"io/flutter/embedding/android/RenderMode.java",
"io/flutter/embedding/android/SplashScreen.java",
"io/flutter/embedding/android/SplashScreenProvider.java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,15 @@ public class AndroidTouchProcessor {
}

// Must match the unpacking code in hooks.dart.
private static final int POINTER_DATA_FIELD_COUNT = 28;
private static final int POINTER_DATA_FIELD_COUNT = 29;
private static final int BYTES_PER_FIELD = 8;

// This value must match the value in framework's platform_view.dart.
// This flag indicates whether the original Android pointer events were batched together.
private static final int POINTER_DATA_FLAG_BATCHED = 1;

@NonNull private final FlutterRenderer renderer;
@NonNull private final MotionEventTracker motionEventTracker;

private static final int _POINTER_BUTTON_PRIMARY = 1;

Expand All @@ -76,6 +77,7 @@ public class AndroidTouchProcessor {
// FlutterRenderer
public AndroidTouchProcessor(@NonNull FlutterRenderer renderer) {
this.renderer = renderer;
this.motionEventTracker = MotionEventTracker.getInstance();
}

/** Sends the given {@link MotionEvent} data to Flutter in a format that Flutter understands. */
Expand Down Expand Up @@ -174,6 +176,8 @@ private void addPointerForIndex(
return;
}

MotionEventTracker.MotionEventId motionEventId = motionEventTracker.track(event);

int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex));

int signalKind =
Expand All @@ -183,6 +187,7 @@ private void addPointerForIndex(

long timeStamp = event.getEventTime() * 1000; // Convert from milliseconds to microseconds.

packet.putLong(motionEventId.getId());
packet.putLong(timeStamp); // time_stamp
packet.putLong(pointerChange); // change
packet.putLong(pointerKind); // kind
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package io.flutter.embedding.android;

import android.util.LongSparseArray;
import android.view.MotionEvent;
import androidx.annotation.Nullable;
import java.util.PriorityQueue;
import java.util.concurrent.atomic.AtomicLong;

/** Tracks the motion events received by the FlutterView. */
public final class MotionEventTracker {

/** Represents a unique identifier corresponding to a motion event. */
public static class MotionEventId {
private static final AtomicLong ID_COUNTER = new AtomicLong(0);
private final long id;

private MotionEventId(long id) {
this.id = id;
}

public static MotionEventId from(long id) {
return new MotionEventId(id);
}

public static MotionEventId createUnique() {
return MotionEventId.from(ID_COUNTER.incrementAndGet());
}

public long getId() {
return id;
}
}

private final LongSparseArray<MotionEvent> eventById;
private final PriorityQueue<Long> unusedEvents;
private static MotionEventTracker INSTANCE;

public static MotionEventTracker getInstance() {
if (INSTANCE == null) {
INSTANCE = new MotionEventTracker();
}
return INSTANCE;
}

private MotionEventTracker() {
eventById = new LongSparseArray<>();
unusedEvents = new PriorityQueue<>();
}

/** Tracks the event and returns a unique MotionEventId identifying the event. */
public MotionEventId track(MotionEvent event) {
MotionEventId eventId = MotionEventId.createUnique();
eventById.put(eventId.id, event);
unusedEvents.add(eventId.id);
return eventId;
}

/**
* Returns the MotionEvent corresponding to the eventId while discarding all the motion events
* that occured prior to the event represented by the eventId. Returns null if this event was
* popped or discarded.
*/
@Nullable
public MotionEvent pop(MotionEventId eventId) {
// remove all the older events.
while (!unusedEvents.isEmpty() && unusedEvents.peek() < eventId.id) {
eventById.remove(unusedEvents.poll());
}

// remove the current event from the heap if it exists.
if (!unusedEvents.isEmpty() && unusedEvents.peek() == eventId.id) {
unusedEvents.poll();
}

MotionEvent event = eventById.get(eventId.id);
eventById.remove(eventId.id);
return event;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ private void touch(@NonNull MethodCall call, @NonNull MethodChannel.Result resul
(int) args.get(11),
(int) args.get(12),
(int) args.get(13),
(int) args.get(14));
(int) args.get(14),
((Number) args.get(15)).longValue());

try {
handler.onTouch(touch);
Expand Down Expand Up @@ -380,6 +381,8 @@ public static class PlatformViewTouch {
public final int source;
/** TODO(mattcarroll): javadoc */
public final int flags;
/** TODO(iskakaushik): javadoc */
public final long motionEventId;

PlatformViewTouch(
int viewId,
Expand All @@ -396,7 +399,8 @@ public static class PlatformViewTouch {
int deviceId,
int edgeFlags,
int source,
int flags) {
int flags,
long motionEventId) {
this.viewId = viewId;
this.downTime = downTime;
this.eventTime = eventTime;
Expand All @@ -412,6 +416,7 @@ public static class PlatformViewTouch {
this.edgeFlags = edgeFlags;
this.source = source;
this.flags = flags;
this.motionEventId = motionEventId;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import androidx.annotation.VisibleForTesting;
import io.flutter.embedding.android.FlutterImageView;
import io.flutter.embedding.android.FlutterView;
import io.flutter.embedding.android.MotionEventTracker;
import io.flutter.embedding.engine.FlutterOverlaySurface;
import io.flutter.embedding.engine.dart.DartExecutor;
import io.flutter.embedding.engine.mutatorsstack.*;
Expand Down Expand Up @@ -93,6 +94,9 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
// Platform view IDs that were displayed since the start of the current frame.
private HashSet<Integer> currentFrameUsedPlatformViewIds;

// Used to acquire the original motion events using the motionEventIds.
private final MotionEventTracker motionEventTracker;

private final PlatformViewsChannel.PlatformViewsHandler channelHandler =
new PlatformViewsChannel.PlatformViewsHandler() {

Expand Down Expand Up @@ -301,8 +305,16 @@ private void ensureValidAndroidVersion(int minSdkVersion) {
}
};

private static MotionEvent toMotionEvent(
float density, PlatformViewsChannel.PlatformViewTouch touch) {
private MotionEvent toMotionEvent(float density, PlatformViewsChannel.PlatformViewTouch touch) {
MotionEventTracker.MotionEventId motionEventId =
MotionEventTracker.MotionEventId.from(touch.motionEventId);
MotionEvent trackedEvent = motionEventTracker.pop(motionEventId);
if (trackedEvent != null) {
return trackedEvent;
}

// TODO (kaushikiska) : warn that we are potentially using an untracked
// event in the platform views.
PointerProperties[] pointerProperties =
parsePointerPropertiesList(touch.rawPointerPropertiesList)
.toArray(new PointerProperties[touch.pointerCount]);
Expand Down Expand Up @@ -339,6 +351,8 @@ public PlatformViewsController() {
platformViewRequests = new SparseArray<>();
platformViews = new SparseArray<>();
mutatorViews = new SparseArray<>();

motionEventTracker = MotionEventTracker.getInstance();
}

/**
Expand Down
2 changes: 2 additions & 0 deletions shell/platform/embedder/embedder.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1225,6 +1225,8 @@ FlutterEngineResult FlutterEngineSendPointerEvent(
for (size_t i = 0; i < events_count; ++i) {
flutter::PointerData pointer_data;
pointer_data.Clear();
// this is currely in use only on android embedding.
pointer_data.embedder_id = 0;
pointer_data.time_stamp = SAFE_ACCESS(current, timestamp, 0);
pointer_data.change = ToPointerDataChange(
SAFE_ACCESS(current, phase, FlutterPointerPhase::kCancel));
Expand Down

0 comments on commit 01c8ecb

Please sign in to comment.