Skip to content

Commit

Permalink
Inject display-related events to virtual display
Browse files Browse the repository at this point in the history 
Mouse and touch events must be sent to the virtual display id (used for
mirroring), other events (like key events) must be sent to the original
display id.

Fixes #4598 <#4598>
Fixes #5137 <#5137>
PR #5370 <#5370>

Co-authored-by: nightmare <mengyanshou@gmail.com>
  • Loading branch information
@rom1v @mengyanshou
rom1v and mengyanshou committed Oct 27, 2024
1 parent d774f63 commit 99b2bb3
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 31 deletions.
74 changes: 46 additions & 28 deletions server/src/main/java/com/genymobile/scrcpy/control/Controller.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import android.os.Build;
import android.os.SystemClock;
import android.view.InputDevice;
import android.view.InputEvent;
import android.view.KeyCharacterMap;
import android.view.KeyEvent;
import android.view.MotionEvent;
Expand All @@ -31,6 +30,28 @@

public class Controller implements AsyncProcessor, VirtualDisplayListener {

/*
* For event injection, there are two display ids:
* - the displayId passed to the constructor (which comes from --display-id passed by the client, 0 for the main display);
* - the virtualDisplayId used for mirroring, notified by the capture instance via the VirtualDisplayListener interface.
*
* (In case the ScreenCapture uses the "SurfaceControl API", then both ids are equals, but this is an implementation detail.)
*
* In order to make events work correctly in all cases:
* - virtualDisplayId must be used for events relative to the display (mouse and touch events with coordinates);
* - displayId must be used for other events (like key events).
*/

private static final class DisplayData {
private final int virtualDisplayId;
private final PositionMapper positionMapper;

private DisplayData(int virtualDisplayId, PositionMapper positionMapper) {
this.virtualDisplayId = virtualDisplayId;
this.positionMapper = positionMapper;
}
}

private static final int DEFAULT_DEVICE_ID = 0;

// control_msg.h values of the pointerId field in inject_touch_event message
Expand All @@ -54,7 +75,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {

private final AtomicBoolean isSettingClipboard = new AtomicBoolean();

private final AtomicReference<PositionMapper> positionMapper = new AtomicReference<>();
private final AtomicReference<DisplayData> displayData = new AtomicReference<>();

private long lastTouchDown;
private final PointersState pointersState = new PointersState();
Expand Down Expand Up @@ -102,8 +123,9 @@ public void dispatchPrimaryClipChanged() {
}

@Override
public void onNewVirtualDisplay(PositionMapper positionMapper) {
this.positionMapper.set(positionMapper);
public void onNewVirtualDisplay(int virtualDisplayId, PositionMapper positionMapper) {
DisplayData data = new DisplayData(virtualDisplayId, positionMapper);
this.displayData.set(data);
}

private UhidManager getUhidManager() {
Expand Down Expand Up @@ -284,7 +306,7 @@ private boolean injectChar(char c) {
return false;
}
for (KeyEvent event : events) {
if (!injectEvent(event, Device.INJECT_MODE_ASYNC)) {
if (!Device.injectEvent(event, displayId, Device.INJECT_MODE_ASYNC)) {
return false;
}
}
Expand All @@ -306,7 +328,12 @@ private int injectText(String text) {
private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) {
long now = SystemClock.uptimeMillis();

Point point = getPhysicalPoint(position);
// it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
DisplayData displayData = this.displayData.get();
assert displayData != null : "Cannot receive a touch event without a display";

Point point = displayData.positionMapper.map(position);
if (point == null) {
Ln.w("Ignore touch event, it was generated for a different device size");
return false;
Expand Down Expand Up @@ -365,7 +392,7 @@ private boolean injectTouch(int action, long pointerId, Position position, float
// First button pressed: ACTION_DOWN
MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) {
if (!Device.injectEvent(downEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false;
}
}
Expand All @@ -376,7 +403,7 @@ private boolean injectTouch(int action, long pointerId, Position position, float
if (!InputManager.setActionButton(pressEvent, actionButton)) {
return false;
}
if (!injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) {
if (!Device.injectEvent(pressEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false;
}

Expand All @@ -390,15 +417,15 @@ private boolean injectTouch(int action, long pointerId, Position position, float
if (!InputManager.setActionButton(releaseEvent, actionButton)) {
return false;
}
if (!injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) {
if (!Device.injectEvent(releaseEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false;
}

if (buttons == 0) {
// Last button released: ACTION_UP
MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) {
if (!Device.injectEvent(upEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false;
}
}
Expand All @@ -409,12 +436,18 @@ private boolean injectTouch(int action, long pointerId, Position position, float

MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,
DEFAULT_DEVICE_ID, 0, source, 0);
return injectEvent(event, Device.INJECT_MODE_ASYNC);
return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);
}

private boolean injectScroll(Position position, float hScroll, float vScroll, int buttons) {
long now = SystemClock.uptimeMillis();
Point point = getPhysicalPoint(position);

// it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
DisplayData displayData = this.displayData.get();
assert displayData != null : "Cannot receive a scroll event without a display";

Point point = displayData.positionMapper.map(position);
if (point == null) {
Ln.w("Ignore scroll event, it was generated for a different device size");
return false;
Expand All @@ -431,18 +464,7 @@ private boolean injectScroll(Position position, float hScroll, float vScroll, in

MotionEvent event = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,
DEFAULT_DEVICE_ID, 0, InputDevice.SOURCE_MOUSE, 0);
return injectEvent(event, Device.INJECT_MODE_ASYNC);
}

private Point getPhysicalPoint(Position position) {
// it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
PositionMapper positionMapper = this.positionMapper.get();
if (positionMapper == null) {
return null;
}

return positionMapper.map(position);
return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);
}

/**
Expand Down Expand Up @@ -520,10 +542,6 @@ private void openHardKeyboardSettings() {
ServiceManager.getActivityManager().startActivity(intent);
}

private boolean injectEvent(InputEvent event, int injectMode) {
return Device.injectEvent(event, displayId, injectMode);
}

private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int injectMode) {
return Device.injectKeyEvent(action, keyCode, repeat, metaState, displayId, injectMode);
}
Expand Down
11 changes: 9 additions & 2 deletions server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,16 @@ public void start(Surface surface) {
virtualDisplay = null;
}

int virtualDisplayId;
PositionMapper positionMapper;
try {
Size videoSize = screenInfo.getVideoSize();
virtualDisplay = ServiceManager.getDisplayManager()
.createVirtualDisplay("scrcpy", videoSize.getWidth(), videoSize.getHeight(), displayId, surface);
virtualDisplayId = virtualDisplay.getDisplay().getDisplayId();
Rect contentRect = new Rect(0, 0, videoSize.getWidth(), videoSize.getHeight());
// The position are relative to the virtual display, not the original display
positionMapper = new PositionMapper(videoSize, contentRect, 0);
Ln.d("Display: using DisplayManager API");
} catch (Exception displayManagerException) {
try {
Expand All @@ -123,6 +129,8 @@ public void start(Surface surface) {
int layerStack = displayInfo.getLayerStack();

setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
virtualDisplayId = displayId;
positionMapper = PositionMapper.from(screenInfo);
Ln.d("Display: using SurfaceControl API");
} catch (Exception surfaceControlException) {
Ln.e("Could not create display using DisplayManager", displayManagerException);
Expand All @@ -132,8 +140,7 @@ public void start(Surface surface) {
}

if (vdListener != null) {
PositionMapper positionMapper = PositionMapper.from(screenInfo);
vdListener.onNewVirtualDisplay(positionMapper);
vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper);
}
}

Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
import com.genymobile.scrcpy.control.PositionMapper;

public interface VirtualDisplayListener {
void onNewVirtualDisplay(PositionMapper positionMapper);
void onNewVirtualDisplay(int displayId, PositionMapper positionMapper);
}

0 comments on commit 99b2bb3

Please sign in to comment.