From 48dcab57abfb4b6c5361ce4ed230c55068a5a8cc Mon Sep 17 00:00:00 2001
From: Sergey Stepanov <sergystepanov@gmail.com>
Date: Wed, 28 Feb 2024 23:26:21 +0300
Subject: [PATCH] Scale cursor speed to native resolution

Try to scale pointer coordinate deltas from a display with one resolution in the browser to a display (frame size) of the emulator.

When you stream a display which size is different from a display on which you use mouse,
the mouse movement (DPI) should be recalculated accordingly.
Used algorithm takes coordinates from the virtual display (cursor over the browser video element) and scales them to the ratio of virtual/real display sizes.
Since the scale ratio value is a floating point number and we can't use subpixel coordinates on the server side, we need to take into account accumulated floating point errors when rounding coordinates to the destination screen (Libretro framebuffer).

See an example: https://codepen.io/sergystepanov/full/MWRWEMY
---
 pkg/api/worker.go                 |  1 +
 pkg/worker/caged/app/app.go       |  1 +
 pkg/worker/coordinatorhandlers.go | 15 ++++++----
 web/js/stream/stream.js           | 46 +++++++++++++++++++++++++------
 4 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/pkg/api/worker.go b/pkg/api/worker.go
index 045fa4296..2b2fc4ddd 100644
--- a/pkg/api/worker.go
+++ b/pkg/api/worker.go
@@ -64,6 +64,7 @@ type (
 	AppVideoInfo struct {
 		W int     `json:"w"`
 		H int     `json:"h"`
+		S int     `json:"s"`
 		A float32 `json:"a"`
 	}
 )
diff --git a/pkg/worker/caged/app/app.go b/pkg/worker/caged/app/app.go
index fc37827eb..b665c48dd 100644
--- a/pkg/worker/caged/app/app.go
+++ b/pkg/worker/caged/app/app.go
@@ -6,6 +6,7 @@ type App interface {
 	AspectEnabled() bool
 	Init() error
 	ViewportSize() (int, int)
+	Scale() float64
 	Start()
 	Close()
 
diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go
index c37e2377b..27340aef9 100644
--- a/pkg/worker/coordinatorhandlers.go
+++ b/pkg/worker/coordinatorhandlers.go
@@ -126,11 +126,14 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
 				}
 			}
 
-			data, err := api.Wrap(api.Out{T: uint8(api.AppVideoChange), Payload: api.AppVideoInfo{
-				W: m.VideoW,
-				H: m.VideoH,
-				A: app.AspectRatio(),
-			}})
+			data, err := api.Wrap(api.Out{
+				T: uint8(api.AppVideoChange),
+				Payload: api.AppVideoInfo{
+					W: m.VideoW,
+					H: m.VideoH,
+					A: app.AspectRatio(),
+					S: int(app.Scale()),
+				}})
 			if err != nil {
 				c.log.Error().Err(err).Msgf("wrap")
 			}
@@ -179,7 +182,7 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
 	response := api.StartGameResponse{Room: api.Room{Rid: r.Id()}, Record: w.conf.Recording.Enabled}
 	if r.App().AspectEnabled() {
 		ww, hh := r.App().ViewportSize()
-		response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio()}
+		response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio(), S: int(r.App().Scale())}
 	}
 
 	return api.Out{Payload: response}
diff --git a/web/js/stream/stream.js b/web/js/stream/stream.js
index 5659663fd..6993bab34 100644
--- a/web/js/stream/stream.js
+++ b/web/js/stream/stream.js
@@ -179,7 +179,9 @@ const stream = (() => {
 
             if (fullscreen && !pointerLocked) {
                 // event.pub(POINTER_LOCK_CHANGE, screen);
-                await screen.requestPointerLock();
+                await screen.requestPointerLock(
+                    // { unadjustedMovement: true,}
+                );
             }
 
             screen.onpointerdown = fullscreen ? handlePointerDown : null;
@@ -188,8 +190,32 @@ const stream = (() => {
             // !to flipped
         })
 
+        let ex = 0, ey = 0;
+        const scaleCursorPos = (x, y) => {
+            const horizontal = screen.videoWidth > screen.videoHeight;
+
+            const arW = horizontal ? screen.offsetHeight * state.aspect : screen.offsetHeight;
+            const arH = horizontal ? screen.offsetHeight : screen.offsetWidth * state.aspect;
+            const sw = arW / screen.videoWidth;
+            const sh = arH / screen.videoHeight;
+
+            const rez = {
+                dx: x / sw + ex,
+                dy: y / sh + ey
+            }
+
+            ex = rez.dx % 1;
+            ey = rez.dy % 1;
+
+            rez.dx -= ex;
+            rez.dy -= ey;
+
+            return rez;
+        }
+
         const handlePointerMove = (e) => {
-            event.pub(MOUSE_MOVED, {dx: e.movementX, dy: e.movementY});
+            const delta = scaleCursorPos(e.movementX, e.movementY);
+            event.pub(MOUSE_MOVED, delta);
         }
 
         event.sub(POINTER_LOCK_CHANGE, (lockedEl) => {
@@ -201,18 +227,22 @@ const stream = (() => {
         const fit = 'contain'
 
         event.sub(APP_VIDEO_CHANGED, (payload) => {
-            const {w, h, a} = payload
+            const {w, h, a, s} = payload
+
+            const scale = !s ? 1 : s;
+            const ww = w * scale;
+            const hh = h * scale;
 
             state.aspect = a
 
-            const a2 = w / h
+            const a2 = ww / hh
 
             state.screen.style['object-fit'] = a.toFixed(6) !== a2.toFixed(6) ? 'fill' : fit
-            state.h = payload.h
-            state.w = Math.floor(payload.h * payload.a)
+            state.h = hh
+            state.w = Math.floor(hh * a)
             // payload.a > 0 && (state.aspect = payload.a)
-            state.screen.setAttribute('width', payload.w)
-            state.screen.setAttribute('height', payload.h)
+            state.screen.setAttribute('width', ww)
+            state.screen.setAttribute('height', hh)
             state.screen.style.aspectRatio = state.aspect
         })