Add keyboard and mouse support

Keyboard and mouse controls will now work if you use the kbMouseSupport parameter in the config for Libretro cores. Be aware that capturing mouse and keyboard controls properly is only possible in fullscreen mode.

Note: In the case of DOSBox, a virtual filesystem handler is not yet implemented, thus each game state will be shared between all rooms (DOS game instances) of CloudRetro.

pkg/api/user.go

@@ -12,8 +12,9 @@ type (
 		PlayerIndex int    `json:"player_index"`
 	}
 	GameStartUserResponse struct {
-		RoomId string        `json:"roomId"`
-		Av     *AppVideoInfo `json:"av"`
+		RoomId  string        `json:"roomId"`
+		Av      *AppVideoInfo `json:"av"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	IceServer struct {
 		Urls       string `json:"urls,omitempty"`

pkg/api/worker.go

@@ -33,8 +33,9 @@ type (
 	}
 	StartGameResponse struct {
 		Room
-		AV     *AppVideoInfo `json:"av"`
-		Record bool
+		AV      *AppVideoInfo `json:"av"`
+		Record  bool          `json:"record"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	RecordGameRequest[T Id] struct {
 		StatefulRoom[T]

pkg/config/config.yaml

@@ -185,11 +185,13 @@ emulator:
       #   - isGlAllowed (bool)
       #   - usesLibCo (bool)
       #   - hasMultitap (bool) -- (removed)
-      #   - coreAspectRatio (bool) -- correct the aspect ratio on the client with the info from the core.
+      #   - coreAspectRatio (bool) -- (deprecated) correct the aspect ratio on the client with the info from the core.
       #   - hid (map[int][]int)
       #       A list of device IDs to bind to the input ports.
+      #       Can be seen in human readable form in the console when worker.debug is enabled.
       #       Some cores allow binding multiple devices to a single port (DosBox), but typically,
       #       you should bind just one device to one port.
+      #   - kbMouseSupport (bool) -- (temp) a flag if the core needs the keyboard and mouse on the client
       #   - vfr (bool)
       #    (experimental)
       #       Enable variable frame rate only for cores that can't produce a constant frame rate.
@@ -213,7 +215,6 @@ emulator:
             mgba_audio_low_pass_filter: enabled
             mgba_audio_low_pass_range: 40
         pcsx:
-          coreAspectRatio: true
           lib: pcsx_rearmed_libretro
           roms: [ "cue", "chd" ]
           # example of folder override
@@ -227,7 +228,6 @@ emulator:
         # https://docs.libretro.com/library/fbneo/
         mame:
           lib: fbneo_libretro
-          coreAspectRatio: true
           roms: [ "zip" ]
         nes:
           lib: nestopia_libretro
@@ -280,7 +280,7 @@ encoder:
     # see: https://trac.ffmpeg.org/wiki/Encode/H.264
     h264:
       # Constant Rate Factor (CRF) 0-51 (default: 23)
-      crf: 26
+      crf: 23
       # ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo
       preset: superfast
       # baseline, main, high, high10, high422, high444

pkg/config/emulator.go

@@ -48,6 +48,7 @@ type LibretroCoreConfig struct {
 	Height          int
 	Hid             map[int][]int
 	IsGlAllowed     bool
+	KbMouseSupport  bool
 	Lib             string
 	Options         map[string]string
 	Options4rom     map[string]map[string]string // <(^_^)>

pkg/coordinator/userapi.go

@@ -37,6 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) }
 func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) }
 
 // StartGame signals the user that everything is ready to start a game.
-func (u *User) StartGame(av *api.AppVideoInfo) {
-	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av})
+func (u *User) StartGame(av *api.AppVideoInfo, kbMouse bool) {
+	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av, KbMouse: kbMouse})
 }

pkg/coordinator/userhandlers.go

@@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc
 		return
 	}
 	u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker")
-	u.StartGame(startGameResp.AV)
+	u.StartGame(startGameResp.AV, startGameResp.KbMouse)
 
 	// send back recording status
 	if conf.Recording.Enabled && rq.Record {

pkg/network/webrtc/webrtc.go

@@ -81,11 +81,15 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType)
 	p.a = audio
 
-	// plug in the [data] channel (in and out)
-	if err = p.addDataChannel("data"); err != nil {
+	err = p.AddChannel("data", func(data []byte) {
+		if len(data) == 0 || p.OnMessage == nil {
+			return
+		}
+		p.OnMessage(data)
+	})
+	if err != nil {
 		return "", err
 	}
-	p.log.Debug().Msg("Added [data] chan")
 
 	p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") }))
 	// Stream provider supposes to send offer
@@ -221,6 +225,19 @@ func (p *Peer) AddCandidate(candidate string, decoder Decoder) error {
 	return nil
 }
 
+func (p *Peer) AddChannel(label string, onMessage func([]byte)) error {
+	ch, err := p.addDataChannel(label)
+	if err != nil {
+		return err
+	}
+	if label == "data" {
+		p.d = ch
+	}
+	ch.OnMessage(func(m webrtc.DataChannelMessage) { onMessage(m.Data) })
+	p.log.Debug().Msgf("Added [%v] chan", label)
+	return nil
+}
+
 func (p *Peer) Disconnect() {
 	if p.conn == nil {
 		return
@@ -232,29 +249,19 @@ func (p *Peer) Disconnect() {
 	p.log.Debug().Msg("WebRTC stop")
 }
 
-// addDataChannel creates a new WebRTC data channel for user input.
+// addDataChannel creates new WebRTC data channel.
 // Default params -- ordered: true, negotiated: false.
-func (p *Peer) addDataChannel(label string) error {
+func (p *Peer) addDataChannel(label string) (*webrtc.DataChannel, error) {
 	ch, err := p.conn.CreateDataChannel(label, nil)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	ch.OnOpen(func() {
-		p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).
-			Msg("Data channel [input] opened")
+		p.log.Debug().Uint16("id", *ch.ID()).Msgf("Data channel [%v] opened", ch.Label())
 	})
 	ch.OnError(p.logx)
-	ch.OnMessage(func(m webrtc.DataChannelMessage) {
-		if len(m.Data) == 0 {
-			return
-		}
-		if p.OnMessage != nil {
-			p.OnMessage(m.Data)
-		}
-	})
-	p.d = ch
-	ch.OnClose(func() { p.log.Debug().Msg("Data channel [input] has been closed") })
-	return nil
+	ch.OnClose(func() { p.log.Debug().Msgf("Data channel [%v] has been closed", ch.Label()) })
+	return ch, nil
 }
 
 func (p *Peer) logx(err error) { p.log.Error().Err(err) }

pkg/worker/caged/app/app.go

@@ -13,7 +13,8 @@ type App interface {
 	SetAudioCb(func(Audio))
 	SetVideoCb(func(Video))
 	SetDataCb(func([]byte))
-	SendControl(port int, data []byte)
+	Input(port int, device byte, data []byte)
+	KbMouseSupport() bool
 }
 
 type Audio struct {

pkg/worker/caged/caged.go

@@ -15,6 +15,12 @@ type Manager struct {
 	log  *logger.Logger
 }
 
+const (
+	RetroPad = libretro.RetroPad
+	Keyboard = libretro.Keyboard
+	Mouse    = libretro.Mouse
+)
+
 type ModName string
 
 const Libretro ModName = "libretro"

pkg/worker/caged/libretro/caged.go

@@ -79,15 +79,16 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) {
 	}
 }
 
-func (c *Caged) AspectEnabled() bool               { return c.base.nano.Aspect }
-func (c *Caged) AspectRatio() float32              { return c.base.AspectRatio() }
-func (c *Caged) PixFormat() uint32                 { return c.Emulator.PixFormat() }
-func (c *Caged) Rotation() uint                    { return c.Emulator.Rotation() }
-func (c *Caged) AudioSampleRate() int              { return c.Emulator.AudioSampleRate() }
-func (c *Caged) ViewportSize() (int, int)          { return c.base.ViewportSize() }
-func (c *Caged) Scale() float64                    { return c.Emulator.Scale() }
-func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) }
-func (c *Caged) Start()                            { go c.Emulator.Start() }
-func (c *Caged) SetSaveOnClose(v bool)             { c.base.SaveOnClose = v }
-func (c *Caged) SetSessionId(name string)          { c.base.SetSessionId(name) }
-func (c *Caged) Close()                            { c.Emulator.Close() }
+func (c *Caged) AspectEnabled() bool              { return c.base.nano.Aspect }
+func (c *Caged) AspectRatio() float32             { return c.base.AspectRatio() }
+func (c *Caged) PixFormat() uint32                { return c.Emulator.PixFormat() }
+func (c *Caged) Rotation() uint                   { return c.Emulator.Rotation() }
+func (c *Caged) AudioSampleRate() int             { return c.Emulator.AudioSampleRate() }
+func (c *Caged) ViewportSize() (int, int)         { return c.base.ViewportSize() }
+func (c *Caged) Scale() float64                   { return c.Emulator.Scale() }
+func (c *Caged) Input(p int, d byte, data []byte) { c.base.Input(p, d, data) }
+func (c *Caged) KbMouseSupport() bool             { return c.base.KbMouseSupport() }
+func (c *Caged) Start()                           { go c.Emulator.Start() }
+func (c *Caged) SetSaveOnClose(v bool)            { c.base.SaveOnClose = v }
+func (c *Caged) SetSessionId(name string)         { c.base.SetSessionId(name) }
+func (c *Caged) Close()                           { c.Emulator.Close() }

pkg/worker/caged/libretro/frontend.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"sync"
-	"sync/atomic"
 	"time"
 	"unsafe"
 
@@ -44,15 +43,14 @@ type Emulator interface {
 	// Close will be called when the game is done
 	Close()
 	// Input passes input to the emulator
-	Input(player int, data []byte)
+	Input(player int, device byte, data []byte)
 	// Scale returns set video scale factor
 	Scale() float64
 }
 
 type Frontend struct {
 	conf    config.Emulator
 	done    chan struct{}
-	input   InputState
 	log     *logger.Logger
 	nano    *nanoarch.Nanoarch
 	onAudio func(app.Audio)
@@ -70,21 +68,12 @@ type Frontend struct {
 	SaveOnClose       bool
 }
 
-// InputState stores full controller state.
-// It consists of:
-//   - uint16 button values
-//   - int16 analog stick values
-type (
-	InputState [maxPort]State
-	State      struct {
-		keys uint32
-		axes [dpadAxes]int32
-	}
-)
+type Device byte
 
 const (
-	maxPort  = 4
-	dpadAxes = 4
+	RetroPad = Device(nanoarch.RetroPad)
+	Keyboard = Device(nanoarch.Keyboard)
+	Mouse    = Device(nanoarch.Mouse)
 )
 
 var (
@@ -129,7 +118,6 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) {
 	f := &Frontend{
 		conf:    conf,
 		done:    make(chan struct{}),
-		input:   NewGameSessionInput(),
 		log:     log,
 		onAudio: noAudio,
 		onData:  noData,
@@ -162,6 +150,7 @@ func (f *Frontend) LoadCore(emu string) {
 		Options4rom:     conf.Options4rom,
 		UsesLibCo:       conf.UsesLibCo,
 		CoreAspectRatio: conf.CoreAspectRatio,
+		KbMouseSupport:  conf.KbMouseSupport,
 	}
 	f.mu.Lock()
 	scale := 1.0
@@ -227,8 +216,6 @@ func (f *Frontend) linkNano(nano *nanoarch.Nanoarch) {
 	}
 	f.nano.WaitReady() // start only when nano is available
 
-	f.nano.OnKeyPress = f.input.isKeyPressed
-	f.nano.OnDpad = f.input.isDpadTouched
 	f.nano.OnVideo = f.handleVideo
 	f.nano.OnAudio = f.handleAudio
 	f.nano.OnDup = f.handleDup
@@ -300,8 +287,8 @@ func (f *Frontend) Flipped() bool                 { return f.nano.IsGL() }
 func (f *Frontend) FrameSize() (int, int)         { return f.nano.BaseWidth(), f.nano.BaseHeight() }
 func (f *Frontend) HasSave() bool                 { return os.Exists(f.HashPath()) }
 func (f *Frontend) HashPath() string              { return f.storage.GetSavePath() }
-func (f *Frontend) Input(player int, data []byte) { f.input.setInput(player, data) }
 func (f *Frontend) IsPortrait() bool              { return f.nano.IsPortrait() }
+func (f *Frontend) KbMouseSupport() bool          { return f.nano.KbMouseSupport() }
 func (f *Frontend) LoadGame(path string) error    { return f.nano.LoadGame(path) }
 func (f *Frontend) PixFormat() uint32             { return f.nano.Video.PixFmt.C }
 func (f *Frontend) RestoreGameState() error       { return f.Load() }
@@ -318,6 +305,17 @@ func (f *Frontend) Tick()                         { f.mu.Lock(); f.nano.Run(); f
 func (f *Frontend) ViewportRecalculate()          { f.mu.Lock(); f.vw, f.vh = f.ViewportCalc(); f.mu.Unlock() }
 func (f *Frontend) ViewportSize() (int, int)      { return f.vw, f.vh }
 
+func (f *Frontend) Input(port int, device byte, data []byte) {
+	switch Device(device) {
+	case RetroPad:
+		f.nano.InputRetropad(port, data)
+	case Keyboard:
+		f.nano.InputKeyboard(port, data)
+	case Mouse:
+		f.nano.InputMouse(port, data)
+	}
+}
+
 func (f *Frontend) ViewportCalc() (nw int, nh int) {
 	w, h := f.FrameSize()
 	nw, nh = w, h
@@ -408,24 +406,3 @@ func (f *Frontend) autosave(periodSec int) {
 		}
 	}
 }
-
-func NewGameSessionInput() InputState { return [maxPort]State{} }
-
-// setInput sets input state for some player in a game session.
-func (s *InputState) setInput(player int, data []byte) {
-	atomic.StoreUint32(&s[player].keys, uint32(uint16(data[1])<<8+uint16(data[0])))
-	for i, axes := 0, len(data); i < dpadAxes && i<<1+3 < axes; i++ {
-		axis := i<<1 + 2
-		atomic.StoreInt32(&s[player].axes[i], int32(data[axis+1])<<8+int32(data[axis]))
-	}
-}
-
-// isKeyPressed checks if some button is pressed by any player.
-func (s *InputState) isKeyPressed(port uint, key int) int {
-	return int((atomic.LoadUint32(&s[port].keys) >> uint(key)) & 1)
-}
-
-// isDpadTouched checks if D-pad is used by any player.
-func (s *InputState) isDpadTouched(port uint, axis uint) (shift int16) {
-	return int16(atomic.LoadInt32(&s[port].axes[axis]))
-}

pkg/worker/caged/libretro/frontend_test.go

@@ -86,7 +86,6 @@ func EmulatorMock(room string, system string) *TestFrontend {
 				Path:     os.TempDir(),
 				MainSave: room,
 			},
-			input:       NewGameSessionInput(),
 			done:        make(chan struct{}),
 			th:          conf.Emulator.Threads,
 			log:         l2,
@@ -340,20 +339,6 @@ func TestStateConcurrency(t *testing.T) {
 	}
 }
 
-func TestConcurrentInput(t *testing.T) {
-	var wg sync.WaitGroup
-	state := NewGameSessionInput()
-	events := 1000
-	wg.Add(2 * events)
-
-	for range events {
-		player := rand.IntN(maxPort)
-		go func() { state.setInput(player, []byte{0, 1}); wg.Done() }()
-		go func() { state.isKeyPressed(uint(player), 100); wg.Done() }()
-	}
-	wg.Wait()
-}
-
 func TestStartStop(t *testing.T) {
 	f1 := DefaultFrontend("sushi", sushi.system, sushi.rom)
 	go f1.Start()