Add keyboard and mouse support

pkg/api/user.go

@@ -12,8 +12,9 @@ type (
 		PlayerIndex int    `json:"player_index"`
 	}
 	GameStartUserResponse struct {
-		RoomId string        `json:"roomId"`
-		Av     *AppVideoInfo `json:"av"`
+		RoomId  string        `json:"roomId"`
+		Av      *AppVideoInfo `json:"av"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	IceServer struct {
 		Urls       string `json:"urls,omitempty"`

pkg/api/worker.go

@@ -33,8 +33,9 @@ type (
 	}
 	StartGameResponse struct {
 		Room
-		AV     *AppVideoInfo `json:"av"`
-		Record bool
+		AV      *AppVideoInfo `json:"av"`
+		Record  bool          `json:"record"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	RecordGameRequest[T Id] struct {
 		StatefulRoom[T]

pkg/config/config.yaml

@@ -185,11 +185,13 @@ emulator:
       #   - isGlAllowed (bool)
       #   - usesLibCo (bool)
       #   - hasMultitap (bool) -- (removed)
-      #   - coreAspectRatio (bool) -- correct the aspect ratio on the client with the info from the core.
+      #   - coreAspectRatio (bool) -- (deprecated) correct the aspect ratio on the client with the info from the core.
       #   - hid (map[int][]int)
       #       A list of device IDs to bind to the input ports.
+      #       Can be seen in human readable form in the console when worker.debug is enabled.
       #       Some cores allow binding multiple devices to a single port (DosBox), but typically,
       #       you should bind just one device to one port.
+      #   - kbMouseSupport (bool) -- (temp) a flag if the core needs the keyboard and mouse on the client
       #   - vfr (bool)
       #    (experimental)
       #       Enable variable frame rate only for cores that can't produce a constant frame rate.
@@ -213,7 +215,6 @@ emulator:
             mgba_audio_low_pass_filter: enabled
             mgba_audio_low_pass_range: 40
         pcsx:
-          coreAspectRatio: true
           lib: pcsx_rearmed_libretro
           roms: [ "cue", "chd" ]
           # example of folder override
@@ -227,7 +228,6 @@ emulator:
         # https://docs.libretro.com/library/fbneo/
         mame:
           lib: fbneo_libretro
-          coreAspectRatio: true
           roms: [ "zip" ]
         nes:
           lib: nestopia_libretro
@@ -280,7 +280,7 @@ encoder:
     # see: https://trac.ffmpeg.org/wiki/Encode/H.264
     h264:
       # Constant Rate Factor (CRF) 0-51 (default: 23)
-      crf: 26
+      crf: 23
       # ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow, placebo
       preset: superfast
       # baseline, main, high, high10, high422, high444

pkg/config/emulator.go

@@ -48,6 +48,7 @@ type LibretroCoreConfig struct {
 	Height          int
 	Hid             map[int][]int
 	IsGlAllowed     bool
+	KbMouseSupport  bool
 	Lib             string
 	Options         map[string]string
 	Options4rom     map[string]map[string]string // <(^_^)>

pkg/coordinator/userapi.go

@@ -37,6 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) }
 func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) }
 
 // StartGame signals the user that everything is ready to start a game.
-func (u *User) StartGame(av *api.AppVideoInfo) {
-	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av})
+func (u *User) StartGame(av *api.AppVideoInfo, kbMouse bool) {
+	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av, KbMouse: kbMouse})
 }

pkg/coordinator/userhandlers.go

@@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc
 		return
 	}
 	u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker")
-	u.StartGame(startGameResp.AV)
+	u.StartGame(startGameResp.AV, startGameResp.KbMouse)
 
 	// send back recording status
 	if conf.Recording.Enabled && rq.Record {

pkg/network/webrtc/webrtc.go

@@ -81,11 +81,15 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType)
 	p.a = audio
 
-	// plug in the [data] channel (in and out)
-	if err = p.addDataChannel("data"); err != nil {
+	err = p.AddChannel("data", func(data []byte) {
+		if len(data) == 0 || p.OnMessage == nil {
+			return
+		}
+		p.OnMessage(data)
+	})
+	if err != nil {
 		return "", err
 	}
-	p.log.Debug().Msg("Added [data] chan")
 
 	p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") }))
 	// Stream provider supposes to send offer
@@ -221,6 +225,19 @@ func (p *Peer) AddCandidate(candidate string, decoder Decoder) error {
 	return nil
 }
 
+func (p *Peer) AddChannel(label string, onMessage func([]byte)) error {
+	ch, err := p.addDataChannel(label)
+	if err != nil {
+		return err
+	}
+	if label == "data" {
+		p.d = ch
+	}
+	ch.OnMessage(func(m webrtc.DataChannelMessage) { onMessage(m.Data) })
+	p.log.Debug().Msgf("Added [%v] chan", label)
+	return nil
+}
+
 func (p *Peer) Disconnect() {
 	if p.conn == nil {
 		return
@@ -232,29 +249,19 @@ func (p *Peer) Disconnect() {
 	p.log.Debug().Msg("WebRTC stop")
 }
 
-// addDataChannel creates a new WebRTC data channel for user input.
+// addDataChannel creates new WebRTC data channel.
 // Default params -- ordered: true, negotiated: false.
-func (p *Peer) addDataChannel(label string) error {
+func (p *Peer) addDataChannel(label string) (*webrtc.DataChannel, error) {
 	ch, err := p.conn.CreateDataChannel(label, nil)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	ch.OnOpen(func() {
-		p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()).
-			Msg("Data channel [input] opened")
+		p.log.Debug().Uint16("id", *ch.ID()).Msgf("Data channel [%v] opened", ch.Label())
 	})
 	ch.OnError(p.logx)
-	ch.OnMessage(func(m webrtc.DataChannelMessage) {
-		if len(m.Data) == 0 {
-			return
-		}
-		if p.OnMessage != nil {
-			p.OnMessage(m.Data)
-		}
-	})
-	p.d = ch
-	ch.OnClose(func() { p.log.Debug().Msg("Data channel [input] has been closed") })
-	return nil
+	ch.OnClose(func() { p.log.Debug().Msgf("Data channel [%v] has been closed", ch.Label()) })
+	return ch, nil
 }
 
 func (p *Peer) logx(err error) { p.log.Error().Err(err) }

pkg/worker/caged/app/app.go

@@ -13,7 +13,8 @@ type App interface {
 	SetAudioCb(func(Audio))
 	SetVideoCb(func(Video))
 	SetDataCb(func([]byte))
-	SendControl(port int, data []byte)
+	Input(port int, device byte, data []byte)
+	KbMouseSupport() bool
 }
 
 type Audio struct {

pkg/worker/caged/caged.go

@@ -15,6 +15,12 @@ type Manager struct {
 	log  *logger.Logger
 }
 
+const (
+	RetroPad = libretro.RetroPad
+	Keyboard = libretro.Keyboard
+	Mouse    = libretro.Mouse
+)
+
 type ModName string
 
 const Libretro ModName = "libretro"

pkg/worker/caged/libretro/caged.go

@@ -79,15 +79,16 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) {
 	}
 }
 
-func (c *Caged) AspectEnabled() bool               { return c.base.nano.Aspect }
-func (c *Caged) AspectRatio() float32              { return c.base.AspectRatio() }
-func (c *Caged) PixFormat() uint32                 { return c.Emulator.PixFormat() }
-func (c *Caged) Rotation() uint                    { return c.Emulator.Rotation() }
-func (c *Caged) AudioSampleRate() int              { return c.Emulator.AudioSampleRate() }
-func (c *Caged) ViewportSize() (int, int)          { return c.base.ViewportSize() }
-func (c *Caged) Scale() float64                    { return c.Emulator.Scale() }
-func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) }
-func (c *Caged) Start()                            { go c.Emulator.Start() }
-func (c *Caged) SetSaveOnClose(v bool)             { c.base.SaveOnClose = v }
-func (c *Caged) SetSessionId(name string)          { c.base.SetSessionId(name) }
-func (c *Caged) Close()                            { c.Emulator.Close() }
+func (c *Caged) AspectEnabled() bool              { return c.base.nano.Aspect }
+func (c *Caged) AspectRatio() float32             { return c.base.AspectRatio() }
+func (c *Caged) PixFormat() uint32                { return c.Emulator.PixFormat() }
+func (c *Caged) Rotation() uint                   { return c.Emulator.Rotation() }
+func (c *Caged) AudioSampleRate() int             { return c.Emulator.AudioSampleRate() }
+func (c *Caged) ViewportSize() (int, int)         { return c.base.ViewportSize() }
+func (c *Caged) Scale() float64                   { return c.Emulator.Scale() }
+func (c *Caged) Input(p int, d byte, data []byte) { c.base.Input(p, d, data) }
+func (c *Caged) KbMouseSupport() bool             { return c.base.KbMouseSupport() }
+func (c *Caged) Start()                           { go c.Emulator.Start() }
+func (c *Caged) SetSaveOnClose(v bool)            { c.base.SaveOnClose = v }
+func (c *Caged) SetSessionId(name string)         { c.base.SetSessionId(name) }
+func (c *Caged) Close()                           { c.Emulator.Close() }

pkg/worker/caged/libretro/frontend.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"sync"
-	"sync/atomic"
 	"time"
 	"unsafe"
 
@@ -44,15 +43,14 @@ type Emulator interface {
 	// Close will be called when the game is done
 	Close()
 	// Input passes input to the emulator
-	Input(player int, data []byte)
+	Input(player int, device byte, data []byte)
 	// Scale returns set video scale factor
 	Scale() float64
 }
 
 type Frontend struct {
 	conf    config.Emulator
 	done    chan struct{}
-	input   InputState
 	log     *logger.Logger
 	nano    *nanoarch.Nanoarch
 	onAudio func(app.Audio)
@@ -70,21 +68,12 @@ type Frontend struct {
 	SaveOnClose       bool
 }
 
-// InputState stores full controller state.
-// It consists of:
-//   - uint16 button values
-//   - int16 analog stick values
-type (
-	InputState [maxPort]State
-	State      struct {
-		keys uint32
-		axes [dpadAxes]int32
-	}
-)
+type Device byte
 
 const (
-	maxPort  = 4
-	dpadAxes = 4
+	RetroPad = Device(nanoarch.RetroPad)
+	Keyboard = Device(nanoarch.Keyboard)
+	Mouse    = Device(nanoarch.Mouse)
 )
 
 var (
@@ -129,7 +118,6 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) {
 	f := &Frontend{
 		conf:    conf,
 		done:    make(chan struct{}),
-		input:   NewGameSessionInput(),
 		log:     log,
 		onAudio: noAudio,
 		onData:  noData,
@@ -162,6 +150,7 @@ func (f *Frontend) LoadCore(emu string) {
 		Options4rom:     conf.Options4rom,
 		UsesLibCo:       conf.UsesLibCo,
 		CoreAspectRatio: conf.CoreAspectRatio,
+		KbMouseSupport:  conf.KbMouseSupport,
 	}
 	f.mu.Lock()
 	scale := 1.0
@@ -227,8 +216,6 @@ func (f *Frontend) linkNano(nano *nanoarch.Nanoarch) {
 	}
 	f.nano.WaitReady() // start only when nano is available
 
-	f.nano.OnKeyPress = f.input.isKeyPressed
-	f.nano.OnDpad = f.input.isDpadTouched
 	f.nano.OnVideo = f.handleVideo
 	f.nano.OnAudio = f.handleAudio
 	f.nano.OnDup = f.handleDup
@@ -300,8 +287,8 @@ func (f *Frontend) Flipped() bool                 { return f.nano.IsGL() }
 func (f *Frontend) FrameSize() (int, int)         { return f.nano.BaseWidth(), f.nano.BaseHeight() }
 func (f *Frontend) HasSave() bool                 { return os.Exists(f.HashPath()) }
 func (f *Frontend) HashPath() string              { return f.storage.GetSavePath() }
-func (f *Frontend) Input(player int, data []byte) { f.input.setInput(player, data) }
 func (f *Frontend) IsPortrait() bool              { return f.nano.IsPortrait() }
+func (f *Frontend) KbMouseSupport() bool          { return f.nano.KbMouseSupport() }
 func (f *Frontend) LoadGame(path string) error    { return f.nano.LoadGame(path) }
 func (f *Frontend) PixFormat() uint32             { return f.nano.Video.PixFmt.C }
 func (f *Frontend) RestoreGameState() error       { return f.Load() }
@@ -318,6 +305,17 @@ func (f *Frontend) Tick()                         { f.mu.Lock(); f.nano.Run(); f
 func (f *Frontend) ViewportRecalculate()          { f.mu.Lock(); f.vw, f.vh = f.ViewportCalc(); f.mu.Unlock() }
 func (f *Frontend) ViewportSize() (int, int)      { return f.vw, f.vh }
 
+func (f *Frontend) Input(port int, device byte, data []byte) {
+	switch Device(device) {
+	case RetroPad:
+		f.nano.InputRetropad(port, data)
+	case Keyboard:
+		f.nano.InputKeyboard(port, data)
+	case Mouse:
+		f.nano.InputMouse(port, data)
+	}
+}
+
 func (f *Frontend) ViewportCalc() (nw int, nh int) {
 	w, h := f.FrameSize()
 	nw, nh = w, h
@@ -408,24 +406,3 @@ func (f *Frontend) autosave(periodSec int) {
 		}
 	}
 }
-
-func NewGameSessionInput() InputState { return [maxPort]State{} }
-
-// setInput sets input state for some player in a game session.
-func (s *InputState) setInput(player int, data []byte) {
-	atomic.StoreUint32(&s[player].keys, uint32(uint16(data[1])<<8+uint16(data[0])))
-	for i, axes := 0, len(data); i < dpadAxes && i<<1+3 < axes; i++ {
-		axis := i<<1 + 2
-		atomic.StoreInt32(&s[player].axes[i], int32(data[axis+1])<<8+int32(data[axis]))
-	}
-}
-
-// isKeyPressed checks if some button is pressed by any player.
-func (s *InputState) isKeyPressed(port uint, key int) int {
-	return int((atomic.LoadUint32(&s[port].keys) >> uint(key)) & 1)
-}
-
-// isDpadTouched checks if D-pad is used by any player.
-func (s *InputState) isDpadTouched(port uint, axis uint) (shift int16) {
-	return int16(atomic.LoadInt32(&s[port].axes[axis]))
-}

pkg/worker/caged/libretro/frontend_test.go

@@ -86,7 +86,6 @@ func EmulatorMock(room string, system string) *TestFrontend {
 				Path:     os.TempDir(),
 				MainSave: room,
 			},
-			input:       NewGameSessionInput(),
 			done:        make(chan struct{}),
 			th:          conf.Emulator.Threads,
 			log:         l2,
@@ -340,20 +339,6 @@ func TestStateConcurrency(t *testing.T) {
 	}
 }
 
-func TestConcurrentInput(t *testing.T) {
-	var wg sync.WaitGroup
-	state := NewGameSessionInput()
-	events := 1000
-	wg.Add(2 * events)
-
-	for range events {
-		player := rand.IntN(maxPort)
-		go func() { state.setInput(player, []byte{0, 1}); wg.Done() }()
-		go func() { state.isKeyPressed(uint(player), 100); wg.Done() }()
-	}
-	wg.Wait()
-}
-
 func TestStartStop(t *testing.T) {
 	f1 := DefaultFrontend("sushi", sushi.system, sushi.rom)
 	go f1.Start()