From d15f04d4a45a4b0ac91d59acc3d5d11af3862c09 Mon Sep 17 00:00:00 2001 From: Sergey Stepanov Date: Mon, 18 Mar 2024 13:47:39 +0300 Subject: [PATCH] Add keyboard and mouse support --- pkg/api/user.go | 5 +- pkg/api/worker.go | 5 +- pkg/config/config.yaml | 2 + pkg/config/emulator.go | 1 + pkg/coordinator/userapi.go | 4 +- pkg/coordinator/userhandlers.go | 2 +- pkg/network/webrtc/webrtc.go | 45 +-- pkg/worker/caged/app/app.go | 3 +- pkg/worker/caged/caged.go | 6 + pkg/worker/caged/libretro/caged.go | 25 +- pkg/worker/caged/libretro/frontend.go | 59 ++-- pkg/worker/caged/libretro/frontend_test.go | 15 - pkg/worker/caged/libretro/nanoarch/input.go | 151 +++++++++ .../caged/libretro/nanoarch/input_test.go | 73 +++++ pkg/worker/caged/libretro/nanoarch/nanoarch.c | 4 + .../caged/libretro/nanoarch/nanoarch.go | 138 +++++--- pkg/worker/caged/libretro/nanoarch/nanoarch.h | 1 + pkg/worker/coordinatorhandlers.go | 31 +- web/index.html | 1 + web/js/api.js | 295 +++++++++++++++++- web/js/app.js | 52 ++- web/js/event.js | 10 + web/js/gui.js | 2 +- web/js/input/keyboard.js | 56 +++- web/js/network/webrtc.js | 22 ++ web/js/settings.js | 2 + web/js/stream.js | 127 ++++++-- 27 files changed, 937 insertions(+), 200 deletions(-) create mode 100644 pkg/worker/caged/libretro/nanoarch/input.go create mode 100644 pkg/worker/caged/libretro/nanoarch/input_test.go diff --git a/pkg/api/user.go b/pkg/api/user.go index aef4305dc..189b61fc9 100644 --- a/pkg/api/user.go +++ b/pkg/api/user.go @@ -12,8 +12,9 @@ type ( PlayerIndex int `json:"player_index"` } GameStartUserResponse struct { - RoomId string `json:"roomId"` - Av *AppVideoInfo `json:"av"` + RoomId string `json:"roomId"` + Av *AppVideoInfo `json:"av"` + KbMouse bool `json:"kb_mouse"` } IceServer struct { Urls string `json:"urls,omitempty"` diff --git a/pkg/api/worker.go b/pkg/api/worker.go index a4078f5e2..cd9284346 100644 --- a/pkg/api/worker.go +++ b/pkg/api/worker.go @@ -33,8 +33,9 @@ type ( } StartGameResponse struct { Room - AV *AppVideoInfo `json:"av"` - Record bool + AV *AppVideoInfo `json:"av"` + Record bool `json:"record"` + KbMouse bool `json:"kb_mouse"` } RecordGameRequest[T Id] struct { StatefulRoom[T] diff --git a/pkg/config/config.yaml b/pkg/config/config.yaml index 60fc6f539..e789f63f5 100644 --- a/pkg/config/config.yaml +++ b/pkg/config/config.yaml @@ -189,6 +189,7 @@ emulator: # A list of device IDs to bind to the input ports. # Some cores allow binding multiple devices to a single port (DosBox), but typically, # you should bind just one device to one port. + # - kbMouseSupport (bool) -- (temp) a flag if the core needs the keyboard and mouse on the client # - vfr (bool) # (experimental) # Enable variable frame rate only for cores that can't produce a constant frame rate. @@ -205,6 +206,7 @@ emulator: list: gba: lib: mgba_libretro + coreAspectRatio: true roms: [ "gba", "gbc" ] options: mgba_audio_low_pass_filter: enabled diff --git a/pkg/config/emulator.go b/pkg/config/emulator.go index 24993cbed..5ba2531e6 100644 --- a/pkg/config/emulator.go +++ b/pkg/config/emulator.go @@ -48,6 +48,7 @@ type LibretroCoreConfig struct { Height int Hid map[int][]int IsGlAllowed bool + KbMouseSupport bool Lib string Options map[string]string Roms []string diff --git a/pkg/coordinator/userapi.go b/pkg/coordinator/userapi.go index ed1ebcead..047fe1d13 100644 --- a/pkg/coordinator/userapi.go +++ b/pkg/coordinator/userapi.go @@ -37,6 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) } func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) } // StartGame signals the user that everything is ready to start a game. -func (u *User) StartGame(av *api.AppVideoInfo) { - u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av}) +func (u *User) StartGame(av *api.AppVideoInfo, kbMouse bool) { + u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av, KbMouse: kbMouse}) } diff --git a/pkg/coordinator/userhandlers.go b/pkg/coordinator/userhandlers.go index 240f9dbe0..cf62d65ba 100644 --- a/pkg/coordinator/userhandlers.go +++ b/pkg/coordinator/userhandlers.go @@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc return } u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker") - u.StartGame(startGameResp.AV) + u.StartGame(startGameResp.AV, startGameResp.KbMouse) // send back recording status if conf.Recording.Enabled && rq.Record { diff --git a/pkg/network/webrtc/webrtc.go b/pkg/network/webrtc/webrtc.go index 25612d06e..37b99e790 100644 --- a/pkg/network/webrtc/webrtc.go +++ b/pkg/network/webrtc/webrtc.go @@ -81,11 +81,15 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp p.log.Debug().Msgf("Added [%s] track", audio.Codec().MimeType) p.a = audio - // plug in the [data] channel (in and out) - if err = p.addDataChannel("data"); err != nil { + err = p.AddChannel("data", func(data []byte) { + if len(data) == 0 || p.OnMessage == nil { + return + } + p.OnMessage(data) + }) + if err != nil { return "", err } - p.log.Debug().Msg("Added [data] chan") p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") })) // Stream provider supposes to send offer @@ -221,6 +225,19 @@ func (p *Peer) AddCandidate(candidate string, decoder Decoder) error { return nil } +func (p *Peer) AddChannel(label string, onMessage func([]byte)) error { + ch, err := p.addDataChannel(label) + if err != nil { + return err + } + if label == "data" { + p.d = ch + } + ch.OnMessage(func(m webrtc.DataChannelMessage) { onMessage(m.Data) }) + p.log.Debug().Msgf("Added [%v] chan", label) + return nil +} + func (p *Peer) Disconnect() { if p.conn == nil { return @@ -232,29 +249,19 @@ func (p *Peer) Disconnect() { p.log.Debug().Msg("WebRTC stop") } -// addDataChannel creates a new WebRTC data channel for user input. +// addDataChannel creates new WebRTC data channel. // Default params -- ordered: true, negotiated: false. -func (p *Peer) addDataChannel(label string) error { +func (p *Peer) addDataChannel(label string) (*webrtc.DataChannel, error) { ch, err := p.conn.CreateDataChannel(label, nil) if err != nil { - return err + return nil, err } ch.OnOpen(func() { - p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()). - Msg("Data channel [input] opened") + p.log.Debug().Uint16("id", *ch.ID()).Msgf("Data channel [%v] opened", ch.Label()) }) ch.OnError(p.logx) - ch.OnMessage(func(m webrtc.DataChannelMessage) { - if len(m.Data) == 0 { - return - } - if p.OnMessage != nil { - p.OnMessage(m.Data) - } - }) - p.d = ch - ch.OnClose(func() { p.log.Debug().Msg("Data channel [input] has been closed") }) - return nil + ch.OnClose(func() { p.log.Debug().Msgf("Data channel [%v] has been closed", ch.Label()) }) + return ch, nil } func (p *Peer) logx(err error) { p.log.Error().Err(err) } diff --git a/pkg/worker/caged/app/app.go b/pkg/worker/caged/app/app.go index 2fd0704b2..74d89432b 100644 --- a/pkg/worker/caged/app/app.go +++ b/pkg/worker/caged/app/app.go @@ -13,7 +13,8 @@ type App interface { SetAudioCb(func(Audio)) SetVideoCb(func(Video)) SetDataCb(func([]byte)) - SendControl(port int, data []byte) + Input(port int, device byte, data []byte) + KbMouseSupport() bool } type Audio struct { diff --git a/pkg/worker/caged/caged.go b/pkg/worker/caged/caged.go index 2328d96f1..85ede127a 100644 --- a/pkg/worker/caged/caged.go +++ b/pkg/worker/caged/caged.go @@ -15,6 +15,12 @@ type Manager struct { log *logger.Logger } +const ( + RetroPad = libretro.RetroPad + Keyboard = libretro.Keyboard + Mouse = libretro.Mouse +) + type ModName string const Libretro ModName = "libretro" diff --git a/pkg/worker/caged/libretro/caged.go b/pkg/worker/caged/libretro/caged.go index dea9bf5c0..8c06776b0 100644 --- a/pkg/worker/caged/libretro/caged.go +++ b/pkg/worker/caged/libretro/caged.go @@ -79,15 +79,16 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) { } } -func (c *Caged) AspectEnabled() bool { return c.base.nano.Aspect } -func (c *Caged) AspectRatio() float32 { return c.base.AspectRatio() } -func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat() } -func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } -func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } -func (c *Caged) ViewportSize() (int, int) { return c.base.ViewportSize() } -func (c *Caged) Scale() float64 { return c.Emulator.Scale() } -func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) } -func (c *Caged) Start() { go c.Emulator.Start() } -func (c *Caged) SetSaveOnClose(v bool) { c.base.SaveOnClose = v } -func (c *Caged) SetSessionId(name string) { c.base.SetSessionId(name) } -func (c *Caged) Close() { c.Emulator.Close() } +func (c *Caged) AspectEnabled() bool { return c.base.nano.Aspect } +func (c *Caged) AspectRatio() float32 { return c.base.AspectRatio() } +func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat() } +func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } +func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } +func (c *Caged) ViewportSize() (int, int) { return c.base.ViewportSize() } +func (c *Caged) Scale() float64 { return c.Emulator.Scale() } +func (c *Caged) Input(p int, d byte, data []byte) { c.base.Input(p, d, data) } +func (c *Caged) KbMouseSupport() bool { return c.base.KbMouseSupport() } +func (c *Caged) Start() { go c.Emulator.Start() } +func (c *Caged) SetSaveOnClose(v bool) { c.base.SaveOnClose = v } +func (c *Caged) SetSessionId(name string) { c.base.SetSessionId(name) } +func (c *Caged) Close() { c.Emulator.Close() } diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go index 77be7be88..52f6c1814 100644 --- a/pkg/worker/caged/libretro/frontend.go +++ b/pkg/worker/caged/libretro/frontend.go @@ -5,7 +5,6 @@ import ( "fmt" "path/filepath" "sync" - "sync/atomic" "time" "unsafe" @@ -44,7 +43,7 @@ type Emulator interface { // Close will be called when the game is done Close() // Input passes input to the emulator - Input(player int, data []byte) + Input(player int, device byte, data []byte) // Scale returns set video scale factor Scale() float64 } @@ -52,7 +51,6 @@ type Emulator interface { type Frontend struct { conf config.Emulator done chan struct{} - input InputState log *logger.Logger nano *nanoarch.Nanoarch onAudio func(app.Audio) @@ -70,21 +68,12 @@ type Frontend struct { SaveOnClose bool } -// InputState stores full controller state. -// It consists of: -// - uint16 button values -// - int16 analog stick values -type ( - InputState [maxPort]State - State struct { - keys uint32 - axes [dpadAxes]int32 - } -) +type Device byte const ( - maxPort = 4 - dpadAxes = 4 + RetroPad = Device(nanoarch.RetroPad) + Keyboard = Device(nanoarch.Keyboard) + Mouse = Device(nanoarch.Mouse) ) var ( @@ -134,7 +123,6 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) { f := &Frontend{ conf: conf, done: make(chan struct{}), - input: NewGameSessionInput(), log: log, onAudio: noAudio, onData: noData, @@ -166,6 +154,7 @@ func (f *Frontend) LoadCore(emu string) { Options: conf.Options, UsesLibCo: conf.UsesLibCo, CoreAspectRatio: conf.CoreAspectRatio, + KbMouseSupport: conf.KbMouseSupport, } f.mu.Lock() scale := 1.0 @@ -231,8 +220,6 @@ func (f *Frontend) linkNano(nano *nanoarch.Nanoarch) { } f.nano.WaitReady() // start only when nano is available - f.nano.OnKeyPress = f.input.isKeyPressed - f.nano.OnDpad = f.input.isDpadTouched f.nano.OnVideo = f.handleVideo f.nano.OnAudio = f.handleAudio f.nano.OnDup = f.handleDup @@ -304,8 +291,8 @@ func (f *Frontend) Flipped() bool { return f.nano.IsGL() } func (f *Frontend) FrameSize() (int, int) { return f.nano.BaseWidth(), f.nano.BaseHeight() } func (f *Frontend) HasSave() bool { return os.Exists(f.HashPath()) } func (f *Frontend) HashPath() string { return f.storage.GetSavePath() } -func (f *Frontend) Input(player int, data []byte) { f.input.setInput(player, data) } func (f *Frontend) IsPortrait() bool { return f.nano.IsPortrait() } +func (f *Frontend) KbMouseSupport() bool { return f.nano.KbMouseSupport() } func (f *Frontend) LoadGame(path string) error { return f.nano.LoadGame(path) } func (f *Frontend) PixFormat() uint32 { return f.nano.Video.PixFmt.C } func (f *Frontend) RestoreGameState() error { return f.Load() } @@ -322,6 +309,17 @@ func (f *Frontend) Tick() { f.mu.Lock(); f.nano.Run(); f func (f *Frontend) ViewportRecalculate() { f.mu.Lock(); f.vw, f.vh = f.ViewportCalc(); f.mu.Unlock() } func (f *Frontend) ViewportSize() (int, int) { return f.vw, f.vh } +func (f *Frontend) Input(port int, device byte, data []byte) { + switch Device(device) { + case RetroPad: + f.nano.InputRetropad(port, data) + case Keyboard: + f.nano.InputKeyboard(port, data) + case Mouse: + f.nano.InputMouse(port, data) + } +} + func (f *Frontend) ViewportCalc() (nw int, nh int) { w, h := f.FrameSize() nw, nh = w, h @@ -412,24 +410,3 @@ func (f *Frontend) autosave(periodSec int) { } } } - -func NewGameSessionInput() InputState { return [maxPort]State{} } - -// setInput sets input state for some player in a game session. -func (s *InputState) setInput(player int, data []byte) { - atomic.StoreUint32(&s[player].keys, uint32(uint16(data[1])<<8+uint16(data[0]))) - for i, axes := 0, len(data); i < dpadAxes && i<<1+3 < axes; i++ { - axis := i<<1 + 2 - atomic.StoreInt32(&s[player].axes[i], int32(data[axis+1])<<8+int32(data[axis])) - } -} - -// isKeyPressed checks if some button is pressed by any player. -func (s *InputState) isKeyPressed(port uint, key int) int { - return int((atomic.LoadUint32(&s[port].keys) >> uint(key)) & 1) -} - -// isDpadTouched checks if D-pad is used by any player. -func (s *InputState) isDpadTouched(port uint, axis uint) (shift int16) { - return int16(atomic.LoadInt32(&s[port].axes[axis])) -} diff --git a/pkg/worker/caged/libretro/frontend_test.go b/pkg/worker/caged/libretro/frontend_test.go index e10d289ea..3a9705769 100644 --- a/pkg/worker/caged/libretro/frontend_test.go +++ b/pkg/worker/caged/libretro/frontend_test.go @@ -85,7 +85,6 @@ func EmulatorMock(room string, system string) *TestFrontend { Path: os.TempDir(), MainSave: room, }, - input: NewGameSessionInput(), done: make(chan struct{}), th: conf.Emulator.Threads, log: l2, @@ -326,20 +325,6 @@ func TestStateConcurrency(t *testing.T) { } } -func TestConcurrentInput(t *testing.T) { - var wg sync.WaitGroup - state := NewGameSessionInput() - events := 1000 - wg.Add(2 * events) - - for i := 0; i < events; i++ { - player := rand.Intn(maxPort) - go func() { state.setInput(player, []byte{0, 1}); wg.Done() }() - go func() { state.isKeyPressed(uint(player), 100); wg.Done() }() - } - wg.Wait() -} - func TestStartStop(t *testing.T) { f1 := DefaultFrontend("sushi", sushi.system, sushi.rom) go f1.Start() diff --git a/pkg/worker/caged/libretro/nanoarch/input.go b/pkg/worker/caged/libretro/nanoarch/input.go new file mode 100644 index 000000000..a7972b800 --- /dev/null +++ b/pkg/worker/caged/libretro/nanoarch/input.go @@ -0,0 +1,151 @@ +package nanoarch + +import ( + "encoding/binary" + "sync" + "sync/atomic" +) + +//#include +//#include "libretro.h" +import "C" + +const ( + Released C.int16_t = iota + Pressed +) + +const RetrokLast = int(C.RETROK_LAST) + +// InputState stores full controller state. +// It consists of: +// - uint16 button values +// - int16 analog stick values +type InputState [maxPort]RetroPadState + +type ( + RetroPadState struct { + keys uint32 + axes [dpadAxes]int32 + } + KeyboardState struct { + keys [RetrokLast]byte + mod uint16 + mu sync.Mutex + } + MouseState struct { + dx, dy atomic.Int32 + buttons atomic.Int32 + } +) + +type MouseBtnState int32 + +type Device byte + +const ( + RetroPad Device = iota + Keyboard + Mouse +) + +const ( + MouseMove = iota + MouseButton +) + +const ( + MouseLeft MouseBtnState = 1 << iota + MouseRight + MouseMiddle +) + +const ( + maxPort = 4 + dpadAxes = 4 +) + +// Input sets input state for some player in a game session. +func (s *InputState) Input(port int, data []byte) { + atomic.StoreUint32(&s[port].keys, uint32(uint16(data[1])<<8+uint16(data[0]))) + for i, axes := 0, len(data); i < dpadAxes && i<<1+3 < axes; i++ { + axis := i<<1 + 2 + atomic.StoreInt32(&s[port].axes[i], int32(data[axis+1])<<8+int32(data[axis])) + } +} + +// IsKeyPressed checks if some button is pressed by any player. +func (s *InputState) IsKeyPressed(port uint, key int) C.int16_t { + return C.int16_t((atomic.LoadUint32(&s[port].keys) >> uint(key)) & 1) +} + +// IsDpadTouched checks if D-pad is used by any player. +func (s *InputState) IsDpadTouched(port uint, axis uint) (shift C.int16_t) { + return C.int16_t(atomic.LoadInt32(&s[port].axes[axis])) +} + +// SetKey sets keyboard state. +// +// 0 1 2 3 4 5 6 +// [ KEY ] P MOD +// +// KEY contains Libretro code of the keyboard key (4 bytes). +// P contains 0 or 1 if the key is pressed (1 byte). +// MOD contains bitmask for Alt | Ctrl | Meta | Shift keys press state (2 bytes). +// +// Returns decoded state from the input bytes. +func (ks *KeyboardState) SetKey(data []byte) (pressed bool, key uint, mod uint16) { + if len(data) != 7 { + return + } + + press := data[4] + pressed = press == 1 + key = uint(binary.BigEndian.Uint32(data)) + mod = binary.BigEndian.Uint16(data[5:]) + ks.mu.Lock() + ks.keys[key] = press + ks.mod = mod + ks.mu.Unlock() + return +} + +func (ks *KeyboardState) Pressed(key uint) C.int16_t { + ks.mu.Lock() + press := ks.keys[key] + ks.mu.Unlock() + if press == 1 { + return Pressed + } + return Released +} + +// ShiftPos sets mouse relative position state. +// +// 0 1 2 3 +// [dx] [dy] +// +// dx and dy are relative mouse coordinates +func (ms *MouseState) ShiftPos(data []byte) { + if len(data) != 4 { + return + } + dx := int16(data[0])<<8 + int16(data[1]) + dy := int16(data[2])<<8 + int16(data[3]) + ms.dx.Add(int32(dx)) + ms.dy.Add(int32(dy)) +} + +func (ms *MouseState) PopX() C.int16_t { return C.int16_t(ms.dx.Swap(0)) } +func (ms *MouseState) PopY() C.int16_t { return C.int16_t(ms.dy.Swap(0)) } + +// SetButtons sets the state MouseBtnState of mouse buttons. +func (ms *MouseState) SetButtons(data byte) { ms.buttons.Store(int32(data)) } + +func (ms *MouseState) Buttons() (l, r, m bool) { + mbs := MouseBtnState(ms.buttons.Load()) + l = mbs&MouseLeft != 0 + r = mbs&MouseRight != 0 + m = mbs&MouseMiddle != 0 + return +} diff --git a/pkg/worker/caged/libretro/nanoarch/input_test.go b/pkg/worker/caged/libretro/nanoarch/input_test.go new file mode 100644 index 000000000..634df14d3 --- /dev/null +++ b/pkg/worker/caged/libretro/nanoarch/input_test.go @@ -0,0 +1,73 @@ +package nanoarch + +import ( + "encoding/binary" + "math/rand" + "sync" + "testing" +) + +func TestConcurrentInput(t *testing.T) { + var wg sync.WaitGroup + state := InputState{} + events := 1000 + wg.Add(2 * events) + + for i := 0; i < events; i++ { + player := rand.Intn(maxPort) + go func() { state.Input(player, []byte{0, 1}); wg.Done() }() + go func() { state.IsKeyPressed(uint(player), 100); wg.Done() }() + } + wg.Wait() +} + +func TestMousePos(t *testing.T) { + data := []byte{0, 0, 0, 0} + + dx := 1111 + dy := 2222 + + binary.BigEndian.PutUint16(data, uint16(dx)) + binary.BigEndian.PutUint16(data[2:], uint16(dy)) + + ms := MouseState{} + ms.ShiftPos(data) + + x := int(ms.PopX()) + y := int(ms.PopY()) + + if x != dx || y != dy { + t.Errorf("invalid state, %v = %v, %v = %v", dx, x, dy, y) + } + + if ms.dx.Load() != 0 || ms.dy.Load() != 0 { + t.Errorf("coordinates weren't cleared") + } +} + +func TestMouseButtons(t *testing.T) { + tests := []struct { + name string + data byte + l bool + r bool + m bool + }{ + {name: "l+r+m+", data: 1 + 2 + 4, l: true, r: true, m: true}, + {name: "l-r-m-", data: 0}, + {name: "l-r+m-", data: 2, r: true}, + {name: "l+r-m+", data: 1 + 4, l: true, m: true}, + } + + ms := MouseState{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ms.SetButtons(test.data) + l, r, m := ms.Buttons() + if l != test.l || r != test.r || m != test.m { + t.Errorf("wrong button state: %v -> %v, %v, %v", test.data, l, r, m) + } + }) + } +} diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.c b/pkg/worker/caged/libretro/nanoarch/nanoarch.c index 4edd73ff7..3300900ec 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.c +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.c @@ -127,6 +127,10 @@ void bridge_clear_all_thread_waits_cb(void *data) { *(retro_environment_t *)data = clear_all_thread_waits_cb; } +void bridge_retro_keyboard_callback(void *cb, bool down, unsigned keycode, uint32_t character, uint16_t keyModifiers) { + (*(retro_keyboard_event_t *) cb)(down, keycode, character, keyModifiers); +} + bool core_environment_cgo(unsigned cmd, void *data) { bool coreEnvironment(unsigned, void *); return coreEnvironment(cmd, data); diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.go b/pkg/worker/caged/libretro/nanoarch/nanoarch.go index 94ff0b0d7..f7afe9fa3 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.go +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.go @@ -26,13 +26,6 @@ import ( */ import "C" -const lastKey = int(C.RETRO_DEVICE_ID_JOYPAD_R3) - -const KeyPressed = 1 -const KeyReleased = 0 - -const MaxPort int = 4 - var ( RGBA5551 = PixFmt{C: 0, BPP: 2} // BIT_FORMAT_SHORT_5_5_5_1 has 5 bits R, 5 bits G, 5 bits B, 1 bit alpha RGBA8888Rev = PixFmt{C: 1, BPP: 4} // BIT_FORMAT_INT_8_8_8_8_REV has 8 bits R, 8 bits G, 8 bits B, 8 bit alpha @@ -41,6 +34,12 @@ var ( type Nanoarch struct { Handlers + + keyboard KeyboardState + mouse MouseState + retropad InputState + + keyboardCb *C.struct_retro_keyboard_callback LastFrameTime int64 LibCo bool meta Metadata @@ -74,8 +73,6 @@ type Nanoarch struct { } type Handlers struct { - OnDpad func(port uint, axis uint) (shift int16) - OnKeyPress func(port uint, key int) int OnAudio func(ptr unsafe.Pointer, frames int) OnVideo func(data []byte, delta int32, fi FrameInfo) OnDup func() @@ -99,6 +96,7 @@ type Metadata struct { Hacks []string Hid map[int][]int CoreAspectRatio bool + KbMouseSupport bool } type PixFmt struct { @@ -125,10 +123,8 @@ var Nan0 = Nanoarch{ Stopped: atomic.Bool{}, limiter: func(fn func()) { fn() }, Handlers: Handlers{ - OnDpad: func(uint, uint) int16 { return 0 }, - OnKeyPress: func(uint, int) int { return 0 }, - OnAudio: func(unsafe.Pointer, int) {}, - OnVideo: func([]byte, int32, FrameInfo) {}, + OnAudio: func(unsafe.Pointer, int) {}, + OnVideo: func([]byte, int32, FrameInfo) {}, OnDup: func() {}, }, } @@ -149,6 +145,7 @@ func (n *Nanoarch) AspectRatio() float32 { return float32(n.sys.av.g func (n *Nanoarch) AudioSampleRate() int { return int(n.sys.av.timing.sample_rate) } func (n *Nanoarch) VideoFramerate() int { return int(n.sys.av.timing.fps) } func (n *Nanoarch) IsPortrait() bool { return 90 == n.Rot%180 } +func (n *Nanoarch) KbMouseSupport() bool { return n.meta.KbMouseSupport } func (n *Nanoarch) BaseWidth() int { return int(n.sys.av.geometry.base_width) } func (n *Nanoarch) BaseHeight() int { return int(n.sys.av.geometry.base_height) } func (n *Nanoarch) WaitReady() { <-n.reserved } @@ -168,6 +165,12 @@ func (n *Nanoarch) CoreLoad(meta Metadata) { // hacks Nan0.hackSkipHwContextDestroy = meta.HasHack("skip_hw_context_destroy") + // reset controllers + n.retropad = InputState{} + n.keyboardCb = nil + n.keyboard = KeyboardState{} + n.mouse = MouseState{} + n.options = &meta.Options filePath := meta.LibPath @@ -295,7 +298,7 @@ func (n *Nanoarch) LoadGame(path string) error { // set default controller types on all ports // needed for nestopia - for i := 0; i < MaxPort; i++ { + for i := 0; i < maxPort; i++ { C.bridge_retro_set_controller_port_device(retroSetControllerPortDevice, C.uint(i), C.RETRO_DEVICE_JOYPAD) } @@ -371,8 +374,34 @@ func (n *Nanoarch) Run() { } } -func (n *Nanoarch) IsGL() bool { return n.Video.gl.enabled } -func (n *Nanoarch) IsStopped() bool { return n.Stopped.Load() } +func (n *Nanoarch) IsGL() bool { return n.Video.gl.enabled } +func (n *Nanoarch) IsStopped() bool { return n.Stopped.Load() } +func (n *Nanoarch) InputRetropad(port int, data []byte) { n.retropad.Input(port, data) } +func (n *Nanoarch) InputKeyboard(_ int, data []byte) { + if n.keyboardCb == nil { + return + } + + // we should preserve the state of pressed buttons for the input poll function (each retro_run) + // and explicitly call the retro_keyboard_callback function when a keyboard event happens + pressed, key, mod := n.keyboard.SetKey(data) + C.bridge_retro_keyboard_callback(unsafe.Pointer(n.keyboardCb), C.bool(pressed), + C.unsigned(key), C.uint32_t(0), C.uint16_t(mod)) +} +func (n *Nanoarch) InputMouse(_ int, data []byte) { + if len(data) == 0 { + return + } + + t := data[0] + state := data[1:] + switch t { + case MouseMove: + n.mouse.ShiftPos(state) + case MouseButton: + n.mouse.SetButtons(state[0]) + } +} func videoSetPixelFormat(format uint32) (C.bool, error) { switch format { @@ -597,29 +626,55 @@ func coreInputPoll() {} //export coreInputState func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.unsigned) C.int16_t { - if uint(port) >= uint(MaxPort) { - return KeyReleased - } - - if device == C.RETRO_DEVICE_ANALOG { - if index > C.RETRO_DEVICE_INDEX_ANALOG_RIGHT || id > C.RETRO_DEVICE_ID_ANALOG_Y { - return 0 + //Nan0.log.Debug().Msgf("%v %v %v %v", port, device, index, id) + + // something like PCSX-ReArmed has 8 ports + if port >= maxPort { + return Released + } + + switch device { + case C.RETRO_DEVICE_JOYPAD: + return Nan0.retropad.IsKeyPressed(uint(port), int(id)) + case C.RETRO_DEVICE_ANALOG: + switch index { + case C.RETRO_DEVICE_INDEX_ANALOG_LEFT: + return Nan0.retropad.IsDpadTouched(uint(port), uint(index*2+id)) + case C.RETRO_DEVICE_INDEX_ANALOG_RIGHT: + case C.RETRO_DEVICE_INDEX_ANALOG_BUTTON: } - axis := index*2 + id - value := Nan0.Handlers.OnDpad(uint(port), uint(axis)) - if value != 0 { - return (C.int16_t)(value) + case C.RETRO_DEVICE_KEYBOARD: + return Nan0.keyboard.Pressed(uint(id)) + case C.RETRO_DEVICE_MOUSE: + switch id { + case C.RETRO_DEVICE_ID_MOUSE_X: + x := Nan0.mouse.PopX() + return x + case C.RETRO_DEVICE_ID_MOUSE_Y: + y := Nan0.mouse.PopY() + return y + case C.RETRO_DEVICE_ID_MOUSE_LEFT: + if l, _, _ := Nan0.mouse.Buttons(); l { + return Pressed + } + case C.RETRO_DEVICE_ID_MOUSE_RIGHT: + if _, r, _ := Nan0.mouse.Buttons(); r { + return Pressed + } + case C.RETRO_DEVICE_ID_MOUSE_WHEELUP: + case C.RETRO_DEVICE_ID_MOUSE_WHEELDOWN: + case C.RETRO_DEVICE_ID_MOUSE_MIDDLE: + if _, _, m := Nan0.mouse.Buttons(); m { + return Pressed + } + case C.RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELUP: + case C.RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELDOWN: + case C.RETRO_DEVICE_ID_MOUSE_BUTTON_4: + case C.RETRO_DEVICE_ID_MOUSE_BUTTON_5: } } - key := int(id) - if key > lastKey || index > 0 || device != C.RETRO_DEVICE_JOYPAD { - return KeyReleased - } - if Nan0.Handlers.OnKeyPress(uint(port), key) == KeyPressed { - return KeyPressed - } - return KeyReleased + return Released } //export coreAudioSample @@ -780,9 +835,20 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { } cInfo.WriteString(fmt.Sprintf("%v: %v%s", cd[i].id, C.GoString(cd[i].desc), delim)) } - Nan0.log.Debug().Msgf("%v", cInfo.String()) + //Nan0.log.Debug().Msgf("%v", cInfo.String()) } return true + case C.RETRO_ENVIRONMENT_GET_INPUT_MAX_USERS: + *(*C.unsigned)(data) = C.unsigned(4) + Nan0.log.Debug().Msgf("Set max users: %v", 4) + return true + case C.RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK: + Nan0.log.Debug().Msgf("Keyboard event callback was set") + Nan0.keyboardCb = (*C.struct_retro_keyboard_callback)(data) + return true + case C.RETRO_ENVIRONMENT_GET_INPUT_BITMASKS: + Nan0.log.Debug().Msgf("Set input bitmasks: false") + return false case C.RETRO_ENVIRONMENT_GET_CLEAR_ALL_THREAD_WAITS_CB: C.bridge_clear_all_thread_waits_cb(data) return true diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.h b/pkg/worker/caged/libretro/nanoarch/nanoarch.h index 0c4b01776..661036434 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.h +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.h @@ -23,6 +23,7 @@ void bridge_retro_set_input_poll(void *f, void *callback); void bridge_retro_set_input_state(void *f, void *callback); void bridge_retro_set_video_refresh(void *f, void *callback); void bridge_clear_all_thread_waits_cb(void *f); +void bridge_retro_keyboard_callback(void *f, bool down, unsigned keycode, uint32_t character, uint16_t keyModifiers); bool core_environment_cgo(unsigned cmd, void *data); int16_t core_input_state_cgo(unsigned port, unsigned device, unsigned index, unsigned id); diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go index 3c65c32d3..da013ef4b 100644 --- a/pkg/worker/coordinatorhandlers.go +++ b/pkg/worker/coordinatorhandlers.go @@ -126,12 +126,14 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke } } - data, err := api.Wrap(api.Out{T: uint8(api.AppVideoChange), Payload: api.AppVideoInfo{ - W: m.VideoW, - H: m.VideoH, - A: app.AspectRatio(), - S: int(app.Scale()), - }}) + data, err := api.Wrap(api.Out{ + T: uint8(api.AppVideoChange), + Payload: api.AppVideoInfo{ + W: m.VideoW, + H: m.VideoH, + A: app.AspectRatio(), + S: int(app.Scale()), + }}) if err != nil { c.log.Error().Err(err).Msgf("wrap") } @@ -159,6 +161,7 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke w.router.SetRoom(nil) return api.EmptyPacket } + if app.Flipped() { m.SetVideoFlip(true) } @@ -170,11 +173,23 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke } c.log.Debug().Msg("Start session input poll") - room.WithWebRTC(user.Session).OnMessage = func(data []byte) { r.App().SendControl(user.Index, data) } + + needsKbMouse := r.App().KbMouseSupport() + + s := room.WithWebRTC(user.Session) + s.OnMessage = func(data []byte) { r.App().Input(user.Index, byte(caged.RetroPad), data) } + if needsKbMouse { + _ = s.AddChannel("keyboard", func(data []byte) { r.App().Input(user.Index, byte(caged.Keyboard), data) }) + _ = s.AddChannel("mouse", func(data []byte) { r.App().Input(user.Index, byte(caged.Mouse), data) }) + } c.RegisterRoom(r.Id()) - response := api.StartGameResponse{Room: api.Room{Rid: r.Id()}, Record: w.conf.Recording.Enabled} + response := api.StartGameResponse{ + Room: api.Room{Rid: r.Id()}, + Record: w.conf.Recording.Enabled, + KbMouse: needsKbMouse, + } if r.App().AspectEnabled() { ww, hh := r.App().ViewportSize() response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio(), S: int(r.App().Scale())} diff --git a/web/index.html b/web/index.html index 8093f5d44..8780794d3 100644 --- a/web/index.html +++ b/web/index.html @@ -1,3 +1,4 @@ + diff --git a/web/js/api.js b/web/js/api.js index 6b93264bd..e79286d43 100644 --- a/web/js/api.js +++ b/web/js/api.js @@ -19,6 +19,277 @@ const endpoints = { APP_VIDEO_CHANGE: 150, } +let transport = { + send: (packet) => { + log.warn('Default transport is used! Change it with the api.transport variable.', packet) + }, + keyboard: (packet) => { + log.warn('Default transport is used! Change it with the api.transport variable.', packet) + }, + mouse: (packet) => { + log.warn('Default transport is used! Change it with the api.transport variable.', packet) + } +} + +const packet = (type, payload, id) => { + const packet = {t: type}; + if (id !== undefined) packet.id = id; + if (payload !== undefined) packet.p = payload; + transport.send(packet); +} + +const decodeBytes = (b) => String.fromCharCode.apply(null, new Uint8Array(b)) + +const keyboardPress = (() => { + // 0 1 2 3 4 5 6 + // [CODE ] P MOD + const buffer = new ArrayBuffer(7); + const dv = new DataView(buffer); + + return (pressed = false, e) => { + if (e.repeat) return; // skip pressed key events + + const key = libretro.mod; + let code = libretro.map('', e.code); + let shift = e.shiftKey; + + // a special Esc for &$&!& Firefox + if (shift && code === 96) { + code = 27; + shift = false; + } + + const mod = 0 + | (e.altKey && key.ALT) + | (e.ctrlKey && key.CTRL) + | (e.metaKey && key.META) + | (shift && key.SHIFT) + | (e.getModifierState('NumLock') && key.NUMLOCK) + | (e.getModifierState('CapsLock') && key.CAPSLOCK) + | (e.getModifierState('ScrollLock') && key.SCROLLOCK) + dv.setUint32(0, code); + dv.setUint8(4, +pressed); + dv.setUint16(5, mod) + transport.keyboard(buffer); + } +})(); + +const mouse = { + MOVEMENT: 0, + BUTTONS: 1 +} + +const mouseMove = (() => { + // 0 1 2 3 4 + // T DX DY + const buffer = new ArrayBuffer(5); + const dv = new DataView(buffer); + + return (dx = 0, dy = 0) => { + dv.setUint8(0, mouse.MOVEMENT); + dv.setInt16(1, dx); + dv.setInt16(3, dy); + transport.mouse(buffer); + } +})(); + +const mousePress = (() => { + // 0 1 + // T B + const buffer = new ArrayBuffer(2); + const dv = new DataView(buffer); + + // 0: Main button pressed, usually the left button or the un-initialized state + // 1: Auxiliary button pressed, usually the wheel button or the middle button (if present) + // 2: Secondary button pressed, usually the right button + // 3: Fourth button, typically the Browser Back button + // 4: Fifth button, typically the Browser Forward button + + const b2r = [1, 4, 2, 0, 0] // browser mouse button to retro button + // assumed that only one button pressed / released + + return (button = 0, pressed = false) => { + dv.setUint8(0, mouse.BUTTONS); + dv.setUint8(1, pressed ? b2r[button] : 0); + transport.mouse(buffer); + } +})(); + + +const libretro = function () {// RETRO_KEYBOARD + const retro = { + '': 0, + 'Unidentified': 0, + 'Unknown': 0, // ??? + 'First': 0, // ??? + 'Backspace': 8, + 'Tab': 9, + 'Clear': 12, + 'Enter': 13, 'Return': 13, + 'Pause': 19, + 'Escape': 27, + 'Space': 32, + 'Exclaim': 33, + 'Quotedbl': 34, + 'Hash': 35, + 'Dollar': 36, + 'Ampersand': 38, + 'Quote': 39, + 'Leftparen': 40, '(': 40, + 'Rightparen': 41, ')': 41, + 'Asterisk': 42, + 'Plus': 43, + 'Comma': 44, + 'Minus': 45, + 'Period': 46, + 'Slash': 47, + 'Digit0': 48, + 'Digit1': 49, + 'Digit2': 50, + 'Digit3': 51, + 'Digit4': 52, + 'Digit5': 53, + 'Digit6': 54, + 'Digit7': 55, + 'Digit8': 56, + 'Digit9': 57, + 'Colon': 58, ':': 58, + 'Semicolon': 59, ';': 59, + 'Less': 60, '<': 60, + 'Equal': 61, '=': 61, + 'Greater': 62, '>': 62, + 'Question': 63, '?': 63, + // RETROK_AT = 64, + 'BracketLeft': 91, '[': 91, + 'Backslash': 92, '\\': 92, + 'BracketRight': 93, ']': 93, + // RETROK_CARET = 94, + // RETROK_UNDERSCORE = 95, + 'Backquote': 96, '`': 96, + 'KeyA': 97, + 'KeyB': 98, + 'KeyC': 99, + 'KeyD': 100, + 'KeyE': 101, + 'KeyF': 102, + 'KeyG': 103, + 'KeyH': 104, + 'KeyI': 105, + 'KeyJ': 106, + 'KeyK': 107, + 'KeyL': 108, + 'KeyM': 109, + 'KeyN': 110, + 'KeyO': 111, + 'KeyP': 112, + 'KeyQ': 113, + 'KeyR': 114, + 'KeyS': 115, + 'KeyT': 116, + 'KeyU': 117, + 'KeyV': 118, + 'KeyW': 119, + 'KeyX': 120, + 'KeyY': 121, + 'KeyZ': 122, + '{': 123, + '|': 124, + '}': 125, + 'Tilde': 126, '~': 126, + 'Delete': 127, + + 'Numpad0': 256, + 'Numpad1': 257, + 'Numpad2': 258, + 'Numpad3': 259, + 'Numpad4': 260, + 'Numpad5': 261, + 'Numpad6': 262, + 'Numpad7': 263, + 'Numpad8': 264, + 'Numpad9': 265, + 'NumpadDecimal': 266, + 'NumpadDivide': 267, + 'NumpadMultiply': 268, + 'NumpadSubtract': 269, + 'NumpadAdd': 270, + 'NumpadEnter': 271, + 'NumpadEqual': 272, + + 'ArrowUp': 273, + 'ArrowDown': 274, + 'ArrowRight': 275, + 'ArrowLeft': 276, + 'Insert': 277, + 'Home': 278, + 'End': 279, + 'PageUp': 280, + 'PageDown': 281, + + 'F1': 282, + 'F2': 283, + 'F3': 284, + 'F4': 285, + 'F5': 286, + 'F6': 287, + 'F7': 288, + 'F8': 289, + 'F9': 290, + 'F10': 291, + 'F11': 292, + 'F12': 293, + 'F13': 294, + 'F14': 295, + 'F15': 296, + + 'NumLock': 300, + 'CapsLock': 301, + 'ScrollLock': 302, + 'ShiftRight': 303, + 'ShiftLeft': 304, + 'ControlRight': 305, + 'ControlLeft': 306, + 'AltRight': 307, + 'AltLeft': 308, + 'MetaRight': 309, + 'MetaLeft': 310, + // RETROK_LSUPER = 311, + // RETROK_RSUPER = 312, + // RETROK_MODE = 313, + // RETROK_COMPOSE = 314, + + // RETROK_HELP = 315, + // RETROK_PRINT = 316, + // RETROK_SYSREQ = 317, + // RETROK_BREAK = 318, + // RETROK_MENU = 319, + 'Power': 320, + // RETROK_EURO = 321, + // RETROK_UNDO = 322, + // RETROK_OEM_102 = 323, + }; + + const retroMod = { + NONE: 0x0000, + SHIFT: 0x01, + CTRL: 0x02, + ALT: 0x04, + META: 0x08, + NUMLOCK: 0x10, + CAPSLOCK: 0x20, + SCROLLOCK: 0x40, + }; + + const _map = (key = '', code = '') => { + return retro[code] || retro[key] || 0 + } + + return { + map: _map, + mod: retroMod, + } +}() + /** * Server API. * @@ -38,6 +309,15 @@ export const api = { getWorkerList: () => packet(endpoints.GET_WORKER_LIST), }, game: { + input: { + keyboard: { + press: keyboardPress, + }, + mouse: { + move: mouseMove, + press: mousePress, + } + }, load: () => packet(endpoints.GAME_LOAD), save: () => packet(endpoints.GAME_SAVE), setPlayerIndex: (i) => packet(endpoints.GAME_SET_PLAYER_INDEX, i), @@ -53,18 +333,3 @@ export const api = { quit: (roomId) => packet(endpoints.GAME_QUIT, {room_id: roomId}), } } - -let transport = { - send: (packet) => { - log.warn('Default transport is used! Change it with the api.transport variable.', packet) - } -} - -const packet = (type, payload, id) => { - const packet = {t: type}; - if (id !== undefined) packet.id = id; - if (payload !== undefined) packet.p = payload; - transport.send(packet); -} - -const decodeBytes = (b) => String.fromCharCode.apply(null, new Uint8Array(b)) diff --git a/web/js/app.js b/web/js/app.js index 408125dca..3a252a747 100644 --- a/web/js/app.js +++ b/web/js/app.js @@ -12,6 +12,7 @@ import { AXIS_CHANGED, CONTROLLER_UPDATED, DPAD_TOGGLE, + FULLSCREEN_CHANGE, GAME_ERROR_NO_FREE_SLOTS, GAME_LOADED, GAME_PLAYER_IDX, @@ -21,11 +22,17 @@ import { GAMEPAD_CONNECTED, GAMEPAD_DISCONNECTED, HELP_OVERLAY_TOGGLED, + KB_MOUSE_FLAG, KEY_PRESSED, KEY_RELEASED, + KEYBOARD_KEY_DOWN, + KEYBOARD_KEY_UP, LATENCY_CHECK_REQUESTED, MENU_HANDLER_ATTACHED, MESSAGE, + MOUSE_MOVED, + MOUSE_PRESSED, + POINTER_LOCK_CHANGE, RECORDING_STATUS_CHANGED, RECORDING_TOGGLED, SETTINGS_CHANGED, @@ -101,7 +108,7 @@ const setState = (newState = app.state.eden) => { }; const onGameRoomAvailable = () => { - // room is ready + stream.forceFullscreenMaybe(); }; const onConnectionReady = () => { @@ -188,7 +195,6 @@ const startGame = () => { retropad.poll.disable(); gui.hide(menuScreen); stream.toggle(true); - stream.forceFullscreenMaybe(); gui.show(keyButtons[KEY.SAVE]); gui.show(keyButtons[KEY.LOAD]); // end clear @@ -211,9 +217,8 @@ const onMessage = (m) => { pub(WEBRTC_ICE_CANDIDATE_RECEIVED, {candidate: payload}); break; case api.endpoint.GAME_START: - if (payload.av) { - pub(APP_VIDEO_CHANGED, payload.av) - } + payload.av && pub(APP_VIDEO_CHANGED, payload.av) + payload.kb_mouse && pub(KB_MOUSE_FLAG); pub(GAME_ROOM_AVAILABLE, {roomId: payload.roomId}); break; case api.endpoint.GAME_SAVE: @@ -259,7 +264,7 @@ const onKeyPress = (data) => { if (KEY.HELP === data.key) helpScreen.show(true, event); } - state.keyPress(data.key); + state.keyPress(data.key, data.code); }; // pre-state key release handler @@ -286,7 +291,7 @@ const onKeyRelease = data => { // change app state if settings if (KEY.SETTINGS === data.key) setState(app.state.settings); - state.keyRelease(data.key); + state.keyRelease(data.key, data.code); }; const updatePlayerIndex = (idx, not_game = false) => { @@ -410,7 +415,12 @@ const app = { ..._default, name: 'game', axisChanged: (id, value) => retropad.setAxisChanged(id, value), - keyPress: key => retropad.setKeyState(key, true), + keyboardInput: (pressed, e) => api.game.input.keyboard.press(pressed, e), + mouseMove: (e) => api.game.input.mouse.move(e.dx, e.dy), + mousePress: (e) => api.game.input.mouse.press(e.b, e.p), + keyPress: (key) => { + retropad.setKeyState(key, true); + }, keyRelease: function (key) { retropad.setKeyState(key, false); @@ -460,6 +470,15 @@ const app = { } }; +// Browser lock API +document.onpointerlockchange = () => { + pub(POINTER_LOCK_CHANGE, document.pointerLockElement); +} + +document.onfullscreenchange = async () => { + pub(FULLSCREEN_CHANGE, document.fullscreenElement); +} + // subscriptions sub(MESSAGE, onMessage); @@ -497,8 +516,19 @@ sub(GAMEPAD_DISCONNECTED, () => message.show('Gamepad disconnected')); sub(MENU_HANDLER_ATTACHED, (data) => { menuScreen.addEventListener(data.event, data.handler, {passive: true}); }); + +// keyboard handler in the Screen Lock mode +sub(KEYBOARD_KEY_DOWN, (v) => state.keyboardInput?.(true, v)); +sub(KEYBOARD_KEY_UP, (v) => state.keyboardInput?.(false, v)); + +// mouse handler in the Screen Lock mode +sub(MOUSE_MOVED, (e) => state.mouseMove?.(e)) +sub(MOUSE_PRESSED, (e) => state.mousePress?.(e)) + +// general keyboard handler sub(KEY_PRESSED, onKeyPress); sub(KEY_RELEASED, onKeyRelease); + sub(SETTINGS_CHANGED, () => message.show('Settings have been updated')); sub(AXIS_CHANGED, onAxisChanged); sub(CONTROLLER_UPDATED, data => webrtc.input(data)); @@ -526,4 +556,8 @@ let [roomId, zone] = room.loadMaybe(); const wid = new URLSearchParams(document.location.search).get('wid'); // if from URL -> start game immediately! socket.init(roomId, wid, zone); -api.transport = socket; +api.transport = { + send: socket.send, + keyboard: webrtc.keyboard, + mouse: webrtc.mouse, +}; diff --git a/web/js/event.js b/web/js/event.js index df189044e..37260b9cb 100644 --- a/web/js/event.js +++ b/web/js/event.js @@ -85,9 +85,18 @@ export const KEY_PRESSED = 'keyPressed'; export const KEY_RELEASED = 'keyReleased'; export const KEYBOARD_TOGGLE_FILTER_MODE = 'keyboardToggleFilterMode'; export const KEYBOARD_KEY_PRESSED = 'keyboardKeyPressed'; +export const KEYBOARD_KEY_DOWN = 'keyboardKeyDown'; +export const KEYBOARD_KEY_UP = 'keyboardKeyUp'; + export const AXIS_CHANGED = 'axisChanged'; export const CONTROLLER_UPDATED = 'controllerUpdated'; +export const MOUSE_MOVED = 'mouseMoved'; +export const MOUSE_PRESSED = 'mousePressed'; + +export const FULLSCREEN_CHANGE = 'fsc'; +export const POINTER_LOCK_CHANGE = 'plc'; + export const DPAD_TOGGLE = 'dpadToggle'; export const STATS_TOGGLE = 'statsToggle'; export const HELP_OVERLAY_TOGGLED = 'helpOverlayToggled'; @@ -98,3 +107,4 @@ export const RECORDING_TOGGLED = 'recordingToggle' export const RECORDING_STATUS_CHANGED = 'recordingStatusChanged' export const APP_VIDEO_CHANGED = 'appVideoChanged' +export const KB_MOUSE_FLAG = 'kbMouseFlag' diff --git a/web/js/gui.js b/web/js/gui.js index 8be85b190..ee5956283 100644 --- a/web/js/gui.js +++ b/web/js/gui.js @@ -33,7 +33,7 @@ const select = (key = '', callback = () => ({}), values = {values: [], labels: [ select.append(_option(0, current === '', 'none')); values.values.forEach((value, index) => { - select.append(_option(value, current === value, values.labels?.[index])); + select.append(_option(value, current == value, values.labels?.[index])); }); return el; diff --git a/web/js/input/keyboard.js b/web/js/input/keyboard.js index 1ccba499f..c31769a21 100644 --- a/web/js/input/keyboard.js +++ b/web/js/input/keyboard.js @@ -1,12 +1,16 @@ import { pub, sub, - KEYBOARD_TOGGLE_FILTER_MODE, AXIS_CHANGED, DPAD_TOGGLE, + FULLSCREEN_CHANGE, + KB_MOUSE_FLAG, KEY_PRESSED, KEY_RELEASED, - KEYBOARD_KEY_PRESSED + KEYBOARD_KEY_PRESSED, + KEYBOARD_KEY_DOWN, + KEYBOARD_KEY_UP, + KEYBOARD_TOGGLE_FILTER_MODE } from 'event'; import {KEY} from 'input'; import {log} from 'log' @@ -47,11 +51,15 @@ const defaultMap = Object.freeze({ }); let keyMap = {}; +let isFullscreen = false; +// special mode for changing button bindings in the options let isKeysFilteredMode = true; +// if the browser supports Keyboard Lock API (Firefox does not) +let hasKeyboardLock = false; const remap = (map = {}) => { settings.set(opts.INPUT_KEYBOARD_MAP, map); - log.info('Keyboard keys have been remapped') + log.debug('Keyboard keys have been remapped'); } sub(KEYBOARD_TOGGLE_FILTER_MODE, data => { @@ -90,7 +98,6 @@ function onDpadToggle(checked) { const onKey = (code, evt, state) => { const key = keyMap[code] - if (key === undefined) return if (dpadState[key] !== undefined) { dpadState[key] = state @@ -103,11 +110,26 @@ const onKey = (code, evt, state) => { return } } - pub(evt, {key: key}) + pub(evt, {key: key, code: code}) } sub(DPAD_TOGGLE, (data) => onDpadToggle(data.checked)); +sub(KB_MOUSE_FLAG, () => { + hasKeyboardLock = ('keyboard' in navigator) && ('lock' in navigator.keyboard); + if (!hasKeyboardLock) { + log.warn("Browser doesn't support keyboard lock! It will be emulated."); + } + + sub(FULLSCREEN_CHANGE, async (fullscreenEl) => { + isFullscreen = !!fullscreenEl; + if (hasKeyboardLock) { + isFullscreen ? await navigator.keyboard.lock() : navigator.keyboard.unlock(); + } + log.debug(`Keyboard lock: ${isFullscreen}`); + }) +}) + /** * Keyboard controls. */ @@ -115,23 +137,29 @@ export const keyboard = { init: () => { keyMap = settings.loadOr(opts.INPUT_KEYBOARD_MAP, defaultMap); const body = document.body; - // !to use prevent default as everyone + body.addEventListener('keyup', e => { e.stopPropagation(); - if (isKeysFilteredMode) { - onKey(e.code, KEY_RELEASED, false) - } else { - pub(KEYBOARD_KEY_PRESSED, {key: e.code}); + !hasKeyboardLock && isFullscreen && e.preventDefault(); + + let lock = isFullscreen; + // hack with Esc up when outside of lock + if (e.code === 'Escape') { + lock = true } + + isKeysFilteredMode ? + (lock ? pub(KEYBOARD_KEY_UP, e) : onKey(e.code, KEY_RELEASED, false)) + : pub(KEYBOARD_KEY_PRESSED, {key: e.code}); }, false); body.addEventListener('keydown', e => { e.stopPropagation(); - if (isKeysFilteredMode) { - onKey(e.code, KEY_PRESSED, true) - } else { + !hasKeyboardLock && isFullscreen && e.preventDefault(); + + isKeysFilteredMode ? + (isFullscreen ? pub(KEYBOARD_KEY_DOWN, e) : onKey(e.code, KEY_PRESSED, true)) : pub(KEYBOARD_KEY_PRESSED, {key: e.code}); - } }); log.info('[input] keyboard has been initialized'); diff --git a/web/js/network/webrtc.js b/web/js/network/webrtc.js index 5e8ae47d4..d9ec33e6a 100644 --- a/web/js/network/webrtc.js +++ b/web/js/network/webrtc.js @@ -10,6 +10,8 @@ import {log} from 'log'; let connection; let dataChannel; +let keyboardChannel; +let mouseChannel; let mediaStream; let candidates = []; let isAnswered = false; @@ -30,6 +32,16 @@ const start = (iceservers) => { log.debug('[rtc] ondatachannel', e.channel.label) e.channel.binaryType = "arraybuffer"; + if (e.channel.label === 'keyboard') { + keyboardChannel = e.channel; + return; + } + + if (e.channel.label === 'mouse') { + mouseChannel = e.channel + return; + } + dataChannel = e.channel; dataChannel.onopen = () => { log.info('[rtc] the input channel has been opened'); @@ -65,6 +77,14 @@ const stop = () => { dataChannel.close(); dataChannel = null; } + if (keyboardChannel) { + keyboardChannel.close(); + keyboardChannel = null; + } + if (mouseChannel) { + mouseChannel.close(); + mouseChannel = null; + } candidates = []; log.info('[rtc] WebRTC has been closed'); } @@ -165,6 +185,8 @@ export const webrtc = { }); isFlushing = false; }, + keyboard: (data) => keyboardChannel.send(data), + mouse: (data) => mouseChannel.send(data), input: (data) => dataChannel.send(data), isConnected: () => connected, isInputReady: () => inputReady, diff --git a/web/js/settings.js b/web/js/settings.js index e63b1394d..3b7ee68ea 100644 --- a/web/js/settings.js +++ b/web/js/settings.js @@ -487,6 +487,8 @@ const render = function () { case opts.INPUT_KEYBOARD_MAP: _option(data).withName('Keyboard bindings') .withClass('keyboard-bindings') + .withDescription( + 'Bindings for RetroPad. There is an alternate ESC key [Shift+`] (tilde) for cores with keyboard+mouse controls (DosBox)') .add(Object.keys(value).map(k => gui.binding(value[k], k, onKeyBindingChange))) .build(); break; diff --git a/web/js/stream.js b/web/js/stream.js index 3ccfaee49..3cb27ee9e 100644 --- a/web/js/stream.js +++ b/web/js/stream.js @@ -1,7 +1,13 @@ import {env} from 'env'; import { + pub, sub, APP_VIDEO_CHANGED, + FULLSCREEN_CHANGE, + KB_MOUSE_FLAG, + MOUSE_MOVED, + MOUSE_PRESSED, + POINTER_LOCK_CHANGE, SETTINGS_CHANGED } from 'event' ; import {gui} from 'gui'; @@ -20,6 +26,7 @@ let options = { state = { screen: screen, fullscreen: false, + kbmLock: false, timerId: null, w: 0, h: 0, @@ -50,6 +57,23 @@ const toggleFullscreen = () => { const getVideoEl = () => screen +const getActualVideoSize = () => { + if (state.fullscreen) { + // we can't get real