diff --git a/pkg/network/webrtc/webrtc.go b/pkg/network/webrtc/webrtc.go index 25612d06e..92e5b0074 100644 --- a/pkg/network/webrtc/webrtc.go +++ b/pkg/network/webrtc/webrtc.go @@ -12,10 +12,11 @@ import ( ) type Peer struct { - api *ApiFactory - conn *webrtc.PeerConnection - log *logger.Logger - OnMessage func(data []byte) + api *ApiFactory + conn *webrtc.PeerConnection + log *logger.Logger + OnMessage func(data []byte) + OnKeyboard func(data []byte) a *webrtc.TrackLocalStaticSample v *webrtc.TrackLocalStaticSample @@ -82,11 +83,32 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp p.a = audio // plug in the [data] channel (in and out) - if err = p.addDataChannel("data"); err != nil { + dChan, err := p.addDataChannel("data") + if err != nil { return "", err } + dChan.OnMessage(func(m webrtc.DataChannelMessage) { + if len(m.Data) == 0 { + return + } + if p.OnMessage != nil { + p.OnMessage(m.Data) + } + }) + p.d = dChan p.log.Debug().Msg("Added [data] chan") + kChan, err := p.addDataChannel("keyboard") + if err != nil { + return "", err + } + kChan.OnMessage(func(m webrtc.DataChannelMessage) { + if p.OnKeyboard != nil { + p.OnKeyboard(m.Data) + } + }) + p.log.Debug().Msg("Added [keyboard] chan") + p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") })) // Stream provider supposes to send offer offer, err := p.conn.CreateOffer(nil) @@ -232,29 +254,19 @@ func (p *Peer) Disconnect() { p.log.Debug().Msg("WebRTC stop") } -// addDataChannel creates a new WebRTC data channel for user input. +// addDataChannel creates new WebRTC data channel. // Default params -- ordered: true, negotiated: false. -func (p *Peer) addDataChannel(label string) error { +func (p *Peer) addDataChannel(label string) (*webrtc.DataChannel, error) { ch, err := p.conn.CreateDataChannel(label, nil) if err != nil { - return err + return nil, err } ch.OnOpen(func() { - p.log.Debug().Str("label", ch.Label()).Uint16("id", *ch.ID()). - Msg("Data channel [input] opened") + p.log.Debug().Uint16("id", *ch.ID()).Msgf("Data channel [%v] opened", ch.Label()) }) ch.OnError(p.logx) - ch.OnMessage(func(m webrtc.DataChannelMessage) { - if len(m.Data) == 0 { - return - } - if p.OnMessage != nil { - p.OnMessage(m.Data) - } - }) - p.d = ch - ch.OnClose(func() { p.log.Debug().Msg("Data channel [input] has been closed") }) - return nil + ch.OnClose(func() { p.log.Debug().Msgf("Data channel [%v] has been closed", ch.Label()) }) + return ch, nil } func (p *Peer) logx(err error) { p.log.Error().Err(err) } diff --git a/pkg/worker/caged/app/app.go b/pkg/worker/caged/app/app.go index fcf34fd9f..19cadbde5 100644 --- a/pkg/worker/caged/app/app.go +++ b/pkg/worker/caged/app/app.go @@ -13,6 +13,7 @@ type App interface { SetVideoCb(func(Video)) SetDataCb(func([]byte)) SendControl(port int, data []byte) + SendKeyboardKey(port int, down bool, key uint) } type Audio struct { diff --git a/pkg/worker/caged/libretro/caged.go b/pkg/worker/caged/libretro/caged.go index dea9bf5c0..04dc278d6 100644 --- a/pkg/worker/caged/libretro/caged.go +++ b/pkg/worker/caged/libretro/caged.go @@ -79,15 +79,16 @@ func (c *Caged) EnableCloudStorage(uid string, storage cloud.Storage) { } } -func (c *Caged) AspectEnabled() bool { return c.base.nano.Aspect } -func (c *Caged) AspectRatio() float32 { return c.base.AspectRatio() } -func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat() } -func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } -func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } -func (c *Caged) ViewportSize() (int, int) { return c.base.ViewportSize() } -func (c *Caged) Scale() float64 { return c.Emulator.Scale() } -func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) } -func (c *Caged) Start() { go c.Emulator.Start() } -func (c *Caged) SetSaveOnClose(v bool) { c.base.SaveOnClose = v } -func (c *Caged) SetSessionId(name string) { c.base.SetSessionId(name) } -func (c *Caged) Close() { c.Emulator.Close() } +func (c *Caged) AspectEnabled() bool { return c.base.nano.Aspect } +func (c *Caged) AspectRatio() float32 { return c.base.AspectRatio() } +func (c *Caged) PixFormat() uint32 { return c.Emulator.PixFormat() } +func (c *Caged) Rotation() uint { return c.Emulator.Rotation() } +func (c *Caged) AudioSampleRate() int { return c.Emulator.AudioSampleRate() } +func (c *Caged) ViewportSize() (int, int) { return c.base.ViewportSize() } +func (c *Caged) Scale() float64 { return c.Emulator.Scale() } +func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) } +func (c *Caged) SendKeyboardKey(port int, down bool, key uint) { c.base.KeyboardEvent(down, key, 0, 0) } +func (c *Caged) Start() { go c.Emulator.Start() } +func (c *Caged) SetSaveOnClose(v bool) { c.base.SaveOnClose = v } +func (c *Caged) SetSessionId(name string) { c.base.SetSessionId(name) } +func (c *Caged) Close() { c.Emulator.Close() } diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go index 481966635..ada3bbc64 100644 --- a/pkg/worker/caged/libretro/frontend.go +++ b/pkg/worker/caged/libretro/frontend.go @@ -47,6 +47,7 @@ type Emulator interface { Input(player int, data []byte) // Scale returns set video scale factor Scale() float64 + KeyboardEvent(down bool, keycode uint, character uint32, keyModifiers uint16) } type Frontend struct { @@ -83,7 +84,7 @@ type ( ) const ( - maxPort = 4 + maxPort = 8 dpadAxes = 4 ) @@ -402,6 +403,10 @@ func (f *Frontend) autosave(periodSec int) { } } +func (f *Frontend) KeyboardEvent(down bool, keycode uint, character uint32, keyModifiers uint16) { + f.nano.KeyboardEvent(down, keycode, character, keyModifiers) +} + func NewGameSessionInput() InputState { return [maxPort]State{} } // setInput sets input state for some player in a game session. diff --git a/pkg/worker/caged/libretro/nanoarch/input.go b/pkg/worker/caged/libretro/nanoarch/input.go new file mode 100644 index 000000000..a55ce6b6b --- /dev/null +++ b/pkg/worker/caged/libretro/nanoarch/input.go @@ -0,0 +1,5 @@ +package nanoarch + +const RetroDeviceTypeShift = 8 + +func RetroDeviceSubclass(base, id int) int { return ((id + 1) << RetroDeviceTypeShift) | base } diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.c b/pkg/worker/caged/libretro/nanoarch/nanoarch.c index 4edd73ff7..3300900ec 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.c +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.c @@ -127,6 +127,10 @@ void bridge_clear_all_thread_waits_cb(void *data) { *(retro_environment_t *)data = clear_all_thread_waits_cb; } +void bridge_retro_keyboard_callback(void *cb, bool down, unsigned keycode, uint32_t character, uint16_t keyModifiers) { + (*(retro_keyboard_event_t *) cb)(down, keycode, character, keyModifiers); +} + bool core_environment_cgo(unsigned cmd, void *data) { bool coreEnvironment(unsigned, void *); return coreEnvironment(cmd, data); diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.go b/pkg/worker/caged/libretro/nanoarch/nanoarch.go index 8850522d1..6dde2f05c 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.go +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.go @@ -41,6 +41,7 @@ var ( type Nanoarch struct { Handlers + Callbacks LastFrameTime int64 LibCo bool meta Metadata @@ -81,6 +82,10 @@ type Handlers struct { OnSystemAvInfo func() } +type Callbacks struct { + KeyboardEvent func(down bool, keycode uint, character uint32, keyModifiers uint16) +} + type FrameInfo struct { W uint H uint @@ -128,6 +133,9 @@ var Nan0 = Nanoarch{ OnAudio: func(unsafe.Pointer, int) {}, OnVideo: func([]byte, int32, FrameInfo) {}, }, + Callbacks: Callbacks{ + KeyboardEvent: func(bool, uint, uint32, uint16) {}, + }, } // init provides a global single instance lock @@ -596,31 +604,33 @@ func coreInputPoll() {} //export coreInputState func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.unsigned) C.int16_t { - if uint(port) >= uint(MaxPort) { - return KeyReleased - } - - if device == C.RETRO_DEVICE_ANALOG { - if index > C.RETRO_DEVICE_INDEX_ANALOG_RIGHT || id > C.RETRO_DEVICE_ID_ANALOG_Y { - return 0 - } - axis := index*2 + id - value := Nan0.Handlers.OnDpad(uint(port), uint(axis)) - if value != 0 { + //Nan0.log.Debug().Msgf("%v %v %v %v", port, device, index, id) + + //if uint(port) >= uint(MaxPort) { + // return KeyReleased + //} + + switch device { + case C.RETRO_DEVICE_JOYPAD: + return C.int16_t(Nan0.Handlers.OnKeyPress(uint(port), int(id))) // 0 1 + case C.RETRO_DEVICE_ANALOG: + switch index { + case C.RETRO_DEVICE_INDEX_ANALOG_LEFT: + value := Nan0.Handlers.OnDpad(uint(port), uint(index*2+id)) + //Nan0.log.Debug().Msgf(">>> analog: %v", value) return (C.int16_t)(value) + case C.RETRO_DEVICE_INDEX_ANALOG_RIGHT: + case C.RETRO_DEVICE_INDEX_ANALOG_BUTTON: } } - key := int(id) - if key > lastKey || index > 0 || device != C.RETRO_DEVICE_JOYPAD { - return KeyReleased - } - if Nan0.Handlers.OnKeyPress(uint(port), key) == KeyPressed { - return KeyPressed - } return KeyReleased } +//func retroKeyboardEvent(down C.bool, keycode C.unsigned, uint32_t character, uint16_t key_modifiers) { +// retroKeyboardEvent(down, keycode, character, key_modifiers) +//} + //export coreAudioSample func coreAudioSample(l, r C.int16_t) { frame := []C.int16_t{l, r} @@ -779,6 +789,20 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool { Nan0.log.Debug().Msgf("%v", cInfo.String()) } return true + case C.RETRO_ENVIRONMENT_GET_INPUT_MAX_USERS: + *(*C.unsigned)(data) = C.unsigned(4) + Nan0.log.Debug().Msgf(">>> set max users: %v", 4) + return true + //case C.RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK: + // Nan0.log.Debug().Msgf("Keyboard event callback was set") + // cb := (*C.struct_retro_keyboard_callback)(data) + // Nan0.Callbacks.KeyboardEvent = func(down bool, keycode uint, character uint32, keyModifiers uint16) { + // C.bridge_retro_keyboard_callback(unsafe.Pointer(&cb.callback), C.bool(down), C.unsigned(keycode), C.uint32_t(character), C.uint16_t(keyModifiers)) + // } + // return true + case C.RETRO_ENVIRONMENT_GET_INPUT_BITMASKS: + Nan0.log.Debug().Msgf(">>> Set input bitmasks") + return false case C.RETRO_ENVIRONMENT_GET_CLEAR_ALL_THREAD_WAITS_CB: C.bridge_clear_all_thread_waits_cb(data) return true diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.h b/pkg/worker/caged/libretro/nanoarch/nanoarch.h index 0c4b01776..661036434 100644 --- a/pkg/worker/caged/libretro/nanoarch/nanoarch.h +++ b/pkg/worker/caged/libretro/nanoarch/nanoarch.h @@ -23,6 +23,7 @@ void bridge_retro_set_input_poll(void *f, void *callback); void bridge_retro_set_input_state(void *f, void *callback); void bridge_retro_set_video_refresh(void *f, void *callback); void bridge_clear_all_thread_waits_cb(void *f); +void bridge_retro_keyboard_callback(void *f, bool down, unsigned keycode, uint32_t character, uint16_t keyModifiers); bool core_environment_cgo(unsigned cmd, void *data); int16_t core_input_state_cgo(unsigned port, unsigned device, unsigned index, unsigned id); diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go index 41e150f8c..bc944e592 100644 --- a/pkg/worker/coordinatorhandlers.go +++ b/pkg/worker/coordinatorhandlers.go @@ -2,6 +2,7 @@ package worker import ( "encoding/base64" + "encoding/binary" "github.com/giongto35/cloud-game/v3/pkg/api" "github.com/giongto35/cloud-game/v3/pkg/com" @@ -169,7 +170,14 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke } c.log.Debug().Msg("Start session input poll") - room.WithWebRTC(user.Session).OnMessage = func(data []byte) { r.App().SendControl(user.Index, data) } + s := room.WithWebRTC(user.Session) + s.OnMessage = func(data []byte) { r.App().SendControl(user.Index, data) } + s.OnKeyboard = func(data []byte) { + press := data[4] == 1 + key := uint(binary.LittleEndian.Uint32(data)) + c.log.Debug().Msgf(">>> got %v, %v, %v", data, press, key) + r.App().SendKeyboardKey(0, press, key) + } c.RegisterRoom(r.Id()) diff --git a/web/index.html b/web/index.html index bc673e5e2..a7856cdfe 100644 --- a/web/index.html +++ b/web/index.html @@ -121,6 +121,7 @@

Options

+ diff --git a/web/js/controller.js b/web/js/controller.js index edd0e61e7..2db4613f5 100644 --- a/web/js/controller.js +++ b/web/js/controller.js @@ -214,7 +214,7 @@ if (KEY.HELP === data.key) helpScreen.show(true, event); } - state.keyPress(data.key); + state.keyPress(data.key, data.code); }; // pre-state key release handler @@ -241,7 +241,7 @@ // change app state if settings if (KEY.SETTINGS === data.key) setState(app.state.settings); - state.keyRelease(data.key); + state.keyRelease(data.key, data.code); }; const updatePlayerIndex = (idx, not_game = false) => { @@ -367,8 +367,33 @@ ..._default, name: 'game', axisChanged: (id, value) => input.setAxisChanged(id, value), - keyPress: key => input.setKeyState(key, true), - keyRelease: function (key) { + keyPress: (key, code) => { + let buffer = new ArrayBuffer(5); + let dv = new DataView(buffer) + + dv.setUint8(4, 1) + const k = libretro.map(key, code); + dv.setUint32(0, k, true) + + console.log("code", code, k) + + webrtc.keyboard(buffer); + input.setKeyState(key, true); + }, + keyRelease: function (key, code) { + + let buffer = new ArrayBuffer(5); + let dv = new DataView(buffer) + + dv.setUint8(4, 0) + const k = libretro.map(key, code); + dv.setUint32(0, k, true) + + console.log("code", code, k) + + + webrtc.keyboard(buffer); + input.setKeyState(key, false); switch (key) { diff --git a/web/js/input/keyboard.js b/web/js/input/keyboard.js index 3a09ba9ef..bcfecb39b 100644 --- a/web/js/input/keyboard.js +++ b/web/js/input/keyboard.js @@ -96,7 +96,7 @@ const keyboard = (() => { return } } - event.pub(evt, {key: key}) + event.pub(evt, {key: key, code: code}) } event.sub(DPAD_TOGGLE, (data) => onDpadToggle(data.checked)); diff --git a/web/js/input/libretro.js b/web/js/input/libretro.js new file mode 100644 index 000000000..9ad87b0dc --- /dev/null +++ b/web/js/input/libretro.js @@ -0,0 +1,166 @@ +// RETRO_KEYBOARD +const retro = { + '': 0, + 'Unknown': 0, // ??? + 'First': 0, // ??? + 'Backspace': 8, + 'Tab': 9, + 'Clear': 12, + 'Enter': 13, 'Return': 13, + 'Pause': 19, + 'Escape': 27, + 'Space': 32, + 'Exclaim': 33, + 'Quotedbl': 34, + 'Hash': 35, + 'Dollar': 36, + 'Ampersand': 38, + 'Quote': 39, + 'Leftparen': 40, '(': 40, + 'Rightparen': 41, ')': 41, + 'Asterisk': 42, + 'Plus': 43, + 'Comma': 44, + 'Minus': 45, + 'Period': 46, + 'Slash': 47, + 'Digit0': 48, + 'Digit1': 49, + 'Digit2': 50, + 'Digit3': 51, + 'Digit4': 52, + 'Digit5': 53, + 'Digit6': 54, + 'Digit7': 55, + 'Digit8': 56, + 'Digit9': 57, + 'Colon': 58, ':': 58, + 'Semicolon': 59, ';': 59, + 'Less': 60, '<': 60, + 'Equal': 61, '=': 61, + 'Greater': 62, '>': 62, + 'Question': 63, '?': 63, + // RETROK_AT = 64, + 'BracketLeft': 91, '[': 91, + 'Backslash': 92, '\\': 92, + 'BracketRight': 93, ']': 93, + // RETROK_CARET = 94, + // RETROK_UNDERSCORE = 95, + 'Backquote': 96, '`': 96, + 'KeyA': 97, + 'KeyB': 98, + 'KeyC': 99, + 'KeyD': 100, + 'KeyE': 101, + 'KeyF': 102, + 'KeyG': 103, + 'KeyH': 104, + 'KeyI': 105, + 'KeyJ': 106, + 'KeyK': 107, + 'KeyL': 108, + 'KeyM': 109, + 'KeyN': 110, + 'KeyO': 111, + 'KeyP': 112, + 'KeyQ': 113, + 'KeyR': 114, + 'KeyS': 115, + 'KeyT': 116, + 'KeyU': 117, + 'KeyV': 118, + 'KeyW': 119, + 'KeyX': 120, + 'KeyY': 121, + 'KeyZ': 122, + // RETROK_LEFTBRACE = 123, + // RETROK_BAR = 124, + // RETROK_RIGHTBRACE = 125, + 'Tilde': 126, '~': 126, + 'Delete': 127, + + // RETROK_KP0 = 256, + // RETROK_KP1 = 257, + // RETROK_KP2 = 258, + // RETROK_KP3 = 259, + // RETROK_KP4 = 260, + // RETROK_KP5 = 261, + // RETROK_KP6 = 262, + // RETROK_KP7 = 263, + // RETROK_KP8 = 264, + // RETROK_KP9 = 265, + // RETROK_KP_PERIOD = 266, + // RETROK_KP_DIVIDE = 267, + // RETROK_KP_MULTIPLY = 268, + // RETROK_KP_MINUS = 269, + // RETROK_KP_PLUS = 270, + // RETROK_KP_ENTER = 271, + // RETROK_KP_EQUALS = 272, + + 'ArrowUp': 273, + 'ArrowDown': 274, + 'ArrowRight': 275, + 'ArrowLeft': 276, + 'Insert': 277, + 'Home': 278, + 'End': 279, + 'PageUp': 280, + 'PageDown': 281, + + 'F1': 282, + 'F2': 283, + 'F3': 284, + 'F4': 285, + 'F5': 286, + 'F6': 287, + 'F7': 288, + 'F8': 289, + 'F9': 290, + 'F10': 291, + 'F11': 292, + 'F12': 293, + 'F13': 294, + 'F14': 295, + 'F15': 296, + + 'NumLock': 300, + 'CapsLock': 301, + 'ScrollLock': 302, + 'ShiftRight': 303, + 'ShiftLeft': 304, + 'ControlRight': 305, + 'ControlLeft': 306, + 'AltRight': 307, + 'AltLeft': 308, + 'MetaRight': 309, + 'MetaLeft': 310, + // RETROK_LSUPER = 311, + // RETROK_RSUPER = 312, + // RETROK_MODE = 313, + // RETROK_COMPOSE = 314, + + // RETROK_HELP = 315, + // RETROK_PRINT = 316, + // RETROK_SYSREQ = 317, + // RETROK_BREAK = 318, + // RETROK_MENU = 319, + // RETROK_POWER = 320, + // RETROK_EURO = 321, + // RETROK_UNDO = 322, + // RETROK_OEM_102 = 323, + + // RETROK_LAST, + + // RETROK_DUMMY = INT_MAX /* Ensure sizeof(enum) == sizeof(int) */ +}; + + +const libretro = (() => { + const _map = (key = '', code = '') => { + return retro[code] || retro[key] || 0 + } + + return { + map: _map, + } +})() diff --git a/web/js/network/webrtc.js b/web/js/network/webrtc.js index 8d72c8fd1..75ba3a9f7 100644 --- a/web/js/network/webrtc.js +++ b/web/js/network/webrtc.js @@ -13,6 +13,7 @@ const webrtc = (() => { let connection; let dataChannel; + let keyboardChannel; let mediaStream; let candidates = []; let isAnswered = false; @@ -31,16 +32,21 @@ const webrtc = (() => { connection.ondatachannel = e => { log.debug('[rtc] ondatachannel', e.channel.label) - dataChannel = e.channel; - dataChannel.onopen = () => { - log.info('[rtc] the input channel has been opened'); - inputReady = true; - event.pub(WEBRTC_CONNECTION_READY) - }; - if (onData) { - dataChannel.onmessage = onData; + + if (e.channel.label === 'keyboard') { + keyboardChannel = e.channel; + } else { + dataChannel = e.channel; + dataChannel.onopen = () => { + log.info('[rtc] the input channel has been opened'); + inputReady = true; + event.pub(WEBRTC_CONNECTION_READY) + }; + if (onData) { + dataChannel.onmessage = onData; + } + dataChannel.onclose = () => log.info('[rtc] the input channel has been closed'); } - dataChannel.onclose = () => log.info('[rtc] the input channel has been closed'); } connection.oniceconnectionstatechange = ice.onIceConnectionStateChange; connection.onicegatheringstatechange = ice.onIceStateChange; @@ -66,6 +72,10 @@ const webrtc = (() => { dataChannel.close(); dataChannel = null; } + if (keyboardChannel) { + keyboardChannel.close(); + keyboardChannel = null; + } candidates = Array(); log.info('[rtc] WebRTC has been closed'); } @@ -173,6 +183,7 @@ const webrtc = (() => { // return false // } // }, + keyboard: (data) => keyboardChannel.send(data), input: (data) => dataChannel.send(data), isConnected: () => connected, isInputReady: () => inputReady,