Use generic audio callback in caged apps

pkg/worker/caged/app/app.go

@@ -0,0 +1,22 @@
+package app
+
+import "unsafe"
+
+type App interface {
+	AudioSampleRate() int
+	Init() error
+	ViewportSize() (int, int)
+	Start()
+	Close()
+
+	SetAudioCb(func(Audio))
+}
+
+type Audio struct {
+	Data     []byte
+	Duration int64
+}
+
+func (a Audio) ToPCM() []int16 {
+	return unsafe.Slice((*int16)(unsafe.Pointer(unsafe.SliceData(a.Data))), len(a.Data)>>1)
+}

pkg/worker/caged/caged.go

@@ -6,19 +6,12 @@ import (
 
 	"github.com/giongto35/cloud-game/v3/pkg/config"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
+	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/libretro"
 )
 
-type App interface {
-	AudioSampleRate() int
-	Init() error
-	ViewportSize() (int, int)
-	Start()
-	Close()
-}
-
 type Manager struct {
-	list map[ModName]App
+	list map[ModName]app.App
 	log  *logger.Logger
 }
 
@@ -27,18 +20,18 @@ type ModName string
 const Libretro ModName = "libretro"
 
 func NewManager(log *logger.Logger) *Manager {
-	return &Manager{log: log, list: make(map[ModName]App)}
+	return &Manager{log: log, list: make(map[ModName]app.App)}
 }
 
-func (m *Manager) Get(name ModName) App { return m.list[name] }
+func (m *Manager) Get(name ModName) app.App { return m.list[name] }
 
 func (m *Manager) Load(name ModName, conf any) error {
 	if name == Libretro {
-		app, err := m.loadLibretro(conf)
+		caged, err := m.loadLibretro(conf)
 		if err != nil {
 			return err
 		}
-		m.list[name] = app
+		m.list[name] = caged
 	}
 	return nil
 }
@@ -60,9 +53,9 @@ func (m *Manager) loadLibretro(conf any) (*libretro.Caged, error) {
 		Recording: r.Interface().(config.Recording),
 	}
 
-	app := libretro.Cage(c, m.log)
-	if err := app.Init(); err != nil {
+	caged := libretro.Cage(c, m.log)
+	if err := caged.Init(); err != nil {
 		return nil, err
 	}
-	return &app, nil
+	return &caged, nil
 }

pkg/worker/caged/libretro/frontend.go

@@ -8,20 +8,22 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/giongto35/cloud-game/v3/pkg/config"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
 	"github.com/giongto35/cloud-game/v3/pkg/os"
+	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/libretro/image"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/libretro/nanoarch"
 )
 
 type Emulator interface {
-	// SetAudio sets the audio callback
-	SetAudio(func(*GameAudio))
+	// SetAudioCb sets the audio callback
+	SetAudioCb(func(app.Audio))
 	// SetVideo sets the video callback
 	SetVideo(func(*GameFrame))
-	Audio() func(*GameAudio)
+	Audio() func(app.Audio)
 	Video() func(*GameFrame)
 	LoadCore(name string)
 	LoadGame(path string) error
@@ -53,45 +55,29 @@ type Emulator interface {
 }
 
 type Frontend struct {
-	onVideo func(*GameFrame)
-	onAudio func(*GameAudio)
-
-	input InputState
-
+	canvas  *image.Canvas
 	conf    config.Emulator
+	done    chan struct{}
+	input   InputState
+	log     *logger.Logger
+	nano    *nanoarch.Nanoarch
+	onAudio func(app.Audio)
+	onVideo func(*GameFrame)
 	storage Storage
+	th      int // draw threads
+	vw, vh  int // out frame size
 
-	// out frame size
-	vw, vh int
-	// draw threads
-	th int
+	mu sync.Mutex
 
-	Canvas            *image.Canvas
 	DisableCanvasPool bool
-
-	done chan struct{}
-	log  *logger.Logger
-
-	frames int
-
-	started chan struct{}
-
-	nano *nanoarch.Nanoarch
-
-	SaveOnClose bool
-
-	mu sync.Mutex
+	SaveOnClose       bool
 }
 
 type (
 	GameFrame struct {
 		Data     *image.Frame
 		Duration time.Duration
 	}
-	GameAudio struct {
-		Data     *[]int16
-		Duration time.Duration
-	}
 	InputEvent struct {
 		RawState []byte
 	}
@@ -114,14 +100,9 @@ const (
 	dpadAxes = 4
 )
 
-const rawAudioBuffer = 4096 // 4K
-var (
-	audioCopyPool sync.Pool
-	audioPool     sync.Pool
-)
-
 var (
-	noAudio   = func(*GameAudio) {}
+	audioPool sync.Pool
+	noAudio   = func(app.Audio) {}
 	noVideo   = func(*GameFrame) {}
 	videoPool sync.Pool
 )
@@ -163,7 +144,6 @@ func NewFrontend(conf config.Emulator, log *logger.Logger) (*Frontend, error) {
 		log:     log,
 		onAudio: noAudio,
 		onVideo: noVideo,
-		started: make(chan struct{}, 1),
 		storage: store,
 		th:      conf.Threads,
 	}
@@ -189,29 +169,17 @@ func (f *Frontend) LoadCore(emu string) {
 	f.mu.Unlock()
 }
 
-func (f *Frontend) handleAudio(data []int16, samples int) {
-	sampleRate := f.nano.AudioSampleRate()
-
-	dst, _ := audioCopyPool.Get().(*[]int16)
-	if dst == nil {
-		x := make([]int16, rawAudioBuffer)
-		dst = &x
-	}
-	xx := (*dst)[:samples]
-	copy(xx, data)
-
-	// 1600 = x / 1000 * 48000 * 2
-	estimate := float64(samples) / float64(sampleRate<<1) * 1000000000
-
-	fr, _ := audioPool.Get().(*GameAudio)
+func (f *Frontend) handleAudio(audio unsafe.Pointer, samples int) {
+	fr, _ := audioPool.Get().(*app.Audio)
 	if fr == nil {
-		fr = &GameAudio{}
+		fr = &app.Audio{}
 	}
-	fr.Data = &xx
-	fr.Duration = time.Duration(estimate) // used in recordings
-	f.onAudio(fr)
+	// !to look if we need a copy
+	fr.Data = unsafe.Slice((*byte)(audio), samples<<1)
+	// due to audio buffering for opus fixed frames and const duration up in the hierarchy,
+	// we skip Duration here
+	f.onAudio(*fr)
 	audioPool.Put(fr)
-	audioCopyPool.Put(dst)
 }
 
 func (f *Frontend) handleVideo(data []byte, delta int64, fi nanoarch.FrameInfo) {
@@ -225,19 +193,19 @@ func (f *Frontend) handleVideo(data []byte, delta int64, fi nanoarch.FrameInfo)
 	}
 	// !to fix possible nil pointer dereference
 	// when the internal pool can be nil during first Get???
-	fr.Data = f.Canvas.Draw(pixFmt, rot, fi.W, fi.H, fi.Packed, bpp, data, f.th)
+	fr.Data = f.canvas.Draw(pixFmt, rot, fi.W, fi.H, fi.Packed, bpp, data, f.th)
 	fr.Duration = time.Duration(delta)
 	f.onVideo(fr)
-	f.Canvas.Put(fr.Data)
+	f.canvas.Put(fr.Data)
 	videoPool.Put(fr)
 }
 
 func (f *Frontend) Shutdown() {
 	f.log.Debug().Msgf("run loop cleanup")
 	f.mu.Lock()
 	f.nano.Shutdown()
-	f.Canvas.Clear()
-	f.SetAudio(noAudio)
+	f.canvas.Clear()
+	f.SetAudioCb(noAudio)
 	f.SetVideo(noVideo)
 	f.mu.Unlock()
 	f.log.Debug().Msgf("run loop finished")
@@ -290,7 +258,7 @@ func (f *Frontend) Start() {
 }
 
 func (f *Frontend) FrameSize() (int, int)         { return f.nano.GeometryBase() }
-func (f *Frontend) Audio() func(*GameAudio)       { return f.onAudio }
+func (f *Frontend) Audio() func(app.Audio)        { return f.onAudio }
 func (f *Frontend) Video() func(*GameFrame)       { return f.onVideo }
 func (f *Frontend) FPS() int                      { return f.nano.VideoFramerate() }
 func (f *Frontend) HashPath() string              { return f.storage.GetSavePath() }
@@ -304,16 +272,16 @@ func (f *Frontend) RestoreGameState() error       { return f.Load() }
 func (f *Frontend) IsPortrait() bool              { return f.nano.IsPortrait() }
 func (f *Frontend) SaveGameState() error          { return f.Save() }
 func (f *Frontend) Scale(factor int)              { w, h := f.ViewportSize(); f.SetViewport(w, h, factor) }
-func (f *Frontend) SetAudio(ff func(*GameAudio))  { f.onAudio = ff }
+func (f *Frontend) SetAudioCb(cb func(app.Audio)) { f.onAudio = cb }
 func (f *Frontend) SetSessionId(name string)      { f.storage.SetMainSaveName(name) }
 func (f *Frontend) SetViewport(width int, height int, scale int) {
 	f.mu.Lock()
 	f.vw, f.vh = width, height
 	mw, mh := f.nano.GeometryMax()
 	size := mw * scale * mh * scale
-	f.Canvas = image.NewCanvas(width, height, size)
+	f.canvas = image.NewCanvas(width, height, size)
 	if f.DisableCanvasPool {
-		f.Canvas.SetEnabled(false)
+		f.canvas.SetEnabled(false)
 	}
 	f.mu.Unlock()
 }

pkg/worker/caged/libretro/frontend_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/giongto35/cloud-game/v3/pkg/config"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
+	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/libretro/nanoarch"
 )
 
@@ -75,7 +76,6 @@ func GetEmulatorMock(room string, system string) *EmulatorMock {
 			done:        make(chan struct{}),
 			th:          conf.Emulator.Threads,
 			log:         l2,
-			started:     make(chan struct{}),
 			SaveOnClose: false,
 		},
 
@@ -101,7 +101,7 @@ func GetDefaultFrontend(room string, system string, rom string) *EmulatorMock {
 	mock := GetEmulatorMock(room, system)
 	mock.loadRom(rom)
 	mock.SetVideo(func(_ *GameFrame) {})
-	mock.SetAudio(func(_ *GameAudio) {})
+	mock.SetAudioCb(func(app.Audio) {})
 
 	return mock
 }
@@ -358,7 +358,7 @@ func TestStateConcurrency(t *testing.T) {
 				t.Errorf("It seems that rom video frame was empty, which is strange!")
 			}
 		})
-		mock.SetAudio(func(_ *GameAudio) {})
+		mock.SetAudioCb(func(app.Audio) {})
 
 		t.Logf("Random seed is [%v]\n", test.seed)
 		t.Logf("Save path is [%v]\n", mock.paths.save)

pkg/worker/caged/libretro/nanoarch/nanoarch.go

@@ -37,7 +37,7 @@ type Nanoarch struct {
 	Handlers struct {
 		OnDpad     func(port uint, axis uint) (shift int16)
 		OnKeyPress func(port uint, key int) int
-		OnAudio    func(data []int16, frames int)
+		OnAudio    func(ptr unsafe.Pointer, frames int)
 		OnVideo    func(data []byte, delta int64, fi FrameInfo)
 	}
 	LastFrameTime int64
@@ -102,12 +102,12 @@ var Nan0 = Nanoarch{
 	Handlers: struct {
 		OnDpad     func(uint, uint) int16
 		OnKeyPress func(uint, int) int
-		OnAudio    func([]int16, int)
+		OnAudio    func(unsafe.Pointer, int)
 		OnVideo    func([]byte, int64, FrameInfo)
 	}{
 		OnDpad:     func(uint, uint) int16 { return 0 },
 		OnKeyPress: func(uint, int) int { return 0 },
-		OnAudio:    func([]int16, int) {},
+		OnAudio:    func(unsafe.Pointer, int) {},
 		OnVideo:    func([]byte, int64, FrameInfo) {},
 	},
 }
@@ -635,10 +635,7 @@ func coreAudioSampleBatch(data unsafe.Pointer, frames C.size_t) C.size_t {
 		}
 		return frames
 	}
-
-	samples := int(frames) << 1
-	Nan0.Handlers.OnAudio(unsafe.Slice((*int16)(data), samples), samples)
-
+	Nan0.Handlers.OnAudio(data, int(frames)<<1)
 	return frames
 }
 

pkg/worker/caged/libretro/recording.go

@@ -1,8 +1,11 @@
 package libretro
 
 import (
+	"time"
+
 	"github.com/giongto35/cloud-game/v3/pkg/config"
 	"github.com/giongto35/cloud-game/v3/pkg/logger"
+	"github.com/giongto35/cloud-game/v3/pkg/worker/caged/app"
 	"github.com/giongto35/cloud-game/v3/pkg/worker/recorder"
 )
 
@@ -27,10 +30,13 @@ func WithRecording(fe Emulator, rec bool, user string, game string, conf config.
 	return rr
 }
 
-func (r *RecordingFrontend) SetAudio(fn func(*GameAudio)) {
-	r.Emulator.SetAudio(func(audio *GameAudio) {
+func (r *RecordingFrontend) SetAudioCb(fn func(app.Audio)) {
+	r.Emulator.SetAudioCb(func(audio app.Audio) {
 		if r.IsRecording() {
-			r.rec.WriteAudio(recorder.Audio{Samples: audio.Data, Duration: audio.Duration})
+			pcm := audio.ToPCM()
+			// example: 1600 = x / 1000 * 48000 * 2
+			l := time.Duration(float64(len(pcm)) / float64(r.AudioSampleRate()<<1) * 1000000000)
+			r.rec.WriteAudio(recorder.Audio{Samples: pcm, Duration: l})
 		}
 		fn(audio)
 	})

pkg/worker/coordinatorhandlers.go

@@ -2,6 +2,7 @@ package worker
 
 import (
 	"encoding/base64"
+	"unsafe"
 
 	"github.com/giongto35/cloud-game/v3/pkg/api"
 	"github.com/giongto35/cloud-game/v3/pkg/com"
@@ -31,6 +32,9 @@ func emulator(wtf any) *libretro.Caged { return wtf.(*libretro.Caged) }
 func recorder(wtf any) *libretro.RecordingFrontend {
 	return (emulator(wtf).Emulator).(*libretro.RecordingFrontend)
 }
+func unwrapAudio(a []byte) []int16 {
+	return unsafe.Slice((*int16)(unsafe.Pointer(unsafe.SliceData(a))), len(a)/2)
+}
 
 func (c *coordinator) HandleWebrtcInit(rq api.WebrtcInitRequest[com.Uid], w *Worker, connApi *webrtc.ApiFactory) api.Out {
 	peer := webrtc.New(c.log, connApi)