Add I420 scaling

.github/workflows/cd/cloudretro.io/config.yaml

@@ -24,14 +24,14 @@ worker:
       domain: cloudretro.io
 
 emulator:
-  threads: 4
   libretro:
     logLevel: 1
     cores:
       list:
         mame:
           options:
-            "fbneo-cpu-speed-adjust": "200%"
             "fbneo-diagnostic-input": "Hold Start"
+        nes:
+          scale: 2
         pcsx:
           altRepo: true

pkg/config/config.yaml

@@ -99,12 +99,9 @@ worker:
   tag:
 
 emulator:
-  # set output viewport scale factor
-  scale: 1
-
   # set the total number of threads for the image processing
-  # (experimental)
-  threads: 4
+  # (removed)
+  threads: 0
 
   aspectRatio:
     # enable aspect ratio changing
@@ -163,6 +160,7 @@ emulator:
       #   - altRepo (bool) prioritize secondary repo as the download source
       #   - lib (string)
       #   - roms ([]string)
+      #   - scale (int) scales the output video frames by this factor.
       #   - folder (string)
       #       By default emulator selection is based on the folder named as cores
       #       in the list (i.e. nes, snes) but if you specify folder param,

pkg/config/emulator.go

@@ -7,7 +7,6 @@ import (
 )
 
 type Emulator struct {
-	Scale       int
 	Threads     int
 	AspectRatio struct {
 		Keep   bool
@@ -54,6 +53,7 @@ type LibretroCoreConfig struct {
 	Lib           string
 	Options       map[string]string
 	Roms          []string
+	Scale         float64
 	UsesLibCo     bool
 	VFR           bool
 	Width         int

pkg/encoder/color/rgba/rgba.go

@@ -0,0 +1,24 @@
+package rgba
+
+import (
+	"image"
+	"image/color"
+)
+
+func ToRGBA(img image.Image, flipped bool) *image.RGBA {
+	bounds := img.Bounds()
+	sw, sh := bounds.Dx(), bounds.Dy()
+	dst := image.NewRGBA(image.Rect(0, 0, sw, sh))
+	for y := 0; y < sh; y++ {
+		yy := y
+		if flipped {
+			yy = sh - y
+		}
+		for x := 0; x < sw; x++ {
+			px := img.At(x, y)
+			rgba := color.RGBAModel.Convert(px).(color.RGBA)
+			dst.Set(x, yy, rgba)
+		}
+	}
+	return dst
+}

pkg/encoder/encoder.go

@@ -22,7 +22,7 @@ type (
 )
 
 type Video struct {
-	encoder Encoder
+	codec   Encoder
 	log     *logger.Logger
 	stopped atomic.Bool
 	y       yuv.Conv
@@ -43,8 +43,8 @@ const (
 // converts them into YUV I420 format,
 // encodes with provided video encoder, and
 // puts the result into the output channel.
-func NewVideoEncoder(enc Encoder, w, h int, log *logger.Logger) *Video {
-	return &Video{encoder: enc, y: yuv.NewYuvConv(w, h), log: log}
+func NewVideoEncoder(codec Encoder, w, h int, scale float64, log *logger.Logger) *Video {
+	return &Video{codec: codec, y: yuv.NewYuvConv(w, h, scale), log: log}
 }
 
 func (v *Video) Encode(frame InFrame) OutFrame {
@@ -55,18 +55,16 @@ func (v *Video) Encode(frame InFrame) OutFrame {
 	}
 
 	yCbCr := v.y.Process(yuv.RawFrame(frame), v.rot, v.pf)
-	v.encoder.LoadBuf(yCbCr)
+	v.codec.LoadBuf(yCbCr)
 	v.y.Put(&yCbCr)
 
-	if bytes := v.encoder.Encode(); len(bytes) > 0 {
+	if bytes := v.codec.Encode(); len(bytes) > 0 {
 		return bytes
 	}
 	return nil
 }
 
-func (v *Video) Info() string {
-	return fmt.Sprintf("libyuv: %v", v.y.Version())
-}
+func (v *Video) Info() string { return fmt.Sprintf("libyuv: %v", v.y.Version()) }
 
 func (v *Video) SetPixFormat(f uint32) {
 	switch f {
@@ -93,15 +91,15 @@ func (v *Video) SetRot(r uint) {
 }
 
 // SetFlip tells the encoder to flip the frames vertically.
-func (v *Video) SetFlip(b bool) { v.encoder.SetFlip(b) }
+func (v *Video) SetFlip(b bool) { v.codec.SetFlip(b) }
 
 func (v *Video) Stop() {
 	v.stopped.Store(true)
 	v.mu.Lock()
 	defer v.mu.Unlock()
 	v.rot = 0
 
-	if err := v.encoder.Shutdown(); err != nil {
+	if err := v.codec.Shutdown(); err != nil {
 		v.log.Error().Err(err).Msg("failed to close the encoder")
 	}
 }

pkg/encoder/yuv/libyuv.go

@@ -54,4 +54,8 @@ func Y420(src []byte, dst []byte, w, h, stride int, dw, dh int, rot uint, pix ui
 	libyuv.Y420(src, dst, w, h, stride, dw, dh, rot, pix, cx, cy)
 }
 
+func Y420Scale(src []byte, dst []byte, w, h int, dw, dh int) {
+	libyuv.Y420Scale(src, dst, w, h, dw, dh)
+}
+
 func Version() string { return fmt.Sprintf("%v mod", libyuv.Version()) }

pkg/encoder/yuv/libyuv/libyuv.go

@@ -9,6 +9,7 @@ package libyuv
 #include "libyuv/version.h"
 #include "libyuv/video_common.h"
 #include "libyuv/rotate.h"
+#include "libyuv/scale.h"
 #include "libyuv/convert.h"
 
 */
@@ -45,4 +46,43 @@ func Y420(src []byte, dst []byte, _, h, stride int, dw, dh int, rot uint, pix ui
 		C.uint32_t(pix))
 }
 
+func Y420Scale(src []byte, dst []byte, w, h int, dw, dh int) {
+	srcWidthUV, dstWidthUV := (w+1)>>1, (dw+1)>>1
+	srcHeightUV, dstHeightUV := (h+1)>>1, (dh+1)>>1
+
+	srcYPlaneSize, dstYPlaneSize := w*h, dw*dh
+	srcUVPlaneSize, dstUVPlaneSize := srcWidthUV*srcHeightUV, dstWidthUV*dstHeightUV
+
+	srcStrideY, dstStrideY := w, dw
+	srcStrideU, dstStrideU := srcWidthUV, dstWidthUV
+	srcStrideV, dstStrideV := srcWidthUV, dstWidthUV
+
+	srcY := (*C.uchar)(&src[0])
+	srcU := (*C.uchar)(&src[srcYPlaneSize])
+	srcV := (*C.uchar)(&src[srcYPlaneSize+srcUVPlaneSize])
+
+	dstY := (*C.uchar)(&dst[0])
+	dstU := (*C.uchar)(&dst[dstYPlaneSize])
+	dstV := (*C.uchar)(&dst[dstYPlaneSize+dstUVPlaneSize])
+
+	C.I420Scale(
+		srcY,
+		C.int(srcStrideY),
+		srcU,
+		C.int(srcStrideU),
+		srcV,
+		C.int(srcStrideV),
+		C.int(w),
+		C.int(h),
+		dstY,
+		C.int(dstStrideY),
+		dstU,
+		C.int(dstStrideU),
+		dstV,
+		C.int(dstStrideV),
+		C.int(dw),
+		C.int(dh),
+		C.enum_FilterMode(C.kFilterNone))
+}
+
 func Version() int { return int(C.LIBYUV_VERSION) }

pkg/encoder/yuv/libyuv/libyuv/scale.h

@@ -21,18 +21,6 @@ typedef enum FilterMode {
     kFilterBox = 3        // Highest quality.
 } FilterModeEnum;
 
-// Scale a YUV plane.
-LIBYUV_API
-void ScalePlane(const uint8_t *src,
-                int src_stride,
-                int src_width,
-                int src_height,
-                uint8_t *dst,
-                int dst_stride,
-                int dst_width,
-                int dst_height,
-                enum FilterMode filtering);
-
 // Scales a YUV 4:2:0 image from the src width and height to the
 // dst width and height.
 // If filtering is kFilterNone, a simple nearest-neighbor algorithm is
@@ -62,4 +50,4 @@ int I420Scale(const uint8_t *src_y,
               int dst_height,
               enum FilterMode filtering);
 
-#endif  // INCLUDE_LIBYUV_SCALE_H_
+#endif  // INCLUDE_LIBYUV_SCALE_H_

pkg/encoder/yuv/yuv.go

@@ -1,10 +1,15 @@
 package yuv
 
-import "sync"
+import (
+	"image"
+	"sync"
+)
 
 type Conv struct {
-	w, h int
-	pool sync.Pool
+	w, h   int
+	sw, sh int
+	scale  float64
+	pool   sync.Pool
 }
 
 type RawFrame struct {
@@ -15,19 +20,22 @@ type RawFrame struct {
 
 type PixFmt uint32
 
-func NewYuvConv(w, h int) Conv {
-	bufSize := w*h + 2*((w+1)/2)*((h+1)/2)
+func NewYuvConv(w, h int, scale float64) Conv {
+	if scale < 1 {
+		scale = 1
+	}
+	sw, sh := round(w, scale), round(h, scale)
+	bufSize := int(float64(sw) * float64(sh) * 1.5)
 	return Conv{
-		w:    w,
-		h:    h,
+		w: w, h: h, sw: sw, sh: sh, scale: scale,
 		pool: sync.Pool{New: func() any { b := make([]byte, bufSize); return &b }},
 	}
 }
 
 // Process converts an image to YUV I420 format inside the internal buffer.
 func (c *Conv) Process(frame RawFrame, rot uint, pf PixFmt) []byte {
-	dx, dy := c.w, c.h
-	cx, cy := c.w, c.h
+	dx, dy := c.w, c.h // dest
+	cx, cy := c.w, c.h // crop
 	if rot == 90 || rot == 270 {
 		cx, cy = cy, cx
 	}
@@ -38,10 +46,31 @@ func (c *Conv) Process(frame RawFrame, rot uint, pf PixFmt) []byte {
 	}
 
 	buf := *c.pool.Get().(*[]byte)
-
 	Y420(frame.Data, buf, frame.W, frame.H, stride, dx, dy, rot, uint32(pf), cx, cy)
+
+	if c.scale > 1 {
+		dstBuf := *c.pool.Get().(*[]byte)
+		Y420Scale(buf, dstBuf, dx, dy, c.sw, c.sh)
+		c.pool.Put(&buf)
+		return dstBuf
+	}
 	return buf
 }
 
-func (c *Conv) Put(x *[]byte)   { c.pool.Put(x) }
-func (c *Conv) Version() string { return Version() }
+func (c *Conv) Put(x *[]byte)        { c.pool.Put(x) }
+func (c *Conv) Version() string      { return Version() }
+func round(x int, scale float64) int { return (int(float64(x)*scale) + 1) & ^1 }
+
+func ToYCbCr(bytes []byte, w, h int) *image.YCbCr {
+	cw, ch := (w+1)/2, (h+1)/2
+
+	i0 := w*h + 0*cw*ch
+	i1 := w*h + 1*cw*ch
+	i2 := w*h + 2*cw*ch
+
+	yuv := image.NewYCbCr(image.Rect(0, 0, w, h), image.YCbCrSubsampleRatio420)
+	yuv.Y = bytes[:i0:i0]
+	yuv.Cb = bytes[i0:i1:i1]
+	yuv.Cr = bytes[i1:i2:i2]
+	return yuv
+}

pkg/worker/caged/libretro/caged.go

@@ -50,8 +50,7 @@ func (c *Caged) Load(game games.GameMetadata, path string) error {
 		return err
 	}
 	w, h := c.ViewportCalc()
-	c.SetViewport(w, h, c.conf.Emulator.Scale)
-
+	c.SetViewport(w, h)
 	return nil
 }
 
@@ -79,6 +78,7 @@ func (c *Caged) PixFormat() uint32                 { return c.Emulator.PixFormat
 func (c *Caged) Rotation() uint                    { return c.Emulator.Rotation() }
 func (c *Caged) AudioSampleRate() int              { return c.Emulator.AudioSampleRate() }
 func (c *Caged) ViewportSize() (int, int)          { return c.Emulator.ViewportSize() }
+func (c *Caged) Scale() float64                    { return c.Emulator.Scale() }
 func (c *Caged) SendControl(port int, data []byte) { c.base.Input(port, data) }
 func (c *Caged) Start()                            { go c.Emulator.Start() }
 func (c *Caged) SetSaveOnClose(v bool)             { c.base.SaveOnClose = v }