From 6a7a0e8d610fd1af6e8c9a04d3cd5188e7cc4210 Mon Sep 17 00:00:00 2001
From: Antoine BERNIER <antoine.bernier@gmail.com>
Date: Thu, 28 Nov 2024 20:35:27 +0100
Subject: [PATCH] onVideoFrame

---
 .../video-texture-use-video-texture.mdx       | 16 +++++-
 src/core/VideoTexture.tsx                     | 54 +++++++++++++++----
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/docs/loaders/video-texture-use-video-texture.mdx b/docs/loaders/video-texture-use-video-texture.mdx
index 8198c9048..82850f5d0 100644
--- a/docs/loaders/video-texture-use-video-texture.mdx
+++ b/docs/loaders/video-texture-use-video-texture.mdx
@@ -24,16 +24,18 @@ export function useVideoTexture(
   {
     unsuspend = 'loadedmetadata',
     start = true,
-    hls: hlsConfig = {},
+    hls = {},
     crossOrigin = 'anonymous',
     muted = true,
     loop = true,
     playsInline = true,
+    onVideoFrame = () => {},
     ...videoProps
   }: {
     unsuspend?: keyof HTMLVideoElementEventMap
     start?: boolean
     hls?: Parameters<typeof getHls>[0]
+    onVideoFrame: (texture: VideoTexture, ...args: Parameters<VideoFrameRequestCallback>) => void
   } & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
 )
 ```
@@ -84,3 +86,15 @@ const texture = useVideoTexture('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3
   hls: { abrEwmaFastLive: 1.0, abrEwmaSlowLive: 3.0, enableWorker: true },
 })
 ```
+
+## `requestVideoFrameCallback` (rVFC)
+
+`useVideoTexture` supports [`requestVideoFrameCallback`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement/requestVideoFrameCallback):
+
+```jsx
+useVideoTexture(src, {
+  onVideoFrame: (texture, now, metadata) => {
+    // const videoFrame = texture.source.data
+  }
+})
+```
\ No newline at end of file
diff --git a/src/core/VideoTexture.tsx b/src/core/VideoTexture.tsx
index a50ed4c8a..17e3c26a2 100644
--- a/src/core/VideoTexture.tsx
+++ b/src/core/VideoTexture.tsx
@@ -1,3 +1,4 @@
+/* eslint react-hooks/exhaustive-deps: 1 */
 import * as React from 'react'
 import * as THREE from 'three'
 import { useEffect, useRef } from 'react'
@@ -31,11 +32,22 @@ export function useVideoTexture(
     muted = true,
     loop = true,
     playsInline = true,
+    onVideoFrame = () => {},
     ...videoProps
   }: {
+    /** Event name that will unsuspend the video */
     unsuspend?: keyof HTMLVideoElementEventMap
+    /** Auto start the video once unsuspended */
     start?: boolean
+    /** HLS config */
     hls?: Parameters<typeof getHls>[0]
+    /**
+     * request Video Frame Callback (rVFC)
+     *
+     * @see https://web.dev/requestvideoframecallback-rvfc/
+     * @see https://www.remotion.dev/docs/video-manipulation
+     * */
+    onVideoFrame?: (texture: VideoTexture, ...args: Parameters<VideoFrameRequestCallback>) => void
   } & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
 ) {
   const gl = useThree((state) => state.gl)
@@ -81,6 +93,9 @@ export function useVideoTexture(
     [srcOrSrcObject]
   )
 
+  const video = texture.source.data as HTMLVideoElement
+  useVideoFrame(video, (...args) => onVideoFrame(texture, ...args))
+
   useEffect(() => {
     start && texture.image.play()
 
@@ -95,23 +110,40 @@ export function useVideoTexture(
   return texture
 }
 
+//
+// VideoTexture
 //
 
 type UseVideoTexture = Parameters<typeof useVideoTexture>
+type VideoTexture = ReturnType<typeof useVideoTexture>
 
-export const VideoTexture = ({
-  children,
-  src,
-  ...config
-}: {
-  children?: (texture: ReturnType<typeof useVideoTexture>) => React.ReactNode
+export type VideoTextureProps = {
+  children?: (texture: VideoTexture) => React.ReactNode
   src: UseVideoTexture[0]
-} & UseVideoTexture[1]) => {
-  const ret = useVideoTexture(src, config)
+} & UseVideoTexture[1]
+
+export const VideoTexture = /* @__PURE__ */ ({ children, src, ...config }: VideoTextureProps) => {
+  const texture = useVideoTexture(src, config)
 
   useEffect(() => {
-    return () => void ret.dispose()
-  }, [ret])
+    return () => void texture.dispose()
+  }, [texture])
+
+  return <>{children?.(texture)}</>
+}
+
+// rVFC hook
+
+const useVideoFrame = (video: HTMLVideoElement, f: (...args: Parameters<VideoFrameRequestCallback>) => void) => {
+  useEffect(() => {
+    if (!video || !video.requestVideoFrameCallback) return
+    let handle: ReturnType<(typeof video)['requestVideoFrameCallback']>
+    const callback: VideoFrameRequestCallback = (...args) => {
+      f(...args)
+      handle = video.requestVideoFrameCallback(callback)
+    }
+    video.requestVideoFrameCallback(callback)
 
-  return <>{children?.(ret)}</>
+    return () => video.cancelVideoFrameCallback(handle)
+  }, [video, f])
 }