Skip to content

Commit

Permalink
feat(VideoTexture): onVideoFrame (#2238)
Browse files Browse the repository at this point in the history
  • Loading branch information
abernier authored Nov 29, 2024
1 parent 0534eb8 commit 56b7182
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 15 deletions.
37 changes: 36 additions & 1 deletion docs/loaders/video-texture-use-video-texture.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,18 @@ export function useVideoTexture(
{
unsuspend = 'loadedmetadata',
start = true,
hls: hlsConfig = {},
hls = {},
crossOrigin = 'anonymous',
muted = true,
loop = true,
playsInline = true,
onVideoFrame,
...videoProps
}: {
unsuspend?: keyof HTMLVideoElementEventMap
start?: boolean
hls?: Parameters<typeof getHls>[0]
onVideoFrame: VideoFrameRequestCallback
} & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
)
```
Expand Down Expand Up @@ -84,3 +86,36 @@ const texture = useVideoTexture('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3
hls: { abrEwmaFastLive: 1.0, abrEwmaSlowLive: 3.0, enableWorker: true },
})
```

## `requestVideoFrameCallback` (rVFC)

`useVideoTexture` supports [`requestVideoFrameCallback`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement/requestVideoFrameCallback):

```jsx
useVideoTexture(src, {
onVideoFrame: (now, metadata) => {}
})
```

## `<VideoTexture>` Component

```tsx
export type VideoTextureProps = {
children?: (texture: THREE.VideoTexture) => React.ReactNode
src: UseVideoTextureParams[0]
} & UseVideoTextureParams[1]
```

You can access the texture via children's render prop:

```jsx
<VideoTexture src="/video.mp4">
{(texture) => <meshBasicMaterial map={texture} />}
```

or exposed via `ref`:

```jsx
const textureRef = useRef()
<VideoTexture ref={textureRef} src="/video.mp4" />
```
66 changes: 52 additions & 14 deletions src/core/VideoTexture.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint react-hooks/exhaustive-deps: 1 */
import * as React from 'react'
import * as THREE from 'three'
import { useEffect, useRef } from 'react'
import { forwardRef, useEffect, useImperativeHandle, useRef } from 'react'
import { useThree } from '@react-three/fiber'
import { suspend } from 'suspend-react'
import { type default as Hls, Events } from 'hls.js'
Expand Down Expand Up @@ -31,11 +32,22 @@ export function useVideoTexture(
muted = true,
loop = true,
playsInline = true,
onVideoFrame,
...videoProps
}: {
/** Event name that will unsuspend the video */
unsuspend?: keyof HTMLVideoElementEventMap
/** Auto start the video once unsuspended */
start?: boolean
/** HLS config */
hls?: Parameters<typeof getHls>[0]
/**
* request Video Frame Callback (rVFC)
*
* @see https://web.dev/requestvideoframecallback-rvfc/
* @see https://www.remotion.dev/docs/video-manipulation
* */
onVideoFrame?: VideoFrameRequestCallback
} & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
) {
const gl = useThree((state) => state.gl)
Expand Down Expand Up @@ -81,6 +93,9 @@ export function useVideoTexture(
[srcOrSrcObject]
)

const video = texture.source.data as HTMLVideoElement
useVideoFrame(video, onVideoFrame)

useEffect(() => {
start && texture.image.play()

Expand All @@ -96,22 +111,45 @@ export function useVideoTexture(
}

//
// VideoTexture
//

type UseVideoTextureParams = Parameters<typeof useVideoTexture>
type VideoTexture = ReturnType<typeof useVideoTexture>

export type VideoTextureProps = {
children?: (texture: VideoTexture) => React.ReactNode
src: UseVideoTextureParams[0]
} & UseVideoTextureParams[1]

export const VideoTexture = /* @__PURE__ */ forwardRef<VideoTexture, VideoTextureProps>(
({ children, src, ...config }, fref) => {
const texture = useVideoTexture(src, config)

type UseVideoTexture = Parameters<typeof useVideoTexture>
useEffect(() => {
return () => void texture.dispose()
}, [texture])

export const VideoTexture = ({
children,
src,
...config
}: {
children?: (texture: ReturnType<typeof useVideoTexture>) => React.ReactNode
src: UseVideoTexture[0]
} & UseVideoTexture[1]) => {
const ret = useVideoTexture(src, config)
useImperativeHandle(fref, () => texture, [texture]) // expose texture through ref

return <>{children?.(texture)}</>
}
)

// rVFC hook

const useVideoFrame = (video: HTMLVideoElement, f?: VideoFrameRequestCallback) => {
useEffect(() => {
return () => void ret.dispose()
}, [ret])
if (!f) return
if (!video.requestVideoFrameCallback) return

let handle: ReturnType<(typeof video)['requestVideoFrameCallback']>
const callback: VideoFrameRequestCallback = (...args) => {
f(...args)
handle = video.requestVideoFrameCallback(callback)
}
video.requestVideoFrameCallback(callback)

return <>{children?.(ret)}</>
return () => video.cancelVideoFrameCallback(handle)
}, [video, f])
}

0 comments on commit 56b7182

Please sign in to comment.