diff --git a/.storybook/stories/FaceControls.stories.tsx b/.storybook/stories/FaceControls.stories.tsx index 0037aa5f1..9855aa21e 100644 --- a/.storybook/stories/FaceControls.stories.tsx +++ b/.storybook/stories/FaceControls.stories.tsx @@ -1,11 +1,15 @@ /* eslint react-hooks/exhaustive-deps: 1 */ +import * as THREE from 'three' import * as React from 'react' import { Meta, StoryObj } from '@storybook/react' +import * as easing from 'maath/easing' import { Setup } from '../Setup' -import { FaceLandmarker, FaceControls, Box } from '../../src' -import { ComponentProps } from 'react' +import { FaceLandmarker, FaceControls, Box, WebcamVideoTexture } from '../../src' +import { ComponentProps, ElementRef, useRef, useState } from 'react' +import { FaceLandmarkerResult } from '@mediapipe/tasks-vision' +import { useFrame, useThree } from '@react-three/fiber' export default { title: 'Controls/FaceControls', @@ -17,10 +21,13 @@ export default { ), ], + tags: ['!autodocs'], // FaceLandmarker cannot have multiple instances } satisfies Meta type Story = StoryObj +// + function FaceControlsScene(props: ComponentProps) { return ( <> @@ -44,3 +51,101 @@ export const FaceControlsSt = { render: (args) => , name: 'Default', } satisfies Story + +// + +function FaceControlsScene2(props: ComponentProps) { + const faceLandmarkerRef = useRef>(null) + const videoTextureRef = useRef>(null) + + const [faceLandmarkerResult, setFaceLandmarkerResult] = useState() + + return ( + <> + + + + + + { + const faceLandmarker = faceLandmarkerRef.current + const videoTexture = videoTextureRef.current + if (!faceLandmarker || !videoTexture) return + + const videoFrame = videoTexture.source.data + const result = faceLandmarker.detectForVideo(videoFrame, now) + setFaceLandmarkerResult(result) + }} + /> + + + + + + + + + + ) +} + +export const FaceControlsSt2 = { + render: (args) => , + name: 'manualDetect', +} satisfies Story + +// + +function FaceControlsScene3(props: ComponentProps) { + const faceControlsRef = useRef>(null) + + const camera = useThree((state) => state.camera) + const [current] = useState(() => new THREE.Object3D()) + + useFrame((_, delta) => { + const target = faceControlsRef.current?.computeTarget() + + if (target) { + // + // A. Define your own damping + // + + const eps = 1e-9 + easing.damp3(current.position, target.position, 0.25, delta, undefined, undefined, eps) + easing.dampE(current.rotation, target.rotation, 0.25, delta, undefined, undefined, eps) + camera.position.copy(current.position) + camera.rotation.copy(current.rotation) + + // + // B. Or maybe with no damping at all? + // + + // camera.position.copy(target.position) + // camera.rotation.copy(target.rotation) + } + }) + + return ( + <> + + + + + + + + + + + + + + ) +} + +export const FaceControlsSt3 = { + render: (args) => , + name: 'manualUpdate', +} satisfies Story diff --git a/docs/controls/face-controls.mdx b/docs/controls/face-controls.mdx index 608f1d2c0..74065dd97 100644 --- a/docs/controls/face-controls.mdx +++ b/docs/controls/face-controls.mdx @@ -5,7 +5,9 @@ sourcecode: src/web/FaceControls.tsx [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-facecontrols) -The camera follows your face. + +The camera follows your (detected) face. +
  • @@ -27,135 +29,68 @@ Prerequisite: wrap into a [`FaceLandmarker`](https://drei.docs.pmnd.rs/misc/face ``` ```tsx -type FaceControlsProps = { - /** The camera to be controlled, default: global state camera */ +export type FaceControlsProps = { + /** The camera to be controlled */ camera?: THREE.Camera - /** Whether to autostart the webcam, default: true */ - autostart?: boolean - /** Enable/disable the webcam, default: true */ - webcam?: boolean - /** A custom video URL or mediaStream, default: undefined */ - webcamVideoTextureSrc?: VideoTextureSrc - /** Disable the rAF camera position/rotation update, default: false */ - manualUpdate?: boolean - /** Disable the rVFC face-detection, default: false */ + /** VideoTexture or WebcamVideoTexture options */ + videoTexture: VideoTextureProps + /** Disable the automatic face-detection => you should provide `faceLandmarkerResult` yourself in this case */ manualDetect?: boolean - /** Callback function to call on "videoFrame" event, default: undefined */ - onVideoFrame?: (e: THREE.Event) => void + /** FaceLandmarker result */ + faceLandmarkerResult?: FaceLandmarkerResult + /** Disable the rAF camera position/rotation update */ + manualUpdate?: boolean /** Reference this FaceControls instance as state's `controls` */ makeDefault?: boolean /** Approximate time to reach the target. A smaller value will reach the target faster. */ smoothTime?: number /** Apply position offset extracted from `facialTransformationMatrix` */ offset?: boolean - /** Offset sensitivity factor, less is more sensible, default: 80 */ + /** Offset sensitivity factor, less is more sensible */ offsetScalar?: number /** Enable eye-tracking */ eyes?: boolean - /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */ + /** Force Facemesh's `origin` to be the middle of the 2 eyes */ eyesAsOrigin?: boolean - /** Constant depth of the Facemesh, default: .15 */ + /** Constant depth of the Facemesh */ depth?: number - /** Enable debug mode, default: false */ + /** Enable debug mode */ debug?: boolean /** Facemesh options, default: undefined */ facemesh?: FacemeshProps } -``` -```tsx -type FaceControlsApi = THREE.EventDispatcher & { - /** Detect faces from the video */ - detect: (video: HTMLVideoElement, time: number) => FaceLandmarkerResult | undefined +export type FaceControlsApi = THREE.EventDispatcher & { /** Compute the target for the camera */ computeTarget: () => THREE.Object3D /** Update camera's position/rotation to the `target` */ update: (delta: number, target?: THREE.Object3D) => void /** ref api */ facemeshApiRef: RefObject - /** ref api */ - webcamApiRef: RefObject - /** Play the video */ - play: () => void - /** Pause the video */ - pause: () => void } ``` -## FaceControls events - -Two `THREE.Event`s are dispatched on `FaceControls` ref object: +## Breaking changes -- `{ type: "stream", stream: MediaStream }` -- when webcam's [`.getUserMedia()`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) promise is resolved -- `{ type: "videoFrame", texture: THREE.VideoTexture, time: number }` -- each time a new video frame is sent to the compositor (thanks to rVFC) +### 9.120.0 -> [!Note] -> rVFC -> -> Internally, `FaceControls` uses [`requestVideoFrameCallback`](https://caniuse.com/mdn-api_htmlvideoelement_requestvideoframecallback), you may need [a polyfill](https://github.com/ThaUnknown/rvfc-polyfill) (for Firefox). +
    -## FaceControls[manualDetect] +`FaceControls` was [simplified](https://github.com/pmndrs/drei/pull/2242). -By default, `detect` is called on each `"videoFrame"`. You can disable this by `manualDetect` and call `detect` yourself. +Following props were deleted: -For example: +- `autostart`: now use `videoTexture.start` +- `webcam`: instead of `webcam: false`, you can now [`manualDetect`](http://localhost:6006/?path=/story/controls-facecontrols--face-controls-st-2) +- `webcamVideoTextureSrc`: now use `videoTexture.src` (or instantiate your own video-texture[^1] outside) +- `onVideoFrame`: now use `videoTexture.onVideoFrame` (or instantiate your own video-texture[^1] outside) -```jsx -const controls = useThree((state) => state.controls) +Following api methods/fields were deleted: -const onVideoFrame = useCallback((event) => { - controls.detect(event.texture.source.data, event.time) -}, [controls]) +- `detect`: you can now [`manualDetect`](http://localhost:6006/?path=/story/controls-facecontrols--face-controls-st-2) outside and pass `faceLandmarkerResult` +- `webcamApiRef`: if you need `videoTextureRef`, instantiate your own video-texture[^1] outside +- `play`/`pause`: same, if you need the `video` object, instantiate your own video-texture[^1] outside - -``` +[^1]: `` or `` -## FaceControls[manualUpdate] - -By default, `update` method is called each rAF `useFrame`. You can disable this by `manualUpdate` and call it yourself: - -```jsx -const controls = useThree((state) => state.controls) - -useFrame((_, delta) => { - controls.update(delta) // 60 or 120 FPS with default damping -}) - - -``` - -Or, if you want your own custom damping, use `computeTarget` method and update the camera pos/rot yourself with: - -```jsx -import * as easing from 'maath/easing' - -const camera = useThree((state) => state.camera) - -const [current] = useState(() => new THREE.Object3D()) - -useFrame((_, delta) => { - const target = controls?.computeTarget() - - if (target) { - // - // A. Define your own damping - // - - const eps = 1e-9 - easing.damp3(current.position, target.position, 0.25, delta, undefined, undefined, eps) - easing.dampE(current.rotation, target.rotation, 0.25, delta, undefined, undefined, eps) - camera.position.copy(current.position) - camera.rotation.copy(current.rotation) - - // - // B. Or maybe with no damping at all? - // - - // camera.position.copy(target.position) - // camera.rotation.copy(target.rotation) - } -}) -``` +
    \ No newline at end of file diff --git a/docs/misc/face-landmarker.mdx b/docs/misc/face-landmarker.mdx index e060081e9..6278b4067 100644 --- a/docs/misc/face-landmarker.mdx +++ b/docs/misc/face-landmarker.mdx @@ -46,3 +46,21 @@ faceLandmarkerOptions.baseOptions.modelAssetPath = modelAssetPath; ``` + +## instance + +You can get the FaceLandmarker instance through `ref`: + +```tsx +const faceLandmarkerRef = useRef>(null) + + + {/* ... */} + +``` + +or using `useFaceLandmarker()` from a descendant component: + +```jsx +const faceLandmarker = useFaceLandmarker() +``` \ No newline at end of file diff --git a/src/web/FaceControls.tsx b/src/web/FaceControls.tsx index 994779df5..723a4d3d7 100644 --- a/src/web/FaceControls.tsx +++ b/src/web/FaceControls.tsx @@ -13,17 +13,17 @@ import { RefObject, createContext, useContext, + ElementRef, } from 'react' import { useFrame, useThree } from '@react-three/fiber' -import type { FaceLandmarkerResult } from '@mediapipe/tasks-vision' +import type { FaceLandmarker, FaceLandmarkerResult } from '@mediapipe/tasks-vision' import { easing } from 'maath' -import { suspend, clear } from 'suspend-react' -import { useVideoTexture } from '../core/VideoTexture' +import { VideoTexture, VideoTextureProps, WebcamVideoTexture } from '..' import { Facemesh, FacemeshApi, FacemeshProps } from './Facemesh' import { useFaceLandmarker } from './FaceLandmarker' -type VideoTextureSrc = Parameters[0] // useVideoTexture 1st arg `src` type +type VideoFrame = Parameters[0] function mean(v1: THREE.Vector3, v2: THREE.Vector3) { return v1.clone().add(v2).multiplyScalar(0.5) @@ -40,55 +40,43 @@ function localToLocal(objSrc: THREE.Object3D, v: THREE.Vector3, objDst: THREE.Ob // export type FaceControlsProps = { - /** The camera to be controlled, default: global state camera */ + /** The camera to be controlled */ camera?: THREE.Camera - /** Whether to autostart the webcam, default: true */ - autostart?: boolean - /** Enable/disable the webcam, default: true */ - webcam?: boolean - /** A custom video URL or mediaStream, default: undefined */ - webcamVideoTextureSrc?: VideoTextureSrc - /** Disable the rAF camera position/rotation update, default: false */ - manualUpdate?: boolean - /** Disable the rVFC face-detection, default: false */ + /** VideoTexture or WebcamVideoTexture options */ + videoTexture: VideoTextureProps + /** Disable the automatic face-detection => you should provide `faceLandmarkerResult` yourself in this case */ manualDetect?: boolean - /** Callback function to call on "videoFrame" event, default: undefined */ - onVideoFrame?: (e: THREE.Event) => void + /** FaceLandmarker result */ + faceLandmarkerResult?: FaceLandmarkerResult + /** Disable the rAF camera position/rotation update */ + manualUpdate?: boolean /** Reference this FaceControls instance as state's `controls` */ makeDefault?: boolean /** Approximate time to reach the target. A smaller value will reach the target faster. */ smoothTime?: number /** Apply position offset extracted from `facialTransformationMatrix` */ offset?: boolean - /** Offset sensitivity factor, less is more sensible, default: 80 */ + /** Offset sensitivity factor, less is more sensible */ offsetScalar?: number /** Enable eye-tracking */ eyes?: boolean - /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */ + /** Force Facemesh's `origin` to be the middle of the 2 eyes */ eyesAsOrigin?: boolean - /** Constant depth of the Facemesh, default: .15 */ + /** Constant depth of the Facemesh */ depth?: number - /** Enable debug mode, default: false */ + /** Enable debug mode */ debug?: boolean /** Facemesh options, default: undefined */ facemesh?: FacemeshProps } export type FaceControlsApi = THREE.EventDispatcher & { - /** Detect faces from the video */ - detect: (video: HTMLVideoElement, time: number) => FaceLandmarkerResult | undefined /** Compute the target for the camera */ computeTarget: () => THREE.Object3D /** Update camera's position/rotation to the `target` */ update: (delta: number, target?: THREE.Object3D) => void /** ref api */ facemeshApiRef: RefObject - /** ref api */ - webcamApiRef: RefObject - /** Play the video */ - play: () => void - /** Pause the video */ - pause: () => void } const FaceControlsContext = /* @__PURE__ */ createContext({} as FaceControlsApi) @@ -107,12 +95,11 @@ export const FaceControls = /* @__PURE__ */ forwardRef { @@ -131,8 +117,6 @@ export const FaceControls = /* @__PURE__ */ forwardRef state.get) const explCamera = camera || defaultCamera - const webcamApiRef = useRef(null) - const facemeshApiRef = useRef(null) // @@ -223,67 +207,49 @@ export const FaceControls = /* @__PURE__ */ forwardRef { + if (manualUpdate) return + update(delta) + }) + // - // detect() + // onVideoFrame (only used if !manualDetect) // - const [faces, setFaces] = useState() - const faceLandmarker = useFaceLandmarker() - const detect = useCallback( - (video, time) => { - const result = faceLandmarker?.detectForVideo(video, time) - setFaces(result) + const videoTextureRef = useRef>(null) - return result + const [_faceLandmarkerResult, setFaceLandmarkerResult] = useState() + const faceLandmarker = useFaceLandmarker() + const onVideoFrame = useCallback>( + (now, metadata) => { + const texture = videoTextureRef.current + if (!texture) return + const videoFrame = texture.source.data as VideoFrame + const result = faceLandmarker?.detectForVideo(videoFrame, now) + setFaceLandmarkerResult(result) }, [faceLandmarker] ) - useFrame((_, delta) => { - if (!manualUpdate) { - update(delta) - } - }) - + // // Ref API + // + const api = useMemo( () => Object.assign(Object.create(THREE.EventDispatcher.prototype), { - detect, computeTarget, update, facemeshApiRef, - webcamApiRef, - // shorthands - play: () => { - webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.play() - }, - pause: () => { - webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.pause() - }, }), - [detect, computeTarget, update] + [computeTarget, update] ) useImperativeHandle(fref, () => api, [api]) // - // events callbacks + // makeDefault (`controls` global state) // - useEffect(() => { - const onVideoFrameCb = (e: THREE.Event) => { - if (!manualDetect) detect(e.texture.source.data, e.time) - if (onVideoFrame) onVideoFrame(e) - } - - api.addEventListener('videoFrame', onVideoFrameCb) - - return () => { - api.removeEventListener('videoFrame', onVideoFrameCb) - } - }, [api, detect, faceLandmarker, manualDetect, onVideoFrame]) - - // `controls` global state useEffect(() => { if (makeDefault) { const old = get().controls @@ -292,14 +258,27 @@ export const FaceControls = /* @__PURE__ */ forwardRef - {webcam && ( + {!manualDetect && ( - + {'src' in videoTextureProps ? ( + + ) : ( + + )} )} @@ -326,103 +305,3 @@ export const FaceControls = /* @__PURE__ */ forwardRef useContext(FaceControlsContext) - -// -// Webcam -// - -type WebcamApi = { - videoTextureApiRef: RefObject -} - -type WebcamProps = { - videoTextureSrc?: VideoTextureSrc - autostart?: boolean -} - -const Webcam = /* @__PURE__ */ forwardRef(({ videoTextureSrc, autostart = true }, fref) => { - const videoTextureApiRef = useRef(null) - - const faceControls = useFaceControls() - - const stream: MediaStream | null = suspend(async () => { - return !videoTextureSrc - ? await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { facingMode: 'user' }, - }) - : Promise.resolve(null) - }, [videoTextureSrc]) - - useEffect(() => { - faceControls.dispatchEvent({ type: 'stream', stream }) - - return () => { - stream?.getTracks().forEach((track) => track.stop()) - clear([videoTextureSrc]) - } - }, [stream, faceControls, videoTextureSrc]) - - // ref-api - const api = useMemo( - () => ({ - videoTextureApiRef, - }), - [] - ) - useImperativeHandle(fref, () => api, [api]) - - return ( - - - - ) -}) - -// -// VideoTexture -// - -type VideoTextureApi = { texture: THREE.VideoTexture } -type VideoTextureProps = { src: VideoTextureSrc; start: boolean } - -const VideoTexture = /* @__PURE__ */ forwardRef(({ src, start }, fref) => { - const texture = useVideoTexture(src, { start }) - const video = texture.source.data - - const faceControls = useFaceControls() - const onVideoFrame = useCallback( - (time: number) => { - faceControls.dispatchEvent({ type: 'videoFrame', texture, time }) - }, - [texture, faceControls] - ) - useVideoFrame(video, onVideoFrame) - - // ref-api - const api = useMemo( - () => ({ - texture, - }), - [texture] - ) - useImperativeHandle(fref, () => api, [api]) - - return <> -}) - -const useVideoFrame = (video: HTMLVideoElement, f: (...args: any) => any) => { - // https://web.dev/requestvideoframecallback-rvfc/ - // https://www.remotion.dev/docs/video-manipulation - useEffect(() => { - if (!video || !video.requestVideoFrameCallback) return - let handle: number - function callback(...args: any) { - f(...args) - handle = video.requestVideoFrameCallback(callback) - } - video.requestVideoFrameCallback(callback) - - return () => video.cancelVideoFrameCallback(handle) - }, [video, f]) -} diff --git a/src/web/FaceLandmarker.tsx b/src/web/FaceLandmarker.tsx index 7cb1b8906..68c29f5bd 100644 --- a/src/web/FaceLandmarker.tsx +++ b/src/web/FaceLandmarker.tsx @@ -1,6 +1,6 @@ /* eslint react-hooks/exhaustive-deps: 1 */ import * as React from 'react' -import { createContext, ReactNode, useContext, useEffect } from 'react' +import { createContext, forwardRef, ReactNode, useContext, useEffect, useImperativeHandle } from 'react' import type { FaceLandmarker as FaceLandmarkerImpl, FaceLandmarkerOptions } from '@mediapipe/tasks-vision' import { clear, suspend } from 'suspend-react' @@ -26,28 +26,28 @@ export const FaceLandmarkerDefaults = { } as FaceLandmarkerOptions, } -export function FaceLandmarker({ - basePath = FaceLandmarkerDefaults.basePath, - options = FaceLandmarkerDefaults.options, - children, -}: FaceLandmarkerProps) { - const opts = JSON.stringify(options) - - const faceLandmarker = suspend(async () => { - const { FilesetResolver, FaceLandmarker } = await import('@mediapipe/tasks-vision') - const vision = await FilesetResolver.forVisionTasks(basePath) - return FaceLandmarker.createFromOptions(vision, options) - }, [basePath, opts]) - - useEffect(() => { - return () => { - faceLandmarker?.close() - clear([basePath, opts]) - } - }, [faceLandmarker, basePath, opts]) - - return {children} -} +export const FaceLandmarker = forwardRef( + ({ basePath = FaceLandmarkerDefaults.basePath, options = FaceLandmarkerDefaults.options, children }, fref) => { + const opts = JSON.stringify(options) + + const faceLandmarker = suspend(async () => { + const { FilesetResolver, FaceLandmarker } = await import('@mediapipe/tasks-vision') + const vision = await FilesetResolver.forVisionTasks(basePath) + return FaceLandmarker.createFromOptions(vision, options) + }, [basePath, opts]) + + useEffect(() => { + return () => { + faceLandmarker?.close() + clear([basePath, opts]) + } + }, [faceLandmarker, basePath, opts]) + + useImperativeHandle(fref, () => faceLandmarker, [faceLandmarker]) // expose faceLandmarker through ref + + return {children} + } +) export function useFaceLandmarker() { return useContext(FaceLandmarkerContext)