Skip to content

Commit

Permalink
feat: FaceControls simplified (#2242)
Browse files Browse the repository at this point in the history 
* simplified

* migration path
  • Loading branch information
@abernier
abernier authored Dec 5, 2024
1 parent f759344 commit 75260b8
Show file tree
Hide file tree
Showing 5 changed files with 239 additions and 302 deletions.
109 changes: 107 additions & 2 deletions .storybook/stories/FaceControls.stories.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
/* eslint react-hooks/exhaustive-deps: 1 */
import * as THREE from 'three'
import * as React from 'react'
import { Meta, StoryObj } from '@storybook/react'
import * as easing from 'maath/easing'

import { Setup } from '../Setup'

import { FaceLandmarker, FaceControls, Box } from '../../src'
import { ComponentProps } from 'react'
import { FaceLandmarker, FaceControls, Box, WebcamVideoTexture } from '../../src'
import { ComponentProps, ElementRef, useRef, useState } from 'react'
import { FaceLandmarkerResult } from '@mediapipe/tasks-vision'
import { useFrame, useThree } from '@react-three/fiber'

export default {
title: 'Controls/FaceControls',
Expand All @@ -17,10 +21,13 @@ export default {
</Setup>
),
],
tags: ['!autodocs'], // FaceLandmarker cannot have multiple instances
} satisfies Meta<typeof FaceControls>

type Story = StoryObj<typeof FaceControls>

//

function FaceControlsScene(props: ComponentProps<typeof FaceControls>) {
return (
<>
Expand All @@ -44,3 +51,101 @@ export const FaceControlsSt = {
render: (args) => <FaceControlsScene {...args} />,
name: 'Default',
} satisfies Story

//

function FaceControlsScene2(props: ComponentProps<typeof FaceControls>) {
const faceLandmarkerRef = useRef<ElementRef<typeof FaceLandmarker>>(null)
const videoTextureRef = useRef<ElementRef<typeof WebcamVideoTexture>>(null)

const [faceLandmarkerResult, setFaceLandmarkerResult] = useState<FaceLandmarkerResult>()

return (
<>
<color attach="background" args={['#303030']} />
<axesHelper />

<React.Suspense fallback={null}>
<FaceLandmarker ref={faceLandmarkerRef}>
<WebcamVideoTexture
ref={videoTextureRef}
onVideoFrame={(now) => {
const faceLandmarker = faceLandmarkerRef.current
const videoTexture = videoTextureRef.current
if (!faceLandmarker || !videoTexture) return

const videoFrame = videoTexture.source.data
const result = faceLandmarker.detectForVideo(videoFrame, now)
setFaceLandmarkerResult(result)
}}
/>

<FaceControls {...props} manualDetect faceLandmarkerResult={faceLandmarkerResult} />
</FaceLandmarker>
</React.Suspense>

<Box args={[0.1, 0.1, 0.1]}>
<meshStandardMaterial />
</Box>
</>
)
}

export const FaceControlsSt2 = {
render: (args) => <FaceControlsScene2 {...args} />,
name: 'manualDetect',
} satisfies Story

//

function FaceControlsScene3(props: ComponentProps<typeof FaceControls>) {
const faceControlsRef = useRef<ElementRef<typeof FaceControls>>(null)

const camera = useThree((state) => state.camera)
const [current] = useState(() => new THREE.Object3D())

useFrame((_, delta) => {
const target = faceControlsRef.current?.computeTarget()

if (target) {
//
// A. Define your own damping
//

const eps = 1e-9
easing.damp3(current.position, target.position, 0.25, delta, undefined, undefined, eps)
easing.dampE(current.rotation, target.rotation, 0.25, delta, undefined, undefined, eps)
camera.position.copy(current.position)
camera.rotation.copy(current.rotation)

//
// B. Or maybe with no damping at all?
//

// camera.position.copy(target.position)
// camera.rotation.copy(target.rotation)
}
})

return (
<>
<color attach="background" args={['#303030']} />
<axesHelper />

<React.Suspense fallback={null}>
<FaceLandmarker>
<FaceControls ref={faceControlsRef} {...props} manualUpdate />
</FaceLandmarker>
</React.Suspense>

<Box args={[0.1, 0.1, 0.1]}>
<meshStandardMaterial />
</Box>
</>
)
}

export const FaceControlsSt3 = {
render: (args) => <FaceControlsScene3 {...args} />,
name: 'manualUpdate',
} satisfies Story
129 changes: 32 additions & 97 deletions docs/controls/face-controls.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ sourcecode: src/web/FaceControls.tsx

[![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-facecontrols)

The camera follows your face.
<Intro>
The camera follows your (detected) face.
</Intro>

<Grid cols={4}>
<li>
Expand All @@ -27,135 +29,68 @@ Prerequisite: wrap into a [`FaceLandmarker`](https://drei.docs.pmnd.rs/misc/face
```

```tsx
type FaceControlsProps = {
/** The camera to be controlled, default: global state camera */
export type FaceControlsProps = {
/** The camera to be controlled */
camera?: THREE.Camera
/** Whether to autostart the webcam, default: true */
autostart?: boolean
/** Enable/disable the webcam, default: true */
webcam?: boolean
/** A custom video URL or mediaStream, default: undefined */
webcamVideoTextureSrc?: VideoTextureSrc
/** Disable the rAF camera position/rotation update, default: false */
manualUpdate?: boolean
/** Disable the rVFC face-detection, default: false */
/** VideoTexture or WebcamVideoTexture options */
videoTexture: VideoTextureProps
/** Disable the automatic face-detection => you should provide `faceLandmarkerResult` yourself in this case */
manualDetect?: boolean
/** Callback function to call on "videoFrame" event, default: undefined */
onVideoFrame?: (e: THREE.Event) => void
/** FaceLandmarker result */
faceLandmarkerResult?: FaceLandmarkerResult
/** Disable the rAF camera position/rotation update */
manualUpdate?: boolean
/** Reference this FaceControls instance as state's `controls` */
makeDefault?: boolean
/** Approximate time to reach the target. A smaller value will reach the target faster. */
smoothTime?: number
/** Apply position offset extracted from `facialTransformationMatrix` */
offset?: boolean
/** Offset sensitivity factor, less is more sensible, default: 80 */
/** Offset sensitivity factor, less is more sensible */
offsetScalar?: number
/** Enable eye-tracking */
eyes?: boolean
/** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */
/** Force Facemesh's `origin` to be the middle of the 2 eyes */
eyesAsOrigin?: boolean
/** Constant depth of the Facemesh, default: .15 */
/** Constant depth of the Facemesh */
depth?: number
/** Enable debug mode, default: false */
/** Enable debug mode */
debug?: boolean
/** Facemesh options, default: undefined */
facemesh?: FacemeshProps
}
```

```tsx
type FaceControlsApi = THREE.EventDispatcher & {
/** Detect faces from the video */
detect: (video: HTMLVideoElement, time: number) => FaceLandmarkerResult | undefined
export type FaceControlsApi = THREE.EventDispatcher & {
/** Compute the target for the camera */
computeTarget: () => THREE.Object3D
/** Update camera's position/rotation to the `target` */
update: (delta: number, target?: THREE.Object3D) => void
/** <Facemesh> ref api */
facemeshApiRef: RefObject<FacemeshApi>
/** <Webcam> ref api */
webcamApiRef: RefObject<WebcamApi>
/** Play the video */
play: () => void
/** Pause the video */
pause: () => void
}
```
## FaceControls events
Two `THREE.Event`s are dispatched on `FaceControls` ref object:
## Breaking changes
- `{ type: "stream", stream: MediaStream }` -- when webcam's [`.getUserMedia()`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) promise is resolved
- `{ type: "videoFrame", texture: THREE.VideoTexture, time: number }` -- each time a new video frame is sent to the compositor (thanks to rVFC)
### 9.120.0
> [!Note]
> rVFC
>
> Internally, `FaceControls` uses [`requestVideoFrameCallback`](https://caniuse.com/mdn-api_htmlvideoelement_requestvideoframecallback), you may need [a polyfill](https://github.com/ThaUnknown/rvfc-polyfill) (for Firefox).
<details>
## FaceControls[manualDetect]
<summary>`FaceControls` was [simplified](https://github.com/pmndrs/drei/pull/2242).</summary>
By default, `detect` is called on each `"videoFrame"`. You can disable this by `manualDetect` and call `detect` yourself.
Following props were deleted:
For example:
- `autostart`: now use `videoTexture.start`
- `webcam`: instead of `webcam: false`, you can now [`manualDetect`](http://localhost:6006/?path=/story/controls-facecontrols--face-controls-st-2)
- `webcamVideoTextureSrc`: now use `videoTexture.src` (or instantiate your own video-texture[^1] outside)
- `onVideoFrame`: now use `videoTexture.onVideoFrame` (or instantiate your own video-texture[^1] outside)
```jsx
const controls = useThree((state) => state.controls)
Following api methods/fields were deleted:
const onVideoFrame = useCallback((event) => {
controls.detect(event.texture.source.data, event.time)
}, [controls])
- `detect`: you can now [`manualDetect`](http://localhost:6006/?path=/story/controls-facecontrols--face-controls-st-2) outside and pass `faceLandmarkerResult`
- `webcamApiRef`: if you need `videoTextureRef`, instantiate your own video-texture[^1] outside
- `play`/`pause`: same, if you need the `video` object, instantiate your own video-texture[^1] outside
<FaceControls makeDefault
manualDetect
onVideoFrame={onVideoFrame}
/>
```
[^1]: `<VideoTexture>` or `<WebcamVideoTexture>`
## FaceControls[manualUpdate]

By default, `update` method is called each rAF `useFrame`. You can disable this by `manualUpdate` and call it yourself:

```jsx
const controls = useThree((state) => state.controls)

useFrame((_, delta) => {
controls.update(delta) // 60 or 120 FPS with default damping
})

<FaceControls makeDefault manualUpdate />
```

Or, if you want your own custom damping, use `computeTarget` method and update the camera pos/rot yourself with:

```jsx
import * as easing from 'maath/easing'

const camera = useThree((state) => state.camera)

const [current] = useState(() => new THREE.Object3D())

useFrame((_, delta) => {
const target = controls?.computeTarget()

if (target) {
//
// A. Define your own damping
//

const eps = 1e-9
easing.damp3(current.position, target.position, 0.25, delta, undefined, undefined, eps)
easing.dampE(current.rotation, target.rotation, 0.25, delta, undefined, undefined, eps)
camera.position.copy(current.position)
camera.rotation.copy(current.rotation)

//
// B. Or maybe with no damping at all?
//

// camera.position.copy(target.position)
// camera.rotation.copy(target.rotation)
}
})
```
</details>
18 changes: 18 additions & 0 deletions docs/misc/face-landmarker.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,21 @@ faceLandmarkerOptions.baseOptions.modelAssetPath = modelAssetPath;

<FaceLandmarker basePath={visionBasePath} options={faceLandmarkerOptions}>
```

## instance

You can get the FaceLandmarker instance through `ref`:

```tsx
const faceLandmarkerRef = useRef<ElementRef<typeof FaceLandmarker>>(null)

<FaceLandmarker ref={faceLandmarkerRef}>
{/* ... */}
</FaceLandmarker>
```

or using `useFaceLandmarker()` from a descendant component:

```jsx
const faceLandmarker = useFaceLandmarker()
```
Loading

0 comments on commit 75260b8

Please sign in to comment.