diff --git a/.storybook/stories/FaceControls.stories.tsx b/.storybook/stories/FaceControls.stories.tsx
new file mode 100644
index 000000000..f5ca126b6
--- /dev/null
+++ b/.storybook/stories/FaceControls.stories.tsx
@@ -0,0 +1,38 @@
+import * as THREE from 'three'
+import * as React from 'react'
+import { withKnobs, number } from '@storybook/addon-knobs'
+import { Vector3 } from 'three'
+
+import { Setup } from '../Setup'
+
+import { FaceLandmarker, FaceControls, Box } from '../../src'
+
+export default {
+ title: 'Controls/FaceControls',
+ component: FaceControls,
+ decorators: [withKnobs, (storyFn) => {storyFn()}],
+}
+
+export const FaceControlsSt = ({ eyes }) => (
+ <>
+
+
+
+
+
+
+
+
+
+
+
+)
+FaceControlsSt.args = {
+ eyes: undefined,
+}
+
+FaceControlsSt.argTypes = {
+ eyes: { control: { type: 'boolean' } },
+}
+
+FaceControlsSt.storyName = 'Default'
diff --git a/README.md b/README.md
index 21f034da7..593e71f0f 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,7 @@ The `native` route of the library **does not** export `Html` or `Loader`. The de
ScrollControls
PresentationControls
KeyboardControls
+ FaceControls
Gizmos
@@ -358,7 +359,7 @@ If available controls have damping enabled by default, they manage their own upd
const controls = useThree((state) => state.controls)
```
-Drei currently exports OrbitControls [](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), MapControls [](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-scene-st), TrackballControls, ArcballControls, FlyControls, DeviceOrientationControls, PointerLockControls [](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), FirstPersonControls [](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) and CameraControls [](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story)
+Drei currently exports OrbitControls [](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), MapControls [](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-scene-st), TrackballControls, ArcballControls, FlyControls, DeviceOrientationControls, PointerLockControls [](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), FirstPersonControls [](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) CameraControls [](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story) and FaceControls [](https://drei.vercel.app/?path=/story/controls-facecontrols)
All controls react to the default camera. If you have a `` in your scene, they will control it. If you need to inject an imperative camera or one that isn't the default, use the `camera` prop: ``.
@@ -595,6 +596,76 @@ function Foo() {
}
```
+#### FaceControls
+
+The camera follows your face.
+
+Pre-requisite: wrap into a `FaceLandmarker` provider
+
+```tsx
+...
+```
+
+```tsx
+
+```
+
+```tsx
+type FaceControlsProps = {
+ /** The camera to be controlled, default: global state camera */
+ camera?: THREE.Camera
+ /** Whether to autostart the webcam, default: true */
+ autostart?: boolean
+ /** Enable/disable the webcam, default: true */
+ webcam?: boolean
+ /** A custom video URL or mediaStream, default: undefined */
+ webcamVideoTextureSrc?: VideoTextureSrc
+ /** Disable the rAF camera position/rotation update, default: false */
+ manualUpdate?: boolean
+ /** Disable the rVF face-detection, default: false */
+ manualDetect?: boolean
+ /** Callback function to call on "videoFrame" event, default: undefined */
+ onVideoFrame?: (e: THREE.Event) => void
+ /** Reference this FaceControls instance as state's `controls` */
+ makeDefault?: boolean
+ /** Approximate time to reach the target. A smaller value will reach the target faster. */
+ smoothTime?: number
+ /** Apply position offset extracted from `facialTransformationMatrix` */
+ offset?: boolean
+ /** Offset sensitivity factor, less is more sensible, default: 80 */
+ offsetScalar?: number
+ /** Enable eye-tracking */
+ eyes?: boolean
+ /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */
+ eyesAsOrigin?: boolean
+ /** Constant depth of the Facemesh, default: .15 */
+ depth?: number
+ /** Enable debug mode, default: false */
+ debug?: boolean
+ /** Facemesh options, default: undefined */
+ facemesh?: FacemeshProps
+}
+```
+
+```tsx
+type FaceControlsApi = THREE.EventDispatcher & {
+ /** Detect faces from the video */
+ detect: (video: HTMLVideoElement, time: number) => void
+ /** Compute the target for the camera */
+ computeTarget: () => THREE.Object3D
+ /** Update camera's position/rotation to the `target` */
+ update: (delta: number, target?: THREE.Object3D) => void
+ /** ref api */
+ facemeshApiRef: RefObject
+ /** ref api */
+ webcamApiRef: RefObject
+ /** Play the video */
+ play: () => void
+ /** Pause the video */
+ pause: () => void
+}
+```
+
# Gizmos
#### GizmoHelper
diff --git a/package.json b/package.json
index 25a71b80d..61a3f512e 100644
--- a/package.json
+++ b/package.json
@@ -55,6 +55,7 @@
},
"dependencies": {
"@babel/runtime": "^7.11.2",
+ "@mediapipe/tasks-vision": "^0.10.0",
"@react-spring/three": "~9.6.1",
"@use-gesture/react": "^10.2.24",
"camera-controls": "^2.3.1",
diff --git a/src/core/FaceControls.tsx b/src/core/FaceControls.tsx
new file mode 100644
index 000000000..d605cc6f4
--- /dev/null
+++ b/src/core/FaceControls.tsx
@@ -0,0 +1,413 @@
+import * as THREE from 'three'
+import * as React from 'react'
+import {
+ useState,
+ Suspense,
+ useEffect,
+ useRef,
+ useCallback,
+ forwardRef,
+ useMemo,
+ useImperativeHandle,
+ RefObject,
+ createContext,
+ useContext,
+} from 'react'
+import { useFrame, useThree } from '@react-three/fiber'
+import { FaceLandmarkerResult } from '@mediapipe/tasks-vision'
+import { easing } from 'maath'
+
+import { useVideoTexture } from './useVideoTexture'
+import { Facemesh, FacemeshApi, FacemeshProps } from './Facemesh'
+import { useFaceLandmarker } from './FaceLandmarker'
+
+type VideoTextureSrc = Parameters[0] // useVideoTexture 1st arg `src` type
+
+function mean(v1: THREE.Vector3, v2: THREE.Vector3) {
+ return v1.clone().add(v2).multiplyScalar(0.5)
+}
+
+function localToLocal(objSrc: THREE.Object3D, v: THREE.Vector3, objDst: THREE.Object3D) {
+ // see: https://discourse.threejs.org/t/object3d-localtolocal/51564
+ const v_world = objSrc.localToWorld(v)
+ return objDst.worldToLocal(v_world)
+}
+
+//
+//
+//
+
+type FaceControlsProps = {
+ /** The camera to be controlled, default: global state camera */
+ camera?: THREE.Camera
+ /** Whether to autostart the webcam, default: true */
+ autostart?: boolean
+ /** Enable/disable the webcam, default: true */
+ webcam?: boolean
+ /** A custom video URL or mediaStream, default: undefined */
+ webcamVideoTextureSrc?: VideoTextureSrc
+ /** Disable the rAF camera position/rotation update, default: false */
+ manualUpdate?: boolean
+ /** Disable the rVF face-detection, default: false */
+ manualDetect?: boolean
+ /** Callback function to call on "videoFrame" event, default: undefined */
+ onVideoFrame?: (e: THREE.Event) => void
+ /** Reference this FaceControls instance as state's `controls` */
+ makeDefault?: boolean
+ /** Approximate time to reach the target. A smaller value will reach the target faster. */
+ smoothTime?: number
+ /** Apply position offset extracted from `facialTransformationMatrix` */
+ offset?: boolean
+ /** Offset sensitivity factor, less is more sensible, default: 80 */
+ offsetScalar?: number
+ /** Enable eye-tracking */
+ eyes?: boolean
+ /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */
+ eyesAsOrigin?: boolean
+ /** Constant depth of the Facemesh, default: .15 */
+ depth?: number
+ /** Enable debug mode, default: false */
+ debug?: boolean
+ /** Facemesh options, default: undefined */
+ facemesh?: FacemeshProps
+}
+
+type FaceControlsApi = THREE.EventDispatcher & {
+ /** Detect faces from the video */
+ detect: (video: HTMLVideoElement, time: number) => void
+ /** Compute the target for the camera */
+ computeTarget: () => THREE.Object3D
+ /** Update camera's position/rotation to the `target` */
+ update: (delta: number, target?: THREE.Object3D) => void
+ /** ref api */
+ facemeshApiRef: RefObject
+ /** ref api */
+ webcamApiRef: RefObject
+ /** Play the video */
+ play: () => void
+ /** Pause the video */
+ pause: () => void
+}
+
+const FaceControlsContext = createContext({} as FaceControlsApi)
+
+export const FaceControls = forwardRef(
+ (
+ {
+ camera,
+ autostart = true,
+ webcam = true,
+ webcamVideoTextureSrc,
+ manualUpdate = false,
+ manualDetect = false,
+ onVideoFrame,
+ smoothTime = 0.25,
+ offset = true,
+ offsetScalar = 80,
+ eyes = false,
+ eyesAsOrigin = true,
+ depth = 0.15,
+ debug = false,
+ facemesh,
+ makeDefault,
+ },
+ fref
+ ) => {
+ const scene = useThree((state) => state.scene)
+ const defaultCamera = useThree((state) => state.camera)
+ const set = useThree((state) => state.set)
+ const get = useThree((state) => state.get)
+ const explCamera = camera || defaultCamera
+
+ const webcamApiRef = useRef(null)
+
+ const facemeshApiRef = useRef(null)
+
+ //
+ // computeTarget()
+ //
+ // Compute `target` position and rotation for the camera (according to )
+ //
+ // 1. 👀 either following the 2 eyes
+ // 2. 👤 or just the head mesh
+ //
+
+ const [target] = useState(() => new THREE.Object3D())
+ const [irisRightDirPos] = useState(() => new THREE.Vector3())
+ const [irisLeftDirPos] = useState(() => new THREE.Vector3())
+ const [irisRightLookAt] = useState(() => new THREE.Vector3())
+ const [irisLeftLookAt] = useState(() => new THREE.Vector3())
+ const computeTarget = useCallback(() => {
+ // same parent as the camera
+ target.parent = explCamera.parent
+
+ const facemeshApi = facemeshApiRef.current
+ if (facemeshApi) {
+ const { outerRef, eyeRightRef, eyeLeftRef } = facemeshApi
+
+ if (eyeRightRef.current && eyeLeftRef.current) {
+ // 1. 👀
+
+ const { irisDirRef: irisRightDirRef } = eyeRightRef.current
+ const { irisDirRef: irisLeftDirRef } = eyeLeftRef.current
+
+ if (irisRightDirRef.current && irisLeftDirRef.current && outerRef.current) {
+ //
+ // position: mean of irisRightDirPos,irisLeftDirPos
+ //
+ irisRightDirPos.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current))
+ irisLeftDirPos.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 0), outerRef.current))
+ target.position.copy(
+ localToLocal(outerRef.current, mean(irisRightDirPos, irisLeftDirPos), explCamera.parent || scene)
+ )
+
+ //
+ // lookAt: mean of irisRightLookAt,irisLeftLookAt
+ //
+ irisRightLookAt.copy(localToLocal(irisRightDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current))
+ irisLeftLookAt.copy(localToLocal(irisLeftDirRef.current, new THREE.Vector3(0, 0, 1), outerRef.current))
+ target.lookAt(outerRef.current.localToWorld(mean(irisRightLookAt, irisLeftLookAt)))
+ }
+ } else {
+ // 2. 👤
+
+ if (outerRef.current) {
+ target.position.copy(localToLocal(outerRef.current, new THREE.Vector3(0, 0, 0), explCamera.parent || scene))
+ target.lookAt(outerRef.current.localToWorld(new THREE.Vector3(0, 0, 1)))
+ }
+ }
+ }
+
+ return target
+ }, [explCamera, irisLeftDirPos, irisLeftLookAt, irisRightDirPos, irisRightLookAt, scene, target])
+
+ //
+ // update()
+ //
+ // Updating the camera `current` position and rotation, following `target`
+ //
+
+ const [current] = useState(() => new THREE.Object3D())
+ const update = useCallback(
+ function (delta, target) {
+ if (explCamera) {
+ target ??= computeTarget()
+
+ if (smoothTime > 0) {
+ // damping current
+ const eps = 1e-9
+ easing.damp3(current.position, target.position, smoothTime, delta, undefined, undefined, eps)
+ easing.dampE(current.rotation, target.rotation, smoothTime, delta, undefined, undefined, eps)
+ } else {
+ // instant
+ current.position.copy(target.position)
+ current.rotation.copy(target.rotation)
+ }
+
+ explCamera.position.copy(current.position)
+ explCamera.rotation.copy(current.rotation)
+ }
+ },
+ [explCamera, computeTarget, smoothTime, current.position, current.rotation]
+ )
+
+ //
+ // detect()
+ //
+
+ const [faces, setFaces] = useState()
+ const faceLandmarker = useFaceLandmarker()
+ const detect = useCallback(
+ (video, time) => {
+ const faces = faceLandmarker?.detectForVideo(video, time)
+ setFaces(faces)
+ },
+ [faceLandmarker]
+ )
+
+ useFrame((_, delta) => {
+ if (!manualUpdate) {
+ update(delta)
+ }
+ })
+
+ // Ref API
+ const api = useMemo(
+ () =>
+ Object.assign(Object.create(THREE.EventDispatcher.prototype), {
+ detect,
+ computeTarget,
+ update,
+ facemeshApiRef,
+ webcamApiRef,
+ // shorthands
+ play: () => {
+ webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.play()
+ },
+ pause: () => {
+ webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.pause()
+ },
+ }),
+ [detect, computeTarget, update]
+ )
+ useImperativeHandle(fref, () => api, [api])
+
+ //
+ // events callbacks
+ //
+
+ useEffect(() => {
+ const onVideoFrameCb = (e: THREE.Event) => {
+ if (!manualDetect) detect(e.texture.source.data, e.time)
+ if (onVideoFrame) onVideoFrame(e)
+ }
+
+ api.addEventListener('videoFrame', onVideoFrameCb)
+
+ return () => {
+ api.removeEventListener('videoFrame', onVideoFrameCb)
+ }
+ }, [api, detect, faceLandmarker, manualDetect, onVideoFrame])
+
+ // `controls` global state
+ useEffect(() => {
+ if (makeDefault) {
+ const old = get().controls
+ set({ controls: api })
+ return () => set({ controls: old })
+ }
+ }, [makeDefault, api, get, set])
+
+ const points = faces?.faceLandmarks[0]
+ const facialTransformationMatrix = faces?.facialTransformationMatrixes?.[0]
+ const faceBlendshapes = faces?.faceBlendshapes?.[0]
+ return (
+
+ {webcam && }
+
+
+
+
+
+ )
+ }
+)
+
+export const useFaceControls = () => useContext(FaceControlsContext)
+
+//
+// Webcam
+//
+
+type WebcamApi = {
+ videoTextureApiRef: RefObject
+}
+
+type WebcamProps = {
+ videoTextureSrc?: VideoTextureSrc
+ autostart?: boolean
+}
+
+const Webcam = forwardRef(({ videoTextureSrc, autostart = true }, fref) => {
+ const [stream, setStream] = useState()
+
+ const videoTextureApiRef = useRef(null)
+
+ const faceControls = useFaceControls()
+ useEffect(() => {
+ let strm: MediaStream
+
+ if (!videoTextureSrc) {
+ navigator.mediaDevices
+ .getUserMedia({
+ audio: false,
+ video: { facingMode: 'user' },
+ })
+ .then((s) => {
+ strm = s
+ faceControls.dispatchEvent({ type: 'stream', stream: strm })
+ setStream(strm)
+ })
+ .catch(console.error)
+ }
+
+ return () => strm?.getTracks().forEach((track) => track.stop())
+ }, [faceControls, videoTextureSrc])
+
+ // ref-api
+ const api = useMemo(
+ () => ({
+ videoTextureApiRef,
+ }),
+ []
+ )
+ useImperativeHandle(fref, () => api, [api])
+
+ return (
+
+
+
+ )
+})
+
+//
+// VideoTexture
+//
+
+type VideoTextureApi = { texture: THREE.VideoTexture }
+type VideoTextureProps = { src: VideoTextureSrc; start: boolean }
+
+const VideoTexture = forwardRef(({ src, start }, fref) => {
+ const texture = useVideoTexture(src, { start })
+ const video = texture.source.data
+
+ const faceControls = useFaceControls()
+ const onVideoFrame = useCallback(
+ (time: number) => {
+ faceControls.dispatchEvent({ type: 'videoFrame', texture, time })
+ },
+ [texture, faceControls]
+ )
+ useVideoFrame(video, onVideoFrame)
+
+ // ref-api
+ const api = useMemo(
+ () => ({
+ texture,
+ }),
+ [texture]
+ )
+ useImperativeHandle(fref, () => api, [api])
+
+ return <>
+})
+
+const useVideoFrame = (video: HTMLVideoElement, f: (...args: any) => any) => {
+ // https://web.dev/requestvideoframecallback-rvfc/
+ // https://www.remotion.dev/docs/video-manipulation
+ useEffect(() => {
+ if (!video || !video.requestVideoFrameCallback) return
+ let handle: number
+ function callback(...args: any) {
+ f(...args)
+ handle = video.requestVideoFrameCallback(callback)
+ }
+ video.requestVideoFrameCallback(callback)
+
+ return () => video.cancelVideoFrameCallback(handle)
+ }, [video, f])
+}
diff --git a/src/core/FaceLandmarker.tsx b/src/core/FaceLandmarker.tsx
new file mode 100644
index 000000000..012f9c230
--- /dev/null
+++ b/src/core/FaceLandmarker.tsx
@@ -0,0 +1,49 @@
+import * as React from 'react'
+import { createContext, ReactNode, useContext, useEffect, useState } from 'react'
+import { FilesetResolver, FaceLandmarker as FaceLandmarkerImpl, FaceLandmarkerOptions } from '@mediapipe/tasks-vision'
+
+const FaceLandmarkerContext = createContext({} as FaceLandmarkerImpl | undefined)
+
+type FaceLandmarkerProps = {
+ basePath?: string
+ options?: FaceLandmarkerOptions
+ children?: ReactNode
+}
+
+export const FaceLandmarkerDefaults = {
+ basePath: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm',
+ options: {
+ baseOptions: {
+ modelAssetPath:
+ 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+ delegate: 'GPU',
+ },
+ runningMode: 'VIDEO',
+ outputFaceBlendshapes: true,
+ outputFacialTransformationMatrixes: true,
+ } as FaceLandmarkerOptions,
+}
+
+export function FaceLandmarker({
+ basePath = FaceLandmarkerDefaults.basePath,
+ options = FaceLandmarkerDefaults.options,
+ children,
+}: FaceLandmarkerProps) {
+ const [faceLandmarker, setFaceLandmarker] = useState()
+ useEffect(() => {
+ let ret: FaceLandmarkerImpl
+
+ FilesetResolver.forVisionTasks(basePath)
+ .then((vision) => FaceLandmarkerImpl.createFromOptions(vision, options))
+ .then((faceLandmarker) => setFaceLandmarker(faceLandmarker))
+ .catch((err) => console.error('error while creating faceLandmarker', err))
+
+ return () => void ret?.close()
+ }, [basePath, options])
+
+ return {children}
+}
+
+export function useFaceLandmarker() {
+ return useContext(FaceLandmarkerContext)
+}
diff --git a/src/core/index.ts b/src/core/index.ts
index 38142c88f..fcdf2b8f5 100644
--- a/src/core/index.ts
+++ b/src/core/index.ts
@@ -38,6 +38,7 @@ export * from './TransformControls'
export * from './PointerLockControls'
export * from './FirstPersonControls'
export * from './CameraControls'
+export * from './FaceControls'
// Gizmos
export * from './GizmoHelper'
@@ -73,6 +74,7 @@ export * from './useTrailTexture'
export * from './useCubeCamera'
export * from './Example'
export * from './SpriteAnimator'
+export * from './FaceLandmarker'
// Modifiers
export * from './CurveModifier'
diff --git a/yarn.lock b/yarn.lock
index 4dab658a9..0461d1e26 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2164,6 +2164,11 @@
"@types/mdx" "^2.0.0"
"@types/react" ">=16"
+"@mediapipe/tasks-vision@^0.10.0":
+ version "0.10.0"
+ resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.0.tgz#d95a816f66889dcb15545636ac7fc78588c1d705"
+ integrity sha512-l7Sqdw9EgO758tLrt+jDqqKnYKjeOFgzC9MRcE4qZnh/XG/o7JBacEF4cAzes+2wsElRCVoVGf8ETllIqgBzWA==
+
"@ndelangen/get-tarball@^3.0.7":
version "3.0.7"
resolved "https://registry.yarnpkg.com/@ndelangen/get-tarball/-/get-tarball-3.0.7.tgz#87c7aef2df4ff4fbdbab6ac9ed32cee142c4b1a3"