Skip to content

Commit

Permalink
Merge branch 'master' into fix/stats-gl-props
Browse files Browse the repository at this point in the history
  • Loading branch information
devloop01 authored Dec 5, 2024
2 parents 0fba9e1 + f759344 commit 0773ae0
Show file tree
Hide file tree
Showing 11 changed files with 295 additions and 20 deletions.
39 changes: 39 additions & 0 deletions .storybook/stories/ScreenVideoTexture.stories.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import * as THREE from 'three'
import * as React from 'react'
import { Suspense } from 'react'
import { Meta, StoryObj } from '@storybook/react'

import { Setup } from '../Setup'

import { Plane, ScreenVideoTexture } from '../../src'

export default {
title: 'Misc/ScreenVideoTexture',
component: ScreenVideoTexture,
decorators: [
(Story) => (
<Setup>
<Story />
</Setup>
),
],
} satisfies Meta<typeof ScreenVideoTexture>

type Story = StoryObj<typeof ScreenVideoTexture>

function ScreenVideoTextureScene(props: React.ComponentProps<typeof ScreenVideoTexture>) {
return (
<Plane args={[4, 2.25]}>
<Suspense fallback={<meshBasicMaterial color="gray" />}>
<ScreenVideoTexture {...props}>
{(texture) => <meshBasicMaterial side={THREE.DoubleSide} map={texture} toneMapped={false} />}
</ScreenVideoTexture>
</Suspense>
</Plane>
)
}

export const ScreenVideoTextureSt = {
render: (args) => <ScreenVideoTextureScene {...args} />,
name: 'Default',
} satisfies Story
39 changes: 39 additions & 0 deletions .storybook/stories/WebcamVideoTexture.stories.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import * as THREE from 'three'
import * as React from 'react'
import { Suspense } from 'react'
import { Meta, StoryObj } from '@storybook/react'

import { Setup } from '../Setup'

import { Plane, WebcamVideoTexture } from '../../src'

export default {
title: 'Misc/WebcamVideoTexture',
component: WebcamVideoTexture,
decorators: [
(Story) => (
<Setup>
<Story />
</Setup>
),
],
} satisfies Meta<typeof WebcamVideoTexture>

type Story = StoryObj<typeof WebcamVideoTexture>

function WebcamVideoTextureScene(props: React.ComponentProps<typeof WebcamVideoTexture>) {
return (
<Plane args={[4, 2.25]}>
<Suspense fallback={<meshBasicMaterial color="gray" />}>
<WebcamVideoTexture {...props}>
{(texture) => <meshBasicMaterial side={THREE.DoubleSide} map={texture} toneMapped={false} />}
</WebcamVideoTexture>
</Suspense>
</Plane>
)
}

export const WebcamVideoTextureSt = {
render: (args) => <WebcamVideoTextureScene {...args} />,
name: 'Default',
} satisfies Story
13 changes: 8 additions & 5 deletions docs/controls/introduction.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
title: Controls
---

If available controls have damping enabled by default, they manage their own updates, remove themselves on unmount, are compatible with the `frameloop="demand"` canvas-flag. They inherit all props from their underlying [THREE controls](https://github.com/mrdoob/three.js/tree/master/examples/jsm/controls). They are the first effects to run before all other useFrames, to ensure that other components may mutate the camera on top of them.
If available controls have damping enabled by default, they manage their own updates, remove themselves on unmount, are compatible with the `frameloop="demand"` canvas-flag. They inherit all props from their underlying [THREE.Controls](https://threejs.org/docs/index.html?q=controls#api/en/extras/Controls). They are the first effects to run before all other useFrames, to ensure that other components may mutate the camera on top of them.

[Some controls](https://github.com/search?q=repo%3Apmndrs%2Fdrei+language%3ATSX+path%3A%2F%5Esrc%5C%2Fcore%5C%2F.*Controls%5C.tsx%2F+makeDefault&type=code) allow you to set `makeDefault`, similar to, for instance, `PerspectiveCamera`. This will set [@react-three/fiber](https://docs.pmnd.rs/react-three-fiber/api/hooks#usethree)'s `controls` field in the root store. This can make it easier in situations where you want controls to be known and other parts of the app could respond to it. Some drei controls already take it into account, like `CameraShake`, `Gizmo` and `TransformControls`.
[Some controls](https://github.com/search?q=repo%3Apmndrs%2Fdrei+language%3ATSX+path%3A%2F%5Esrc%5C%2F.*Controls%5C.tsx%2F+makeDefault&type=code) allow you to set `makeDefault`, similar to, for instance, `PerspectiveCamera`. This will set [@react-three/fiber](https://docs.pmnd.rs/react-three-fiber/api/hooks#usethree)'s `controls` field in the root store. This can make it easier in situations where you want controls to be known and other parts of the app could respond to it. Some drei controls already take it into account, like `CameraShake`, `Gizmo` and `TransformControls`.

```tsx
<CameraControls makeDefault />
Expand All @@ -14,8 +14,11 @@ If available controls have damping enabled by default, they manage their own upd
const controls = useThree((state) => state.controls)
```

Drei currently exports OrbitControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), MapControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-st), TrackballControls, ArcballControls, FlyControls, DeviceOrientationControls, PointerLockControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), FirstPersonControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) CameraControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story) and FaceControls [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-facecontrols)
Drei currently exports `OrbitControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-orbitcontrols--orbit-controls-story), `MapControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-mapcontrols--map-controls-st), `TrackballControls`, `ArcballControls`, `FlyControls`, `DeviceOrientationControls`, `PointerLockControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-pointerlockcontrols--pointer-lock-controls-scene-st), `FirstPersonControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-firstpersoncontrols--first-person-controls-story) `CameraControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-cameracontrols--camera-controls-story) `FaceControls` [![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.vercel.app/?path=/story/controls-facecontrols) and other [`*Controls`](https://github.com/search?q=repo%3Apmndrs%2Fdrei+language%3ATSX+path%3A%2F%5Esrc%5C%2F.*Controls%5C.tsx%2F&type=code)

All controls react to the default camera. If you have a `<PerspectiveCamera makeDefault />` in your scene, they will control it. If you need to inject an imperative camera or one that isn't the default, use the `camera` prop: `<OrbitControls camera={MyCamera} />`.
Some controls drive an object, not a camera, eg: `PresentationControls`.

PointerLockControls additionally supports a `selector` prop, which enables the binding of `click` event handlers for control activation to other elements than `document` (e.g. a 'Click here to play' button). All elements matching the `selector` prop will activate the controls. It will also center raycast events by default, so regular onPointerOver/etc events on meshes will continue to work.
But all controls involving a camera, react to the default one. If you have a `<PerspectiveCamera makeDefault />` in your scene, they will control it. If you need to inject an imperative camera or one that isn't the default, use the `camera` prop: `<OrbitControls camera={MyCamera} />`.


`PointerLockControls` additionally supports a `selector` prop, which enables the binding of `click` event handlers for control activation to other elements than `document` (e.g. a 'Click here to play' button). All elements matching the `selector` prop will activate the controls. It will also center raycast events by default, so regular onPointerOver/etc events on meshes will continue to work.
4 changes: 4 additions & 0 deletions docs/getting-started/introduction.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ nav: -1
[![Discord Shield](https://img.shields.io/discord/740090768164651008?style=flat&colorA=000000&colorB=000000&label=discord&logo=discord&logoColor=ffffff)](https://discord.com/channels/740090768164651008/741751532592038022)
[![Open in GitHub Codespaces](https://img.shields.io/static/v1?&message=Open%20in%20%20Codespaces&style=flat&colorA=000000&colorB=000000&label=GitHub&logo=github&logoColor=ffffff)](https://github.com/codespaces/new?template_repository=pmndrs%2Fdrei)

<Entries />

## INSTALL

If you make a component that is generic enough to be useful to others, think about [CONTRIBUTING](CONTRIBUTING.md)!

```bash
Expand Down
26 changes: 26 additions & 0 deletions docs/loaders/screen-video-texture.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
---
title: ScreenVideoTexture
sourcecode: src/web/ScreenVideoTexture.tsx
---

[![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.pmnd.rs/?path=/story/misc-screenvideotexture) ![](https://img.shields.io/badge/-suspense-brightgreen)

<Intro>Create a video texture from [`getDisplayMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia)</Intro>

```tsx
export type ScreenVideoTextureProps = Omit<VideoTextureProps, 'src'> & {
options?: DisplayMediaStreamOptions
}
```
```jsx
<ScreenVideoTexture>
{(texture) => <meshBasicMaterial map={texture} />}
```
or exposed via `ref`:
```jsx
const textureRef = useRef()
<ScreenVideoTexture ref={textureRef} />
```
37 changes: 36 additions & 1 deletion docs/loaders/video-texture-use-video-texture.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,18 @@ export function useVideoTexture(
{
unsuspend = 'loadedmetadata',
start = true,
hls: hlsConfig = {},
hls = {},
crossOrigin = 'anonymous',
muted = true,
loop = true,
playsInline = true,
onVideoFrame,
...videoProps
}: {
unsuspend?: keyof HTMLVideoElementEventMap
start?: boolean
hls?: Parameters<typeof getHls>[0]
onVideoFrame: VideoFrameRequestCallback
} & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
)
```
Expand Down Expand Up @@ -84,3 +86,36 @@ const texture = useVideoTexture('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3
hls: { abrEwmaFastLive: 1.0, abrEwmaSlowLive: 3.0, enableWorker: true },
})
```

## `requestVideoFrameCallback` (rVFC)

`useVideoTexture` supports [`requestVideoFrameCallback`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement/requestVideoFrameCallback):

```jsx
useVideoTexture(src, {
onVideoFrame: (now, metadata) => {}
})
```

## `<VideoTexture>` Component

```tsx
export type VideoTextureProps = {
children?: (texture: THREE.VideoTexture) => React.ReactNode
src: UseVideoTextureParams[0]
} & UseVideoTextureParams[1]
```

You can access the texture via children's render prop:

```jsx
<VideoTexture src="/video.mp4">
{(texture) => <meshBasicMaterial map={texture} />}
```

or exposed via `ref`:

```jsx
const textureRef = useRef()
<VideoTexture ref={textureRef} src="/video.mp4" />
```
26 changes: 26 additions & 0 deletions docs/loaders/webcam-video-texture.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
---
title: WebcamVideoTexture
sourcecode: src/web/WebcamVideoTexture.tsx
---

[![](https://img.shields.io/badge/-storybook-%23ff69b4)](https://drei.pmnd.rs/?path=/story/misc-webcamvideotexture) ![](https://img.shields.io/badge/-suspense-brightgreen)

<Intro>Create a video texture from [`getUserMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia)</Intro>

```tsx
export type WebcamVideoTextureProps = Omit<VideoTextureProps, 'src'> & {
constraints?: MediaStreamConstraints
}
```
```jsx
<WebcamVideoTexture>
{(texture) => <meshBasicMaterial map={texture} />}
```
or exposed via `ref`:
```jsx
const textureRef = useRef()
<WebcamVideoTexture ref={textureRef} />
```
66 changes: 52 additions & 14 deletions src/core/VideoTexture.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint react-hooks/exhaustive-deps: 1 */
import * as React from 'react'
import * as THREE from 'three'
import { useEffect, useRef } from 'react'
import { forwardRef, useEffect, useImperativeHandle, useRef } from 'react'
import { useThree } from '@react-three/fiber'
import { suspend } from 'suspend-react'
import { type default as Hls, Events } from 'hls.js'
Expand Down Expand Up @@ -31,11 +32,22 @@ export function useVideoTexture(
muted = true,
loop = true,
playsInline = true,
onVideoFrame,
...videoProps
}: {
/** Event name that will unsuspend the video */
unsuspend?: keyof HTMLVideoElementEventMap
/** Auto start the video once unsuspended */
start?: boolean
/** HLS config */
hls?: Parameters<typeof getHls>[0]
/**
* request Video Frame Callback (rVFC)
*
* @see https://web.dev/requestvideoframecallback-rvfc/
* @see https://www.remotion.dev/docs/video-manipulation
* */
onVideoFrame?: VideoFrameRequestCallback
} & Partial<Omit<HTMLVideoElement, 'children' | 'src' | 'srcObject'>> = {}
) {
const gl = useThree((state) => state.gl)
Expand Down Expand Up @@ -81,6 +93,9 @@ export function useVideoTexture(
[srcOrSrcObject]
)

const video = texture.source.data as HTMLVideoElement
useVideoFrame(video, onVideoFrame)

useEffect(() => {
start && texture.image.play()

Expand All @@ -96,22 +111,45 @@ export function useVideoTexture(
}

//
// VideoTexture
//

type UseVideoTextureParams = Parameters<typeof useVideoTexture>
type VideoTexture = ReturnType<typeof useVideoTexture>

export type VideoTextureProps = {
children?: (texture: VideoTexture) => React.ReactNode
src: UseVideoTextureParams[0]
} & UseVideoTextureParams[1]

export const VideoTexture = /* @__PURE__ */ forwardRef<VideoTexture, VideoTextureProps>(
({ children, src, ...config }, fref) => {
const texture = useVideoTexture(src, config)

type UseVideoTexture = Parameters<typeof useVideoTexture>
useEffect(() => {
return () => void texture.dispose()
}, [texture])

export const VideoTexture = ({
children,
src,
...config
}: {
children?: (texture: ReturnType<typeof useVideoTexture>) => React.ReactNode
src: UseVideoTexture[0]
} & UseVideoTexture[1]) => {
const ret = useVideoTexture(src, config)
useImperativeHandle(fref, () => texture, [texture]) // expose texture through ref

return <>{children?.(texture)}</>
}
)

// rVFC hook

const useVideoFrame = (video: HTMLVideoElement, f?: VideoFrameRequestCallback) => {
useEffect(() => {
return () => void ret.dispose()
}, [ret])
if (!f) return
if (!video.requestVideoFrameCallback) return

let handle: ReturnType<(typeof video)['requestVideoFrameCallback']>
const callback: VideoFrameRequestCallback = (...args) => {
f(...args)
handle = video.requestVideoFrameCallback(callback)
}
video.requestVideoFrameCallback(callback)

return <>{children?.(ret)}</>
return () => video.cancelVideoFrameCallback(handle)
}, [video, f])
}
26 changes: 26 additions & 0 deletions src/web/ScreenVideoTexture.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import * as React from 'react'
import { forwardRef, useEffect } from 'react'
import { suspend, clear } from 'suspend-react'
import { VideoTexture, VideoTextureProps } from '..'

export type ScreenVideoTextureProps = Omit<VideoTextureProps, 'src'> & {
options?: DisplayMediaStreamOptions
}

/**
* Create a video texture from [`getDisplayMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia)
*/
export const ScreenVideoTexture = /* @__PURE__ */ forwardRef<THREE.VideoTexture, ScreenVideoTextureProps>(
({ options = { video: true }, ...props }, fref) => {
const mediaStream = suspend(() => navigator.mediaDevices.getDisplayMedia(options), [])

useEffect(() => {
return () => {
mediaStream?.getTracks().forEach((track) => track.stop())
clear([])
}
}, [mediaStream])

return <VideoTexture ref={fref} {...props} src={mediaStream} />
}
)
35 changes: 35 additions & 0 deletions src/web/WebcamVideoTexture.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import * as React from 'react'
import { forwardRef, useEffect } from 'react'
import { suspend, clear } from 'suspend-react'
import { VideoTexture, VideoTextureProps } from '..'

export type WebcamVideoTextureProps = Omit<VideoTextureProps, 'src'> & {
constraints?: MediaStreamConstraints
}

/**
* Create a video texture from [`getUserMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia)
*/
export const WebcamVideoTexture = /* @__PURE__ */ forwardRef<THREE.VideoTexture, WebcamVideoTextureProps>(
(
{
constraints = {
audio: false,
video: { facingMode: 'user' },
},
...props
},
fref
) => {
const mediaStream = suspend(() => navigator.mediaDevices.getUserMedia(constraints), [])

useEffect(() => {
return () => {
mediaStream?.getTracks().forEach((track) => track.stop())
clear([])
}
}, [mediaStream])

return <VideoTexture ref={fref} {...props} src={mediaStream} />
}
)
Loading

0 comments on commit 0773ae0

Please sign in to comment.