From 88cfa312112cd270ac0a7c8c280b51fb822a05ea Mon Sep 17 00:00:00 2001 From: nahkd123 Date: Thu, 3 Oct 2024 01:10:45 +0700 Subject: [PATCH] Video rendering and encoding --- README.md | 7 +++- nahara-motion-ui/package.json | 3 +- nahara-motion-ui/src/ui/bar/TopBar.svelte | 49 ++++++++++++++++++++++- nahara-motion-video/.gitignore | 2 + nahara-motion-video/package.json | 22 ++++++++++ nahara-motion-video/src/encoder.ts | 28 +++++++++++++ nahara-motion-video/src/index.ts | 4 ++ nahara-motion-video/src/muxer.ts | 26 ++++++++++++ nahara-motion-video/src/pipeline.ts | 38 ++++++++++++++++++ nahara-motion-video/src/render.ts | 43 ++++++++++++++++++++ nahara-motion-video/tsconfig.json | 17 ++++++++ nahara-motion/package.json | 2 +- pnpm-lock.yaml | 31 ++++++++++++++ pnpm-workspace.yaml | 1 + 14 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 nahara-motion-video/.gitignore create mode 100644 nahara-motion-video/package.json create mode 100644 nahara-motion-video/src/encoder.ts create mode 100644 nahara-motion-video/src/index.ts create mode 100644 nahara-motion-video/src/muxer.ts create mode 100644 nahara-motion-video/src/pipeline.ts create mode 100644 nahara-motion-video/src/render.ts create mode 100644 nahara-motion-video/tsconfig.json diff --git a/README.md b/README.md index 9dba57d..76b673a 100644 --- a/README.md +++ b/README.md @@ -33,4 +33,9 @@ cd nahara-motion-ui pnpm install pnpm dev # Open 127.0.0.1:5173 to see the app -``` \ No newline at end of file +``` + +## Modules +- `nahara-motion`: Nahara's Motion Engine +- `nahara-motion-ui`: The front-facing user interface for Nahara's Motion +- `nahara-motion-video`: Render and encode the scene to video file (`.mp4` in this case) diff --git a/nahara-motion-ui/package.json b/nahara-motion-ui/package.json index 4af96b9..37ee007 100644 --- a/nahara-motion-ui/package.json +++ b/nahara-motion-ui/package.json @@ -10,7 +10,8 @@ "check": "svelte-check --tsconfig ./tsconfig.json && tsc -p tsconfig.node.json" }, "dependencies": { - "@nahara/motion": "workspace:*" + "@nahara/motion": "workspace:*", + "@nahara/motion-video": "workspace:*" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^3.1.1", diff --git a/nahara-motion-ui/src/ui/bar/TopBar.svelte b/nahara-motion-ui/src/ui/bar/TopBar.svelte index 2c08b37..b836c82 100644 --- a/nahara-motion-ui/src/ui/bar/TopBar.svelte +++ b/nahara-motion-ui/src/ui/bar/TopBar.svelte @@ -1,13 +1,60 @@
-
File
+ +
File
Edit
Window
Help
diff --git a/nahara-motion-video/.gitignore b/nahara-motion-video/.gitignore new file mode 100644 index 0000000..de4d1f0 --- /dev/null +++ b/nahara-motion-video/.gitignore @@ -0,0 +1,2 @@ +dist +node_modules diff --git a/nahara-motion-video/package.json b/nahara-motion-video/package.json new file mode 100644 index 0000000..acb33d8 --- /dev/null +++ b/nahara-motion-video/package.json @@ -0,0 +1,22 @@ +{ + "name": "@nahara/motion-video", + "version": "1.0.0", + "description": "Nahara Motion Video", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc --project tsconfig.json", + "watch": "tsc --watch --project tsconfig.json", + "test": "echo \"Tests are not available for this package.\" && exit 0" + }, + "keywords": [], + "author": "", + "license": "MIT", + "dependencies": { + "@nahara/motion": "workspace:*", + "mp4-muxer": "^5.1.3" + }, + "devDependencies": { + "typescript": "^5.6.2" + } +} diff --git a/nahara-motion-video/src/encoder.ts b/nahara-motion-video/src/encoder.ts new file mode 100644 index 0000000..e1ea0a5 --- /dev/null +++ b/nahara-motion-video/src/encoder.ts @@ -0,0 +1,28 @@ +import { IPipeline } from "./pipeline.js"; + +export class EncoderPipeline implements IPipeline { + next?: (output: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]) => any; + encoder?: VideoEncoder; + + constructor(public readonly options: VideoEncoderConfig) {} + + async initialize(next: (output: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]) => any): Promise { + this.next = next; + this.encoder = new VideoEncoder({ + output: (chunk, meta) => next([chunk, meta]), + error() {} + }); + this.encoder.configure(this.options); + } + + consume(input: VideoFrame): void { + this.encoder!.encode(input); + input.close(); + } + + async finalize(): Promise { + await this.encoder!.flush(); + console.log("EncoderPipeline: Flushed"); + this.encoder!.close(); + } +} \ No newline at end of file diff --git a/nahara-motion-video/src/index.ts b/nahara-motion-video/src/index.ts new file mode 100644 index 0000000..8247a7a --- /dev/null +++ b/nahara-motion-video/src/index.ts @@ -0,0 +1,4 @@ +export * from "./encoder.js"; +export * from "./muxer.js"; +export * from "./pipeline.js"; +export * from "./render.js"; \ No newline at end of file diff --git a/nahara-motion-video/src/muxer.ts b/nahara-motion-video/src/muxer.ts new file mode 100644 index 0000000..9346fc9 --- /dev/null +++ b/nahara-motion-video/src/muxer.ts @@ -0,0 +1,26 @@ +import { ArrayBufferTarget, Muxer, MuxerOptions } from "mp4-muxer"; +import { IPipeline } from "./pipeline.js"; + +export class MuxerPipeline implements IPipeline<[EncodedVideoChunk, EncodedVideoChunkMetadata | undefined], void, ArrayBuffer> { + muxer?: Muxer; + + constructor(public readonly options: (MuxerOptions["video"] & {})) {} + + async initialize(): Promise { + this.muxer = new Muxer({ + target: new ArrayBufferTarget(), + video: this.options, + fastStart: "fragmented" + }); + } + + consume(input: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]): void { + this.muxer!.addVideoChunk(...input); + } + + async finalize(): Promise { + console.log("MuxerPipeline: Finalizing..."); + this.muxer!.finalize(); + return this.muxer!.target.buffer; + } +} \ No newline at end of file diff --git a/nahara-motion-video/src/pipeline.ts b/nahara-motion-video/src/pipeline.ts new file mode 100644 index 0000000..e520d11 --- /dev/null +++ b/nahara-motion-video/src/pipeline.ts @@ -0,0 +1,38 @@ +/** + * Represent a pipeline in the system. + * @param I The input chunk type. + * @param O The output chunk type, which will be passed to next part of pipeline or collected by consumer. + * @param R The result of the pipeline, which will be collected by consumer (does not pass to other pipelines!). Mainly + * used for video file. + */ +export interface IPipeline { + initialize(next: (output: O) => any): Promise; + consume(input: I): void; + finalize(): Promise; +} + +export type AnyPipeline = IPipeline; + +export function concat(a: IPipeline, b: IPipeline): IPipeline; +export function concat(...pipelines: AnyPipeline[]): AnyPipeline; +export function concat(...pipelines: AnyPipeline[]): AnyPipeline { + let consumer: (inputs: any) => void; + + return { + async initialize(next) { + for (let i = pipelines.length - 1; i >= 0; i--) { + const pipeline = pipelines[i]; + await pipeline.initialize(next); + next = inputs => pipeline.consume(inputs); + } + + consumer = next; + }, + consume: i => consumer(i), + async finalize() { + const out: any[] = []; + for (const p of pipelines) out.push(await p.finalize()); + return out; + }, + }; +} \ No newline at end of file diff --git a/nahara-motion-video/src/render.ts b/nahara-motion-video/src/render.ts new file mode 100644 index 0000000..1bf687d --- /dev/null +++ b/nahara-motion-video/src/render.ts @@ -0,0 +1,43 @@ +import { BaseCanvasRenderingContext2D, IScene } from "@nahara/motion"; +import { IPipeline } from "./pipeline.js"; + +export class SceneRenderPipeline implements IPipeline { + next?: (output: VideoFrame) => any; + canvas?: OffscreenCanvas; + ctx?: BaseCanvasRenderingContext2D; + + constructor( + public readonly width: number, + public readonly height: number, + public readonly frameRate: number, + public readonly frames: number + ) {} + + async initialize(next: (output: VideoFrame) => any): Promise { + this.next = next; + this.canvas = new OffscreenCanvas(this.width, this.height); + this.ctx = this.canvas.getContext("2d")!; + } + + consume(input: IScene): void { + for (let f = 0; f < this.frames; f++) { + const timestamp = Math.floor(f * 1000000 / this.frameRate); + const delta = Math.floor((f + 1) * 1000000 / this.frameRate) - timestamp; + + this.ctx!.reset(); + this.ctx!.fillStyle = "#000"; + this.ctx!.fillRect(0, 0, this.width, this.height); + input.renderFrame({ + canvas: this.ctx!, + containerSize: { x: this.width, y: this.height }, + time: timestamp / 1000, + timeDelta: delta / 1000 + }); + const frame = new VideoFrame(this.canvas!, { timestamp, duration: delta }); + this.next!(frame); + } + } + + async finalize(): Promise { + } +} \ No newline at end of file diff --git a/nahara-motion-video/tsconfig.json b/nahara-motion-video/tsconfig.json new file mode 100644 index 0000000..d7cf22c --- /dev/null +++ b/nahara-motion-video/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "target": "ESNext", + "outDir": "dist", + "incremental": true, + "strict": true, + "declaration": true, + "skipLibCheck": true + }, + "include": [ + "src" + ], + "exclude": [ + "node_modules" + ] +} \ No newline at end of file diff --git a/nahara-motion/package.json b/nahara-motion/package.json index ebc3d41..4bfd541 100644 --- a/nahara-motion/package.json +++ b/nahara-motion/package.json @@ -11,7 +11,7 @@ }, "keywords": [], "author": "", - "license": "ISC", + "license": "MIT", "devDependencies": { "@types/wicg-file-system-access": "^2023.10.5", "typescript": "^5.6.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27e96b7..0c08855 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@nahara/motion': specifier: workspace:* version: link:../nahara-motion + '@nahara/motion-video': + specifier: workspace:* + version: link:../nahara-motion-video devDependencies: '@sveltejs/vite-plugin-svelte': specifier: ^3.1.1 @@ -46,6 +49,19 @@ importers: specifier: ^5.4.6 version: 5.4.8(sass-embedded@1.78.0) + nahara-motion-video: + dependencies: + '@nahara/motion': + specifier: workspace:* + version: link:../nahara-motion + mp4-muxer: + specifier: ^5.1.3 + version: 5.1.3 + devDependencies: + typescript: + specifier: ^5.6.2 + version: 5.6.2 + packages: /@ampproject/remapping@2.3.0: @@ -465,6 +481,10 @@ packages: resolution: {integrity: sha512-BV9NplVgLmSi4mwKzD8BD/NQ8erOY/nUE/GpgWe2ckx+wIQF5RyRirn/QsSSCPeulVpc3RA/iJt6DpfTIZps0Q==} dev: true + /@types/dom-webcodecs@0.1.11: + resolution: {integrity: sha512-yPEZ3z7EohrmOxbk/QTAa0yonMFkNkjnVXqbGb7D4rMr+F1dGQ8ZUFxXkyLLJuiICPejZ0AZE9Rrk9wUCczx4A==} + dev: false + /@types/estree@1.0.5: resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} dev: true @@ -473,6 +493,10 @@ packages: resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==} dev: true + /@types/wicg-file-system-access@2020.9.8: + resolution: {integrity: sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==} + dev: false + /@types/wicg-file-system-access@2023.10.5: resolution: {integrity: sha512-e9kZO9kCdLqT2h9Tw38oGv9UNzBBWaR1MzuAavxPcsV/7FJ3tWbU6RI3uB+yKIDPGLkGVbplS52ub0AcRLvrhA==} dev: true @@ -767,6 +791,13 @@ packages: minimist: 1.2.8 dev: true + /mp4-muxer@5.1.3: + resolution: {integrity: sha512-artYS8Y7l9Cgqi1WyoNZtHW+pXOM0Tg75eupENoSEx59hMJlBqXxsTgJQ8T0ycrb6h9lw9iNKVB8cKEn7HjG2A==} + dependencies: + '@types/dom-webcodecs': 0.1.11 + '@types/wicg-file-system-access': 2020.9.8 + dev: false + /mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 95c915c..c4e89af 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,4 @@ packages: - 'nahara-motion' - 'nahara-motion-ui' +- 'nahara-motion-video'