Skip to content

Commit

Permalink
Video rendering and encoding
Browse files Browse the repository at this point in the history
  • Loading branch information
nahkd123 committed Oct 2, 2024
1 parent ac0b1e3 commit 88cfa31
Show file tree
Hide file tree
Showing 14 changed files with 269 additions and 4 deletions.
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,9 @@ cd nahara-motion-ui
pnpm install
pnpm dev
# Open 127.0.0.1:5173 to see the app
```
```

## Modules
- `nahara-motion`: Nahara's Motion Engine
- `nahara-motion-ui`: The front-facing user interface for Nahara's Motion
- `nahara-motion-video`: Render and encode the scene to video file (`.mp4` in this case)
3 changes: 2 additions & 1 deletion nahara-motion-ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
"check": "svelte-check --tsconfig ./tsconfig.json && tsc -p tsconfig.node.json"
},
"dependencies": {
"@nahara/motion": "workspace:*"
"@nahara/motion": "workspace:*",
"@nahara/motion-video": "workspace:*"
},
"devDependencies": {
"@sveltejs/vite-plugin-svelte": "^3.1.1",
Expand Down
49 changes: 48 additions & 1 deletion nahara-motion-ui/src/ui/bar/TopBar.svelte
Original file line number Diff line number Diff line change
@@ -1,13 +1,60 @@
<script lang="ts">
import { concat, EncoderPipeline, MuxerPipeline, SceneRenderPipeline, type IPipeline } from "@nahara/motion-video";
import { app } from "../../appglobal";
import { openMenuAt } from "../menu/MenuHost.svelte";
import type { IScene } from "@nahara/motion";
const currentProject = app.currentProjectStore;
const currentScene = app.currentSceneStore;
function openFileMenu(e: MouseEvent) {
openMenuAt(e.clientX, e.clientY, [
{
type: "simple",
name: "Quick render",
async click() {
const frameRate = 60;
const pipeline: IPipeline<IScene, any, [any, any, ArrayBuffer]> = concat(
new SceneRenderPipeline(
$currentScene!.metadata.size.x,
$currentScene!.metadata.size.y,
frameRate, frameRate * 10
),
new EncoderPipeline({
codec: "avc1.4d0028",
width: $currentScene!.metadata.size.x,
height: $currentScene!.metadata.size.y,
bitrate: 10_000_000,
framerate: frameRate
}),
new MuxerPipeline({
codec: "avc",
width: $currentScene!.metadata.size.x,
height: $currentScene!.metadata.size.y,
frameRate: frameRate
})
);
await pipeline.initialize(console.log);
pipeline.consume($currentScene!);
const result = await pipeline.finalize();
const blob = new Blob([result[2]]);
const downloadLink = document.createElement("a");
downloadLink.href = URL.createObjectURL(blob);
downloadLink.download = "video.mp4";
downloadLink.click();
URL.revokeObjectURL(downloadLink.href);
},
}
]);
}
</script>

<div class="menu-bar">
<div class="left">
<div class="menu">File</div>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="menu" role="button" tabindex="0" on:click={openFileMenu}>File</div>
<div class="menu">Edit</div>
<div class="menu">Window</div>
<div class="menu">Help</div>
Expand Down
2 changes: 2 additions & 0 deletions nahara-motion-video/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dist
node_modules
22 changes: 22 additions & 0 deletions nahara-motion-video/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"name": "@nahara/motion-video",
"version": "1.0.0",
"description": "Nahara Motion Video",
"type": "module",
"main": "dist/index.js",
"scripts": {
"build": "tsc --project tsconfig.json",
"watch": "tsc --watch --project tsconfig.json",
"test": "echo \"Tests are not available for this package.\" && exit 0"
},
"keywords": [],
"author": "",
"license": "MIT",
"dependencies": {
"@nahara/motion": "workspace:*",
"mp4-muxer": "^5.1.3"
},
"devDependencies": {
"typescript": "^5.6.2"
}
}
28 changes: 28 additions & 0 deletions nahara-motion-video/src/encoder.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { IPipeline } from "./pipeline.js";

export class EncoderPipeline implements IPipeline<VideoFrame, [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined], any> {
next?: (output: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]) => any;
encoder?: VideoEncoder;

constructor(public readonly options: VideoEncoderConfig) {}

async initialize(next: (output: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]) => any): Promise<void> {
this.next = next;
this.encoder = new VideoEncoder({
output: (chunk, meta) => next([chunk, meta]),
error() {}
});
this.encoder.configure(this.options);
}

consume(input: VideoFrame): void {
this.encoder!.encode(input);
input.close();
}

async finalize(): Promise<any> {
await this.encoder!.flush();
console.log("EncoderPipeline: Flushed");
this.encoder!.close();
}
}
4 changes: 4 additions & 0 deletions nahara-motion-video/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
export * from "./encoder.js";
export * from "./muxer.js";
export * from "./pipeline.js";
export * from "./render.js";
26 changes: 26 additions & 0 deletions nahara-motion-video/src/muxer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { ArrayBufferTarget, Muxer, MuxerOptions } from "mp4-muxer";
import { IPipeline } from "./pipeline.js";

export class MuxerPipeline implements IPipeline<[EncodedVideoChunk, EncodedVideoChunkMetadata | undefined], void, ArrayBuffer> {
muxer?: Muxer<ArrayBufferTarget>;

constructor(public readonly options: (MuxerOptions<any>["video"] & {})) {}

async initialize(): Promise<void> {
this.muxer = new Muxer({
target: new ArrayBufferTarget(),
video: this.options,
fastStart: "fragmented"
});
}

consume(input: [EncodedVideoChunk, EncodedVideoChunkMetadata | undefined]): void {
this.muxer!.addVideoChunk(...input);
}

async finalize(): Promise<ArrayBuffer> {
console.log("MuxerPipeline: Finalizing...");
this.muxer!.finalize();
return this.muxer!.target.buffer;
}
}
38 changes: 38 additions & 0 deletions nahara-motion-video/src/pipeline.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/**
* Represent a pipeline in the system.
* @param I The input chunk type.
* @param O The output chunk type, which will be passed to next part of pipeline or collected by consumer.
* @param R The result of the pipeline, which will be collected by consumer (does not pass to other pipelines!). Mainly
* used for video file.
*/
export interface IPipeline<I, O, R> {
initialize(next: (output: O) => any): Promise<void>;
consume(input: I): void;
finalize(): Promise<R>;
}

export type AnyPipeline = IPipeline<any, any, any>;

export function concat<I, M, O, R1, R2>(a: IPipeline<I, M, R1>, b: IPipeline<M, O, R2>): IPipeline<I, O, [R1, R2]>;
export function concat(...pipelines: AnyPipeline[]): AnyPipeline;
export function concat(...pipelines: AnyPipeline[]): AnyPipeline {
let consumer: (inputs: any) => void;

return {
async initialize(next) {
for (let i = pipelines.length - 1; i >= 0; i--) {
const pipeline = pipelines[i];
await pipeline.initialize(next);
next = inputs => pipeline.consume(inputs);
}

consumer = next;
},
consume: i => consumer(i),
async finalize() {
const out: any[] = [];
for (const p of pipelines) out.push(await p.finalize());
return out;
},
};
}
43 changes: 43 additions & 0 deletions nahara-motion-video/src/render.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { BaseCanvasRenderingContext2D, IScene } from "@nahara/motion";
import { IPipeline } from "./pipeline.js";

export class SceneRenderPipeline implements IPipeline<IScene, VideoFrame, any> {
next?: (output: VideoFrame) => any;
canvas?: OffscreenCanvas;
ctx?: BaseCanvasRenderingContext2D;

constructor(
public readonly width: number,
public readonly height: number,
public readonly frameRate: number,
public readonly frames: number
) {}

async initialize(next: (output: VideoFrame) => any): Promise<void> {
this.next = next;
this.canvas = new OffscreenCanvas(this.width, this.height);
this.ctx = this.canvas.getContext("2d")!;
}

consume(input: IScene): void {
for (let f = 0; f < this.frames; f++) {
const timestamp = Math.floor(f * 1000000 / this.frameRate);
const delta = Math.floor((f + 1) * 1000000 / this.frameRate) - timestamp;

this.ctx!.reset();
this.ctx!.fillStyle = "#000";
this.ctx!.fillRect(0, 0, this.width, this.height);
input.renderFrame({
canvas: this.ctx!,
containerSize: { x: this.width, y: this.height },
time: timestamp / 1000,
timeDelta: delta / 1000
});
const frame = new VideoFrame(this.canvas!, { timestamp, duration: delta });
this.next!(frame);
}
}

async finalize(): Promise<any> {
}
}
17 changes: 17 additions & 0 deletions nahara-motion-video/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"compilerOptions": {
"module": "NodeNext",
"target": "ESNext",
"outDir": "dist",
"incremental": true,
"strict": true,
"declaration": true,
"skipLibCheck": true
},
"include": [
"src"
],
"exclude": [
"node_modules"
]
}
2 changes: 1 addition & 1 deletion nahara-motion/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
},
"keywords": [],
"author": "",
"license": "ISC",
"license": "MIT",
"devDependencies": {
"@types/wicg-file-system-access": "^2023.10.5",
"typescript": "^5.6.2"
Expand Down
31 changes: 31 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pnpm-workspace.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
packages:
- 'nahara-motion'
- 'nahara-motion-ui'
- 'nahara-motion-video'

0 comments on commit 88cfa31

Please sign in to comment.