diff --git a/apps/desktop/src-tauri/src/media.rs b/apps/desktop/src-tauri/src/media.rs index 42bf264e..3b55744e 100644 --- a/apps/desktop/src-tauri/src/media.rs +++ b/apps/desktop/src-tauri/src/media.rs @@ -411,7 +411,7 @@ impl MediaRecorder { println!("Starting audio recording and processing..."); let audio_output_chunk_pattern = format!("{}/audio_recording_%03d.aac", audio_file_path_owned); let audio_segment_list_filename = format!("{}/segment_list.txt", audio_file_path_owned); - let video_output_chunk_pattern = format!("{}/video_recording_%03d.mp4", video_file_path_owned); + let video_output_chunk_pattern = format!("{}/video_recording_%03d.ts", video_file_path_owned); let video_segment_list_filename = format!("{}/segment_list.txt", video_file_path_owned); let mut audio_filters = Vec::new(); @@ -458,7 +458,7 @@ impl MediaRecorder { "-segment_time", "3", "-segment_time_delta", "0.01", "-segment_list", &video_segment_list_filename, - "-segment_format", "mp4", + "-segment_format", "ts", "-movflags", "frag_keyframe+empty_moov", "-reset_timestamps", "1", &video_output_chunk_pattern, diff --git a/apps/desktop/src-tauri/src/upload.rs b/apps/desktop/src-tauri/src/upload.rs index ee2ea963..2c339f0f 100644 --- a/apps/desktop/src-tauri/src/upload.rs +++ b/apps/desktop/src-tauri/src/upload.rs @@ -2,9 +2,14 @@ use reqwest; use std::fs::File; use std::io::Read; use std::path::Path; +use std::process::{Command, Output}; +use std::str; +use std::fs; +use regex::Regex; use serde_json::Value as JsonValue; use crate::recording::RecordingOptions; +use crate::utils::{ffmpeg_path_as_str}; pub async fn upload_file( options: Option, @@ -14,6 +19,9 @@ pub async fn upload_file( if let Some(ref options) = options { println!("Uploading video..."); + let duration = get_video_duration(&file_path).map_err(|e| format!("Failed to get video duration: {}", e))?; + let duration_str = duration.to_string(); + let file_name = Path::new(&file_path) .file_name() .and_then(|name| name.to_str()) @@ -24,14 +32,33 @@ pub async fn upload_file( let server_url_base: &'static str = dotenv_codegen::dotenv!("NEXT_PUBLIC_URL"); let server_url = format!("{}/api/upload/signed", server_url_base); + + let body: serde_json::Value; + + if file_type == "video" { + let (codec_name, width, height, frame_rate, bit_rate) = log_video_info(&file_path).map_err(|e| format!("Failed to log video info: {}", e))?; + + body = serde_json::json!({ + "userId": options.user_id, + "fileKey": file_key, + "awsBucket": options.aws_bucket, + "awsRegion": options.aws_region, + "duration": duration_str, + "resolution": format!("{}x{}", width, height), + "framerate": frame_rate, + "bandwidth": bit_rate, + "videoCodec": codec_name, + }); + } else { - // Create the request body for the Next.js handler - let body = serde_json::json!({ - "userId": options.user_id, - "fileKey": file_key, - "awsBucket": options.aws_bucket, - "awsRegion": options.aws_region, - }); + body = serde_json::json!({ + "userId": options.user_id, + "fileKey": file_key, + "awsBucket": options.aws_bucket, + "awsRegion": options.aws_region, + "duration": duration_str, + }); + } let client = reqwest::Client::new(); let server_response = client.post(server_url) @@ -117,4 +144,60 @@ pub async fn upload_file( } else { return Err("No recording options provided".to_string()); } +} + +pub fn get_video_duration(file_path: &str) -> Result { + let ffmpeg_binary_path_str = ffmpeg_path_as_str().unwrap().to_owned(); + + let output = Command::new(ffmpeg_binary_path_str) + .arg("-i") + .arg(file_path) + .output()?; + + let output_str = str::from_utf8(&output.stderr).unwrap(); + let duration_regex = Regex::new(r"Duration: (\d{2}):(\d{2}):(\d{2})\.\d{2}").unwrap(); + let caps = duration_regex.captures(output_str).unwrap(); + + let hours: f64 = caps.get(1).unwrap().as_str().parse().unwrap(); + let minutes: f64 = caps.get(2).unwrap().as_str().parse().unwrap(); + let seconds: f64 = caps.get(3).unwrap().as_str().parse().unwrap(); + + let duration = hours * 3600.0 + minutes * 60.0 + seconds; + + Ok(duration) +} + +fn log_video_info(file_path: &str) -> Result<(String, String, String, String, String), String> { + let output: Output = Command::new("ffprobe") + .arg("-v") + .arg("error") + .arg("-show_entries") + .arg("stream=bit_rate,codec_name,height,width,r_frame_rate") + .arg("-of") + .arg("default=noprint_wrappers=1:nokey=1") + .arg(file_path) + .output() + .map_err(|e| format!("Failed to run ffprobe: {}", e))?; + + if !output.status.success() { + return Err(format!( + "ffprobe exited with non-zero status: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + + let info = String::from_utf8_lossy(&output.stdout); + let info_parts: Vec<&str> = info.split('\n').collect(); + let codec_name = info_parts[0].to_string(); + let width: String = info_parts[1].to_string(); + let height: String = info_parts[2].to_string(); + + // Parse frame rate as a fraction and convert to float + let frame_rate_parts: Vec<&str> = info_parts[3].split('/').collect(); + let frame_rate: f64 = frame_rate_parts[0].parse::().unwrap() / frame_rate_parts[1].parse::().unwrap(); + let frame_rate: String = frame_rate.to_string(); + + let bit_rate: String = info_parts[4].to_string(); + + Ok((codec_name, width, height, frame_rate, bit_rate)) } \ No newline at end of file diff --git a/apps/web/app/api/playlist/route.ts b/apps/web/app/api/playlist/route.ts index dae50b06..84069430 100644 --- a/apps/web/app/api/playlist/route.ts +++ b/apps/web/app/api/playlist/route.ts @@ -153,46 +153,61 @@ export async function GET(request: NextRequest) { MaxKeys: 1, }); + let audioSegment; const audioSegmentCommand = new ListObjectsV2Command({ Bucket: bucket, Prefix: audioPrefix, MaxKeys: 1, }); - const [videoSegment, audioSegment] = await Promise.all([ + try { + audioSegment = await s3Client.send(audioSegmentCommand); + } catch (error) { + console.warn("No audio segment found for this video", error); + } + + console.log("audioSegment", audioSegment); + + const [videoSegment] = await Promise.all([ s3Client.send(videoSegmentCommand), - s3Client.send(audioSegmentCommand), ]); - const [videoMetadata, audioMetadata] = await Promise.all([ + let audioMetadata; + const [videoMetadata] = await Promise.all([ s3Client.send( new HeadObjectCommand({ Bucket: bucket, Key: videoSegment.Contents?.[0]?.Key ?? "", }) ), - s3Client.send( + ]); + + if (audioSegment?.KeyCount && audioSegment?.KeyCount > 0) { + audioMetadata = await s3Client.send( new HeadObjectCommand({ Bucket: bucket, Key: audioSegment.Contents?.[0]?.Key ?? "", }) - ), - ]); + ); + } const generatedPlaylist = await generateMasterPlaylist( videoMetadata?.Metadata?.resolution ?? "", + videoMetadata?.Metadata?.bandwidth ?? "", process.env.NEXT_PUBLIC_URL + "/api/playlist?userId=" + userId + "&videoId=" + videoId + "&videoType=video", - process.env.NEXT_PUBLIC_URL + - "/api/playlist?userId=" + - userId + - "&videoId=" + - videoId + - "&videoType=audio", + audioMetadata + ? process.env.NEXT_PUBLIC_URL + + "/api/playlist?userId=" + + userId + + "&videoId=" + + videoId + + "&videoType=audio" + : null, video.xStreamInfo ?? "" ); diff --git a/apps/web/app/record/Record.tsx b/apps/web/app/record/Record.tsx index e04cd535..643ce8b7 100644 --- a/apps/web/app/record/Record.tsx +++ b/apps/web/app/record/Record.tsx @@ -34,7 +34,11 @@ import { ActionButton } from "./_components/ActionButton"; import toast from "react-hot-toast"; import { FFmpeg } from "@ffmpeg/ffmpeg"; import { fetchFile } from "@ffmpeg/util"; -import { getLatestVideoId, saveLatestVideoId } from "@cap/utils"; +import { + getLatestVideoId, + saveLatestVideoId, + getVideoDuration, +} from "@cap/utils"; import { isUserOnProPlan } from "@cap/utils"; import { LogoBadge } from "@cap/ui"; @@ -70,7 +74,7 @@ class AsyncTaskQueue { this.resolveEmptyPromise(); this.resolveEmptyPromise = null; } - this.processQueue(); // Recursively call processQueue to handle the next task + this.processQueue(); } } } @@ -134,7 +138,6 @@ export const Record = ({ const [isCenteredHorizontally, setIsCenteredHorizontally] = useState(false); const [isCenteredVertically, setIsCenteredVertically] = useState(false); - const recordingIntervalRef = useRef(null); const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); const defaultRndHandeStyles = { @@ -389,7 +392,6 @@ export const Record = ({ saveLatestVideoId(videoCreateData.id); setStartingRecording(true); - recordingIntervalRef.current = null; const combinedStream = new MediaStream(); const canvas = document.createElement("canvas"); @@ -567,36 +569,45 @@ export const Record = ({ videoRecorderOptions ); - function recordVideoChunk() { - if (recordingIntervalRef.current === "stop") { - return; - } + let chunks = []; + let segmentStartTime = Date.now(); + const recordingStartTime = Date.now(); - videoRecorder.start(); + videoRecorder.ondataavailable = async (event) => { + if (event.data.size > 0) { + chunks.push(event.data); + const segmentEndTime = Date.now(); + const segmentDuration = (segmentEndTime - segmentStartTime) / 1000.0; - recordingIntervalRef.current = setInterval(() => { - if (videoRecorder.state === "recording") videoRecorder.stop(); + const videoDuration = (Date.now() - recordingStartTime) / 1000.0; - if (videoRecorder) recordVideoChunk(); - }, 5000); - } + console.log("Video duration:", videoDuration); + console.log("Segment duration:", segmentDuration); + console.log("Start:", Math.max(videoDuration - segmentDuration, 0)); - videoRecorder.ondataavailable = async (event) => { - if (event.data.size > 0) { muxQueue.enqueue(async () => { - await muxSegment({ - data: event.data, - mimeType: videoRecorderOptions.mimeType, - final: recordingIntervalRef.current === "stop" ? true : false, - }); + try { + await muxSegment({ + data: chunks, + mimeType: videoRecorderOptions.mimeType, + final: videoRecorder.state !== "recording", + start: Math.max(videoDuration - segmentDuration, 0), + end: videoDuration, + segmentTime: segmentDuration, + }); + } catch (error) { + console.error("Error in muxSegment:", error); + readyToStopRecording.current = true; + } }); + + segmentStartTime = segmentEndTime; } }; - recordVideoChunk(); + videoRecorder.start(3000); setVideoRecorder(videoRecorder); - setIsRecording(true); setStartingRecording(false); }; @@ -605,36 +616,67 @@ export const Record = ({ data, mimeType, final, + start, + end, + segmentTime, }: { - data: Blob; + data: Blob[]; mimeType: string; final: boolean; + start: number; + end: number; + segmentTime: number; }) => { return new Promise(async (resolve, reject) => { console.log("Muxing segment"); const segmentIndex = totalSegments.current; - const videoSegment = new Blob([data], { type: "video/webm" }); - let audioData; + const segmentIndexString = String(segmentIndex).padStart(3, "0"); + const videoSegment = new Blob(data, { type: "video/webm" }); + const segmentPaths = { + tempInput: `temp_segment_${segmentIndexString}${ + mimeType.includes("mp4") ? ".mp4" : ".webm" + }`, + videoInput: `input_segment_${segmentIndexString}.ts`, + videoOutput: `video_segment_${segmentIndexString}.ts`, + audioOutput: `audio_segment_${segmentIndexString}.aac`, + }; if (videoSegment) { - const segmentIndexString = String(segmentIndex).padStart(3, "0"); - const videoFile = await fetchFile(URL.createObjectURL(videoSegment)); - ffmpegRef.current.writeFile( - `video_segment_${segmentIndexString}${ - mimeType.includes("mp4") ? ".mp4" : ".webm" - }`, - videoFile - ); - - const segmentPaths = { - videoInput: `video_segment_${segmentIndexString}${ - mimeType.includes("mp4") ? ".mp4" : ".webm" - }`, - videoOutput: `video_segment_${segmentIndexString}.mp4`, - audioOutput: `audio_segment_${segmentIndexString}.aac`, - }; + + await ffmpegRef.current.writeFile(segmentPaths.tempInput, videoFile); + + const tempVideoCommand = [ + "-ss", + start.toFixed(2), + "-to", + end.toFixed(2), + "-i", + segmentPaths.tempInput, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "0", + "-pix_fmt", + "yuv420p", + "-r", + "30", + "-c:a", + "aac", + "-f", + "hls", + segmentPaths.videoInput, + ]; + + try { + await ffmpegRef.current.exec(tempVideoCommand); + } catch (error) { + console.error("Error executing tempVideoCommand with FFmpeg:", error); + return reject(error); + } const videoFFmpegCommand = [ "-i", @@ -649,14 +691,21 @@ export const Record = ({ "yuv420p", "-r", "30", - "-f", - "mp4", segmentPaths.videoOutput, ]; - await ffmpegRef.current.exec(videoFFmpegCommand); + try { + await ffmpegRef.current.exec(videoFFmpegCommand); + } catch (error) { + console.error( + "Error executing videoFFmpegCommand with FFmpeg:", + error + ); + return reject(error); + } if (videoStream && selectedAudioDeviceLabel !== "None") { + console.log("Muxing audio"); const audioFFmpegCommand = [ "-i", segmentPaths.videoInput, @@ -670,21 +719,56 @@ export const Record = ({ "aac_low", segmentPaths.audioOutput, ]; - await ffmpegRef.current.exec(audioFFmpegCommand); + try { + await ffmpegRef.current.exec(audioFFmpegCommand); + console.log( + "hereaudioexecuted: ", + await ffmpegRef.current.listDir("/") + ); + } catch (error) { + console.error( + "Error executing audioFFmpegCommand with FFmpeg:", + error + ); + console.log( + "audio error here: ", + await ffmpegRef.current.listDir("/") + ); + return reject(error); + } } - const videoData = await ffmpegRef.current.readFile( - segmentPaths.videoOutput - ); + let videoData; + try { + videoData = await ffmpegRef.current.readFile( + segmentPaths.videoOutput + ); + console.log("file list: ", await ffmpegRef.current.listDir("/")); + } catch (error) { + console.error("Error reading video file with FFmpeg:", error); + return reject(error); + } + let audioData; if (videoStream && selectedAudioDeviceLabel !== "None") { - audioData = await ffmpegRef.current.readFile( - segmentPaths.audioOutput - ); + try { + audioData = await ffmpegRef.current.readFile( + segmentPaths.audioOutput + ); + + console.log("Found audio data:", audioData); + } catch (error) { + console.error("Error reading audio file with FFmpeg:", error); + console.log( + "audio error here: ", + await ffmpegRef.current.listDir("/") + ); + return reject(error); + } } const segmentFilenames = { - video: `video/video_recording_${segmentIndexString}.mp4`, + video: `video/video_recording_${segmentIndexString}.ts`, audio: `audio/audio_recording_${segmentIndexString}.aac`, }; @@ -734,6 +818,7 @@ export const Record = ({ file: videoData, filename: segmentFilenames.video, videoId, + duration: segmentTime.toFixed(1), }); if (videoStream && selectedAudioDeviceLabel !== "None" && audioData) { @@ -741,6 +826,7 @@ export const Record = ({ file: audioData, filename: segmentFilenames.audio, videoId, + duration: segmentTime.toFixed(1), }); } } catch (error) { @@ -748,11 +834,39 @@ export const Record = ({ reject(error); } - await ffmpegRef.current.deleteFile(segmentPaths.videoInput); - await ffmpegRef.current.deleteFile(segmentPaths.videoOutput); + console.log("herelast: ", await ffmpegRef.current.listDir("/")); - if (videoStream && selectedAudioDeviceLabel !== "None" && audioData) { - await ffmpegRef.current.deleteFile(segmentPaths.audioOutput); + // tempInput: `temp_segment_${segmentIndexString}${ + // mimeType.includes("mp4") ? ".mp4" : ".webm" + // }`, + // videoInput: `input_segment_${segmentIndexString}.ts`, + // videoOutput: `video_segment_${segmentIndexString}.ts`, + // audioOutput: `audio_segment_${segmentIndexString}.aac`, + + try { + await ffmpegRef.current.deleteFile(segmentPaths.tempInput); + } catch (error) { + console.error("Error deleting temp input file:", error); + } + + try { + await ffmpegRef.current.deleteFile(segmentPaths.videoInput); + } catch (error) { + console.error("Error deleting video input file:", error); + } + + try { + await ffmpegRef.current.deleteFile(segmentPaths.videoOutput); + } catch (error) { + console.error("Error deleting video output file:", error); + } + + if (audioData) { + try { + await ffmpegRef.current.deleteFile(segmentPaths.audioOutput); + } catch (error) { + console.error("Error deleting audio output file:", error); + } } if (final) { @@ -776,15 +890,31 @@ export const Record = ({ file, filename, videoId, + duration, }: { file: Uint8Array | string; filename: string; videoId: string; + duration?: string; }) => { const formData = new FormData(); formData.append("filename", filename); formData.append("videoId", videoId); - formData.append("blobData", new Blob([file], { type: "video/mp2t" })); + let mimeType; + if (filename.endsWith(".aac")) { + mimeType = "audio/aac"; + } else if (filename.endsWith(".jpg")) { + mimeType = "image/jpeg"; + } else { + mimeType = "video/mp2t"; + } + formData.append("blobData", new Blob([file], { type: mimeType })); + if (duration) { + formData.append("duration", String(duration)); + } + if (filename.includes("video")) { + formData.append("framerate", "30"); + } await fetch(`${process.env.NEXT_PUBLIC_URL}/api/upload/new`, { method: "POST", @@ -797,12 +927,6 @@ export const Record = ({ console.log("---Stopping recording function fired here---"); - if (recordingIntervalRef.current) { - clearInterval(recordingIntervalRef.current); - recordingIntervalRef.current = "stop"; - console.log("Recording interval stopped"); - } - if (videoRecorder) { videoRecorder.stop(); console.log("Video recorder stopped"); diff --git a/apps/web/app/s/[videoId]/_components/ShareVideo.tsx b/apps/web/app/s/[videoId]/_components/ShareVideo.tsx index 603569f4..ece53755 100644 --- a/apps/web/app/s/[videoId]/_components/ShareVideo.tsx +++ b/apps/web/app/s/[videoId]/_components/ShareVideo.tsx @@ -263,9 +263,6 @@ export const ShareVideo = ({ {isLoading && (
- {data.jobStatus !== "COMPLETE" && data.skipProcessing === false && ( -

Video is processing...

- )}
)} {isLoading === false && ( @@ -296,7 +293,7 @@ export const ShareVideo = ({ { return false; }; + +function createVid(url: string) { + const vid = document.createElement("video"); + vid.src = url; + vid.controls = false; + vid.muted = true; + vid.autoplay = false; + return vid; +} + +export function getVideoDuration(blob: Blob) { + return new Promise((res, rej) => { + const url = URL.createObjectURL(blob); + const vid = createVid(url); + vid.addEventListener("timeupdate", (_evt) => { + res(vid.duration); + vid.src = ""; + URL.revokeObjectURL(url); + }); + vid.onerror = (evt) => { + rej(evt); + URL.revokeObjectURL(url); + }; + vid.currentTime = 1e101; + }); +}