From 0efb0742f98b98aaf2a0915b26eade3f2c22485b Mon Sep 17 00:00:00 2001
From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com>
Date: Thu, 6 Jun 2024 17:48:51 +0100
Subject: [PATCH] fix: Web recorder on tab change performance improvements

---
 .../windows/inner/callbackTemplate.ts         |   2 +-
 .../src/utils/auth/callback.template.ts       |   2 +-
 apps/web/app/api/video/metadata/route.ts      |  60 ++++++
 apps/web/app/record/Record.tsx                | 204 ++++++++++--------
 apps/web/utils/video/upload/helpers.ts        |   1 -
 packages/utils/src/helpers.ts                 |  38 ++--
 6 files changed, 196 insertions(+), 111 deletions(-)
 create mode 100644 apps/web/app/api/video/metadata/route.ts

diff --git a/apps/desktop/src/components/windows/inner/callbackTemplate.ts b/apps/desktop/src/components/windows/inner/callbackTemplate.ts
index 0f1df435..0aefd895 100644
--- a/apps/desktop/src/components/windows/inner/callbackTemplate.ts
+++ b/apps/desktop/src/components/windows/inner/callbackTemplate.ts
@@ -8,7 +8,7 @@ export default `
   <link rel="preconnect" href="https://fonts.googleapis.com">
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet">
-  <title>My Notes</title>
+  <title>Cap Auth</title>
   <style>
     body {
       font-family: 'Open Sans', sans-serif;
diff --git a/apps/desktop/src/utils/auth/callback.template.ts b/apps/desktop/src/utils/auth/callback.template.ts
index d9d884d2..fd51ff59 100644
--- a/apps/desktop/src/utils/auth/callback.template.ts
+++ b/apps/desktop/src/utils/auth/callback.template.ts
@@ -8,7 +8,7 @@ export default `
   <link rel="preconnect" href="https://fonts.googleapis.com">
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
   <link href="https://fonts.googleapis.com/css2?family=Open+Sans&display=swap" rel="stylesheet">
-  <title>My Notes</title>
+  <title>Cap Auth</title>
   <style>
     body {
       width: 100%;
diff --git a/apps/web/app/api/video/metadata/route.ts b/apps/web/app/api/video/metadata/route.ts
new file mode 100644
index 00000000..1c70b516
--- /dev/null
+++ b/apps/web/app/api/video/metadata/route.ts
@@ -0,0 +1,60 @@
+import { type NextRequest } from "next/server";
+import { getCurrentUser } from "@cap/database/auth/session";
+import { videos } from "@cap/database/schema";
+import { db } from "@cap/database";
+import { eq } from "drizzle-orm";
+
+export async function PUT(request: NextRequest) {
+  const user = await getCurrentUser();
+  const { videoId, metadata } = await request.json();
+  const userId = user?.id as string;
+
+  if (!user || !videoId || !metadata) {
+    console.error("Missing required data in /api/video/metadata/route.ts");
+
+    return new Response(JSON.stringify({ error: true }), {
+      status: 401,
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+  }
+
+  const query = await db.select().from(videos).where(eq(videos.id, videoId));
+
+  if (query.length === 0) {
+    return new Response(JSON.stringify({ error: true }), {
+      status: 401,
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+  }
+
+  const ownerId = query[0].ownerId;
+
+  if (ownerId !== userId) {
+    return new Response(JSON.stringify({ error: true }), {
+      status: 401,
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+  }
+
+  await db
+    .update(videos)
+    .set({
+      metadata: metadata,
+    })
+    .where(eq(videos.id, videoId));
+
+  return new Response(
+    JSON.stringify({
+      status: 200,
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+  );
+}
diff --git a/apps/web/app/record/Record.tsx b/apps/web/app/record/Record.tsx
index 643ce8b7..f2656cf1 100644
--- a/apps/web/app/record/Record.tsx
+++ b/apps/web/app/record/Record.tsx
@@ -34,11 +34,7 @@ import { ActionButton } from "./_components/ActionButton";
 import toast from "react-hot-toast";
 import { FFmpeg } from "@ffmpeg/ffmpeg";
 import { fetchFile } from "@ffmpeg/util";
-import {
-  getLatestVideoId,
-  saveLatestVideoId,
-  getVideoDuration,
-} from "@cap/utils";
+import { getLatestVideoId, saveLatestVideoId } from "@cap/utils";
 import { isUserOnProPlan } from "@cap/utils";
 import { LogoBadge } from "@cap/ui";
 
@@ -395,98 +391,92 @@ export const Record = ({
 
     const combinedStream = new MediaStream();
     const canvas = document.createElement("canvas");
-    const ctx = canvas.getContext("2d");
+    const ctx = canvas.getContext("2d", { willReadFrequently: true });
+
+    let screenX: number,
+      screenY: number,
+      screenWidth: number,
+      screenHeight: number;
+    let webcamX: number,
+      webcamY: number,
+      webcamWidth: number,
+      webcamHeight: number;
+    let newWebcamWidth: number,
+      newWebcamHeight: number,
+      offsetX: number,
+      offsetY: number;
+    let intervalId: NodeJS.Timeout | null = null;
 
-    // Get the video container element
     const videoContainer = document.querySelector(
       ".video-container"
     ) as HTMLElement;
+    const screenPreview = document.getElementById(
+      "screenPreview"
+    ) as HTMLVideoElement;
+    const webcamPreview = document.getElementById(
+      "webcamPreview"
+    ) as HTMLVideoElement;
+
+    // Set canvas dimensions to video container dimensions
+    canvas.width = videoContainer.clientWidth;
+    canvas.height = videoContainer.clientHeight;
+
+    // Calculate coordinates and dimensions once
+    if (screenPreview && videoContainer) {
+      const screenRect = screenPreview.getBoundingClientRect();
+      const containerRect = videoContainer.getBoundingClientRect();
 
-    // Set the minimum width to 1000 pixels
-    const minWidth = 1000;
-
-    // Calculate the aspect ratio of the video container
-    const aspectRatio =
-      videoContainer.clientWidth / videoContainer.clientHeight;
-
-    // Set the canvas width to the minimum width or the video container width, whichever is larger
-    canvas.width = Math.max(minWidth, videoContainer.clientWidth);
+      screenX =
+        (screenRect.left - containerRect.left) *
+        (canvas.width / containerRect.width);
+      screenY =
+        (screenRect.top - containerRect.top) *
+        (canvas.height / containerRect.height);
+      screenWidth = screenRect.width * (canvas.width / containerRect.width);
+      screenHeight = screenRect.height * (canvas.height / containerRect.height);
+    }
 
-    // Calculate the canvas height based on the width and aspect ratio
-    canvas.height = canvas.width / aspectRatio;
+    if (
+      webcamPreview &&
+      videoContainer &&
+      selectedVideoDeviceLabel !== "None"
+    ) {
+      const webcamRect = webcamPreview.getBoundingClientRect();
+      const containerRect = videoContainer.getBoundingClientRect();
 
-    let animationFrameId: number;
+      webcamX =
+        (webcamRect.left - containerRect.left) *
+        (canvas.width / containerRect.width);
+      webcamY =
+        (webcamRect.top - containerRect.top) *
+        (canvas.height / containerRect.height);
+      webcamWidth = webcamRect.width * (canvas.width / containerRect.width);
+      webcamHeight = webcamRect.height * (canvas.height / containerRect.height);
+
+      const videoAspectRatio =
+        webcamPreview.videoWidth / webcamPreview.videoHeight;
+      newWebcamWidth = webcamWidth * 2;
+      newWebcamHeight = newWebcamWidth / videoAspectRatio;
+      offsetX = (newWebcamWidth - webcamWidth) / 2;
+      offsetY = (newWebcamHeight - webcamHeight) / 2;
+    }
 
     const drawCanvas = () => {
-      if (ctx) {
+      if (ctx && screenPreview && videoContainer) {
         ctx.clearRect(0, 0, canvas.width, canvas.height);
-
-        const screenPreview = document.getElementById(
-          "screenPreview"
-        ) as HTMLVideoElement;
-        const webcamPreview = document.getElementById(
-          "webcamPreview"
-        ) as HTMLVideoElement;
-        const videoContainer = document.querySelector(
-          ".video-container"
-        ) as HTMLElement;
-
-        if (screenPreview && videoContainer) {
-          const screenRect = screenPreview.getBoundingClientRect();
-          const containerRect = videoContainer.getBoundingClientRect();
-
-          const screenX =
-            (screenRect.left - containerRect.left) *
-            (canvas.width / containerRect.width);
-          const screenY =
-            (screenRect.top - containerRect.top) *
-            (canvas.height / containerRect.height);
-          const screenWidth =
-            screenRect.width * (canvas.width / containerRect.width);
-          const screenHeight =
-            screenRect.height * (canvas.height / containerRect.height);
-
-          ctx.drawImage(
-            screenPreview,
-            screenX,
-            screenY,
-            screenWidth,
-            screenHeight
-          );
-        }
+        ctx.drawImage(
+          screenPreview,
+          screenX,
+          screenY,
+          screenWidth,
+          screenHeight
+        );
 
         if (
           webcamPreview &&
           videoContainer &&
           selectedVideoDeviceLabel !== "None"
         ) {
-          const webcamRect = webcamPreview.getBoundingClientRect();
-          const containerRect = videoContainer.getBoundingClientRect();
-
-          const webcamX =
-            (webcamRect.left - containerRect.left) *
-            (canvas.width / containerRect.width);
-          const webcamY =
-            (webcamRect.top - containerRect.top) *
-            (canvas.height / containerRect.height);
-          const webcamWidth =
-            webcamRect.width * (canvas.width / containerRect.width);
-          const webcamHeight =
-            webcamRect.height * (canvas.height / containerRect.height);
-
-          // Calculate the aspect ratio of the webcam video
-          const videoAspectRatio =
-            webcamPreview.videoWidth / webcamPreview.videoHeight;
-
-          // Calculate the new dimensions to maintain the aspect ratio while fitting within the webcam preview
-          let newWebcamWidth = webcamWidth * 2;
-          let newWebcamHeight = newWebcamWidth / videoAspectRatio;
-
-          // Calculate the offset to center the webcam video
-          const offsetX = (newWebcamWidth - webcamWidth) / 2;
-          const offsetY = (newWebcamHeight - webcamHeight) / 2;
-
-          // Create a circular clip path for the webcam preview
           ctx.save();
           ctx.beginPath();
           ctx.arc(
@@ -497,8 +487,6 @@ export const Record = ({
             2 * Math.PI
           );
           ctx.clip();
-
-          // Draw the webcam preview on the canvas with the zoom adjustment
           ctx.drawImage(
             webcamPreview,
             webcamX - offsetX,
@@ -506,15 +494,41 @@ export const Record = ({
             newWebcamWidth,
             newWebcamHeight
           );
-
           ctx.restore();
         }
-
-        animationFrameId = requestAnimationFrame(drawCanvas);
       }
     };
 
-    drawCanvas();
+    const startDrawing = () => {
+      const drawLoop = () => {
+        drawCanvas();
+        animationFrameId = requestAnimationFrame(drawLoop);
+      };
+      drawLoop();
+      return animationFrameId;
+    };
+
+    const intervalDrawing = () => setInterval(drawCanvas, 1000 / 30); // 30 fps
+
+    // Start drawing on canvas using requestAnimationFrame
+    let animationFrameId: number = 0;
+    animationFrameId = startDrawing();
+    let animationIntervalId: number | NodeJS.Timeout | null = null;
+
+    // Fallback to setInterval if the tab is not active
+    document.addEventListener("visibilitychange", () => {
+      if (document.hidden) {
+        cancelAnimationFrame(animationFrameId);
+        animationIntervalId = intervalDrawing();
+      } else {
+        if (animationIntervalId) {
+          clearInterval(animationIntervalId);
+          animationIntervalId = null;
+        }
+        animationFrameId = startDrawing();
+      }
+    });
+
     const canvasStream = canvas.captureStream(30);
     if (canvasStream.getVideoTracks().length === 0) {
       console.error("Canvas stream has no video tracks");
@@ -1205,6 +1219,22 @@ export const Record = ({
     }
   }, [audioDevicesRef, videoDevicesRef]);
 
+  // useEffect(() => {
+  //   const audio = new Audio("/sample-9s.mp3");
+  //   audio.loop = true;
+  //   audio.volume = 1;
+
+  //   const playAudio = () => {
+  //     audio.play();
+  //     window.removeEventListener("mousemove", playAudio);
+  //   };
+
+  //   window.addEventListener("mousemove", playAudio);
+
+  //   return () => {
+  //     window.removeEventListener("mousemove", playAudio);
+  //   };
+  // }, []);
   if (isLoading) {
     return (
       <div className="w-full h-screen flex items-center justify-center">
diff --git a/apps/web/utils/video/upload/helpers.ts b/apps/web/utils/video/upload/helpers.ts
index 0fcfc376..bcb90c01 100644
--- a/apps/web/utils/video/upload/helpers.ts
+++ b/apps/web/utils/video/upload/helpers.ts
@@ -47,7 +47,6 @@ export async function uploadToS3({
   });
   formData.append("file", blobData);
 
-  // Execute the upload using the presigned URL
   const uploadResponse = await fetch(presignedPostData.url, {
     method: "POST",
     body: formData,
diff --git a/packages/utils/src/helpers.ts b/packages/utils/src/helpers.ts
index 8930d69c..c4e53a52 100644
--- a/packages/utils/src/helpers.ts
+++ b/packages/utils/src/helpers.ts
@@ -60,28 +60,24 @@ export const isUserPro = async () => {
   return false;
 };
 
-function createVid(url: string) {
-  const vid = document.createElement("video");
-  vid.src = url;
-  vid.controls = false;
-  vid.muted = true;
-  vid.autoplay = false;
-  return vid;
-}
+export async function getBlobDuration(blob: Blob) {
+  const tempVideoEl = document.createElement("video") as HTMLVideoElement;
 
-export function getVideoDuration(blob: Blob) {
-  return new Promise((res, rej) => {
-    const url = URL.createObjectURL(blob);
-    const vid = createVid(url);
-    vid.addEventListener("timeupdate", (_evt) => {
-      res(vid.duration);
-      vid.src = "";
-      URL.revokeObjectURL(url);
+  const durationP = new Promise((resolve, reject) => {
+    tempVideoEl.addEventListener("loadedmetadata", () => {
+      if (tempVideoEl.duration === Infinity) {
+        tempVideoEl.currentTime = Number.MAX_SAFE_INTEGER;
+        tempVideoEl.ontimeupdate = () => {
+          tempVideoEl.ontimeupdate = null;
+          resolve(tempVideoEl.duration);
+          tempVideoEl.currentTime = 0;
+        };
+      } else resolve(tempVideoEl.duration);
     });
-    vid.onerror = (evt) => {
-      rej(evt);
-      URL.revokeObjectURL(url);
-    };
-    vid.currentTime = 1e101;
+    tempVideoEl.onerror = () => resolve(0);
   });
+
+  tempVideoEl.src = URL.createObjectURL(blob);
+
+  return durationP;
 }