From d3ffd3d33abe9851b347f0c982dedef13c310c43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lucas=20Gon=C3=A7alves=20Marchi?= <lucasgmarchi@gmail.com>
Date: Thu, 14 Nov 2024 09:45:03 -0300
Subject: [PATCH] fix(Call): Fix flash talk indicator (#825)

---
 package.json                                 |   1 +
 src/lib/components/Polling.svelte            |   3 +-
 src/lib/components/calling/CallScreen.svelte |   6 +-
 src/lib/media/Voice.ts                       | 163 ++++++++++++++-----
 src/lib/wasm/MultipassStore.ts               |   6 +-
 vite.config.js                               |   2 +
 6 files changed, 135 insertions(+), 46 deletions(-)

diff --git a/package.json b/package.json
index 66004040d..48f4c9c86 100644
--- a/package.json
+++ b/package.json
@@ -65,6 +65,7 @@
         "trystero": "^0.20.0",
         "uuid": "^9.0.1",
         "vite-plugin-node-polyfills": "^0.21.0",
+        "voice-activity-detection": "^0.0.5",
         "warp-wasm": "1.5.1"
     }
 }
diff --git a/src/lib/components/Polling.svelte b/src/lib/components/Polling.svelte
index f035d10df..3aed5d4d3 100644
--- a/src/lib/components/Polling.svelte
+++ b/src/lib/components/Polling.svelte
@@ -1,4 +1,5 @@
 <script lang="ts">
+    import { callScreenVisible } from "$lib/media/Voice"
     import { Store } from "$lib/state/Store"
     import { UIStore } from "$lib/state/ui"
     import { MultipassStoreInstance } from "$lib/wasm/MultipassStore"
@@ -12,7 +13,7 @@
     async function poll() {
         // add processes here.
         updateTypingIndicators()
-        await MultipassStoreInstance.fetchAllFriendsAndRequests()
+        await MultipassStoreInstance.fetchAllFriendsAndRequests(!get(callScreenVisible))
 
         // Increase the interval exponentially until it reaches the provided rate
         if (currentInterval < rate) {
diff --git a/src/lib/components/calling/CallScreen.svelte b/src/lib/components/calling/CallScreen.svelte
index 56270ba79..69b25f5d8 100644
--- a/src/lib/components/calling/CallScreen.svelte
+++ b/src/lib/components/calling/CallScreen.svelte
@@ -13,9 +13,9 @@
     import type { Chat } from "$lib/types"
     import VolumeMixer from "./VolumeMixer.svelte"
     import { createEventDispatcher, onDestroy, onMount } from "svelte"
-    import { callInProgress, callTimeout, makeCallSound, TIME_TO_SHOW_CONNECTING, TIME_TO_SHOW_END_CALL_FEEDBACK, timeCallStarted, usersAcceptedTheCall, usersDeniedTheCall, VoiceRTCInstance } from "$lib/media/Voice"
+    import { callInProgress, callScreenVisible, callTimeout, makeCallSound, TIME_TO_SHOW_CONNECTING, TIME_TO_SHOW_END_CALL_FEEDBACK, timeCallStarted, usersAcceptedTheCall, usersDeniedTheCall, VoiceRTCInstance } from "$lib/media/Voice"
     import { log } from "$lib/utils/Logger"
-    import { playSound, SoundHandler, Sounds } from "../utils/SoundHandler"
+    import { playSound, Sounds } from "../utils/SoundHandler"
     import { MultipassStoreInstance } from "$lib/wasm/MultipassStore"
     import { debounce } from "$lib/utils/Functions"
 
@@ -188,6 +188,7 @@
     }
 
     onMount(async () => {
+        callScreenVisible.set(true)
         if ($makeCallSound) {
             stopMakeCallSound()
         }
@@ -238,6 +239,7 @@
     })
 
     onDestroy(() => {
+        callScreenVisible.set(false)
         window.removeEventListener("keydown", handleKeyDown)
         window.removeEventListener("keyup", handleKeyUp)
         callTimeout.set(false)
diff --git a/src/lib/media/Voice.ts b/src/lib/media/Voice.ts
index e71308f7b..4cc78d183 100644
--- a/src/lib/media/Voice.ts
+++ b/src/lib/media/Voice.ts
@@ -5,12 +5,13 @@ import { create_cancellable_handler, type Cancellable } from "$lib/utils/Cancell
 import { log } from "$lib/utils/Logger"
 import { RaygunStoreInstance } from "$lib/wasm/RaygunStore"
 import Peer, { DataConnection } from "peerjs"
-import { _ } from "svelte-i18n"
+import { _, t } from "svelte-i18n"
 import { get, writable, type Writable } from "svelte/store"
 import type { Room } from "trystero"
 import { joinRoom } from "trystero"
 import { NoiseSuppressorWorklet_Name } from "@timephy/rnnoise-wasm"
 import NoiseSuppressorWorklet from "@timephy/rnnoise-wasm/NoiseSuppressorWorklet?worker&url"
+import vad from "voice-activity-detection"
 
 const CALL_ACK = "CALL_ACCEPT"
 
@@ -26,6 +27,21 @@ export const connectionOpened = writable(false)
 export const timeCallStarted: Writable<Date | null> = writable(null)
 export const callInProgress: Writable<string | null> = writable(null)
 export const makeCallSound = writable<SoundHandler | undefined>(undefined)
+export const callScreenVisible = writable(false)
+
+const relaysToTest = [
+    "wss://nostr-pub.wellorder.net",
+    "wss://brb.io",
+    "wss://relay.snort.social",
+    "wss://relay.damus.io",
+    "wss://nostr.mom",
+    "wss://relay.nostr.band",
+    "wss://nostr.oxtr.dev",
+    "wss://nostr.fmt.wiz.biz",
+    "wss://nostr-relay.digitalmob.ro",
+    "wss://nostr.openchain.fr",
+]
+const relaysAvailable: Writable<string[]> = writable(relaysToTest)
 
 export enum VoiceRTCMessageType {
     UpdateUser = "UPDATE_USER",
@@ -87,54 +103,67 @@ export type StreamMetaHandler = {
     remove(): void
 }
 
-function handleStreamMeta(did: string, stream: MediaStream): StreamMetaHandler {
+async function handleStreamMeta(did: string, stream: MediaStream): Promise<StreamMetaHandler> {
     const audioContext = new window.AudioContext()
     const analyser = audioContext.createAnalyser()
     analyser.fftSize = AUDIO_WINDOW_SIZE
     analyser.smoothingTimeConstant = 0.1
-    const dataArray = new Uint8Array(analyser.frequencyBinCount)
     let noiseSuppressionNode: AudioWorkletNode
-    let checker: NodeJS.Timeout
+    let voiceStopTimeout: NodeJS.Timeout | null = null
+    let speaking = false
 
-    audioContext.audioWorklet.addModule(NoiseSuppressorWorklet).then(() => {
-        noiseSuppressionNode = new AudioWorkletNode(audioContext, NoiseSuppressorWorklet_Name)
-        const mediaStreamSource = audioContext.createMediaStreamSource(stream)
-        mediaStreamSource.connect(noiseSuppressionNode).connect(analyser)
+    await audioContext.audioWorklet.addModule(NoiseSuppressorWorklet)
 
-        function volume() {
-            analyser.getByteFrequencyData(dataArray)
-            return dataArray.reduce((prev, value) => (prev && prev > value ? prev : value))
-        }
+    noiseSuppressionNode = new AudioWorkletNode(audioContext, NoiseSuppressorWorklet_Name)
+    const mediaStreamSource = audioContext.createMediaStreamSource(stream)
+    mediaStreamSource.connect(noiseSuppressionNode).connect(analyser)
 
-        function updateMeta(did: string) {
-            let muted = stream.getAudioTracks().some(track => !track.enabled || track.readyState === "ended")
-            let speaking = false
-            let user = Store.getUser(did)
-            let current = get(user)
-            let vol = volume()
-            if (!muted && vol > VOLUME_THRESHOLD) {
-                speaking = true
-            }
-            if (current.media.is_muted !== muted || current.media.is_playing_audio !== speaking) {
-                user.update(u => ({
-                    ...u,
-                    media: {
-                        ...u.media,
-                        is_muted: muted,
-                        is_playing_audio: speaking,
-                    },
-                }))
+    function updateMeta(did: string) {
+        let muted = stream.getAudioTracks().some(track => !track.enabled || track.readyState === "ended")
+        let user = Store.getUser(did)
+
+        user.update(u => ({
+            ...u,
+            media: {
+                ...u.media,
+                is_muted: muted,
+                is_playing_audio: speaking,
+            },
+        }))
+    }
+
+    const options = {
+        onVoiceStart: () => {
+            VoiceRTCInstance.localVideoCurrentSrc!.volume = 1
+            if (voiceStopTimeout) {
+                clearTimeout(voiceStopTimeout)
+                voiceStopTimeout = null
             }
-        }
+            let user = Store.getUser(did)
+            log.debug(`Voice detected from ${get(user).name}.`)
+            speaking = true
 
-        checker = setInterval(() => updateMeta(did), 300)
-    })
+            updateMeta(did)
+        },
+        onVoiceStop: () => {
+            voiceStopTimeout = setTimeout(() => {
+                VoiceRTCInstance.localVideoCurrentSrc!.volume = 0
+                let user = Store.getUser(did)
+                log.debug(`Voice Stopped from ${get(user).name}.`)
+                speaking = false
+                updateMeta(did)
+            }, 200)
+        },
+    }
+    const voiceDetector = vad(audioContext, stream, options)
+    voiceDetector.connect()
 
     return {
         remove: () => {
             analyser.disconnect()
             if (noiseSuppressionNode) noiseSuppressionNode.disconnect()
-            if (checker) clearInterval(checker)
+            voiceDetector.disconnect()
+            voiceDetector.destroy()
         },
     }
 }
@@ -177,7 +206,7 @@ export class Participant {
         if (this.streamHandler) {
             this.streamHandler.remove()
         }
-        this.streamHandler = handleStreamMeta(this.did, stream)
+        this.streamHandler = await handleStreamMeta(this.did, stream)
         this.stream = stream
     }
 
@@ -220,6 +249,9 @@ export class CallRoom {
                 this.start = new Date()
             }
         })
+        room.onPeerTrack((stream, peer, _meta) => {
+            log.debug(`Receiving track from ${peer}`)
+        })
         room.onPeerLeave(peer => {
             log.debug(`Peer ${peer} left the room`)
             let participant = Object.entries(this.participants).find(p => p[1].remotePeerId === peer)
@@ -409,6 +441,8 @@ export class VoiceRTC {
     }
 
     private async setupLocalPeer(reset?: boolean) {
+        // TODO(Lucas): Work on that in a next PR
+        // this.testGoodRelaysForCall()
         if ((reset && this.localPeer) || this.localPeer?.disconnected || this.localPeer?.destroyed) {
             this.localPeer.destroy()
             this.localPeer = null
@@ -604,14 +638,63 @@ export class VoiceRTC {
         return accepted
     }
 
+    /**
+     * Tests the connectivity of relay servers for initiating calls.
+     *
+     * This method iterates over a list of relay URLs specified in `relaysToTest` and attempts to establish a WebSocket
+     * connection with each one. It performs the following actions for each relay:
+     *
+     * - **On Successful Connection (`socket.onopen`):**
+     *   - Adds the relay URL to the `relaysWithSuccessfulConnection` array.
+     *   - Sends a "ping" message over the WebSocket connection.
+     *
+     * - **On Connection Error (`socket.onerror`):**
+     *   - Logs a warning message with the relay URL and error details.
+     *   - Removes the relay from the `remainingRelays` array.
+     *   - Closes the WebSocket connection.
+     *   - Updates the `relaysAvailable` store with the updated list of remaining relays.
+     *
+     * After testing all relays, it logs the list of relays with successful connections for debugging purposes.
+     *
+     * **Side Effects:**
+     * - Updates the `relaysAvailable` store by removing relays that failed to connect.
+     * - Logs warnings and debug information to assist with monitoring and troubleshooting.
+     *
+     * @private
+     */
+    private testGoodRelaysForCall() {
+        let remainingRelays: string[] = get(relaysAvailable)
+        let relaysWithSuccessfulConnection: string[] = []
+        for (let i = 0; i < relaysToTest.length; i++) {
+            let currentRelayUrl = relaysToTest[i]
+
+            const socket = new WebSocket(currentRelayUrl)
+
+            socket.onerror = error => {
+                remainingRelays = remainingRelays.filter(relay => relay !== currentRelayUrl)
+                socket.close()
+                relaysAvailable.set(remainingRelays)
+            }
+
+            socket.onopen = () => {
+                relaysWithSuccessfulConnection.push(currentRelayUrl)
+                socket.send("ping")
+            }
+        }
+        log.debug(`Relays connected: ${relaysWithSuccessfulConnection}`)
+    }
+
     private createAndSetRoom() {
         log.debug(`Creating/Joining room in channel ${this.channel}`)
+        log.info("Remaining relay urls to create room: ", get(relaysAvailable))
+
         Store.updateMuted(true)
+
         this.call = new CallRoom(
             joinRoom(
                 {
                     appId: "uplink",
-                    relayUrls: ["wss://nostr-pub.wellorder.net", "wss://relay.snort.social", "wss://nostr.oxtr.dev", "wss://relay.nostr.band", "wss://nostr.mom", "wss://nostr-relay.digitalmob.ro"],
+                    // relayUrls: get(relaysAvailable),
                     relayRedundancy: 3,
                 },
                 this.channel!
@@ -651,6 +734,7 @@ export class VoiceRTC {
     }
 
     async leaveCall(sendEndCallMessage = false) {
+        callScreenVisible.set(false)
         callInProgress.set(null)
         timeCallStarted.set(null)
         usersDeniedTheCall.set([])
@@ -696,7 +780,7 @@ export class VoiceRTC {
             if (this.localStreamHandler) {
                 this.localStreamHandler.remove()
             }
-            this.localStreamHandler = handleStreamMeta(get(Store.state.user).key, this.localStream)
+            this.localStreamHandler = await handleStreamMeta(get(Store.state.user).key, this.localStream)
             if (this.localVideoCurrentSrc) {
                 this.localVideoCurrentSrc.srcObject = this.localStream
                 await this.localVideoCurrentSrc.play()
@@ -711,10 +795,7 @@ export class VoiceRTC {
         let localStream
         localStream = await navigator.mediaDevices.getUserMedia({
             video: true,
-            audio: {
-                echoCancellation: true,
-                noiseSuppression: true,
-            },
+            audio: true,
         })
         localStream.getVideoTracks().forEach(track => {
             track.enabled = this.callOptions.video.enabled
diff --git a/src/lib/wasm/MultipassStore.ts b/src/lib/wasm/MultipassStore.ts
index 7a05fb776..c78399da1 100644
--- a/src/lib/wasm/MultipassStore.ts
+++ b/src/lib/wasm/MultipassStore.ts
@@ -172,11 +172,13 @@ class MultipassStore {
         return failure(WarpError.MULTIPASS_NOT_FOUND)
     }
 
-    async fetchAllFriendsAndRequests() {
+    async fetchAllFriendsAndRequests(listFriends: boolean = true) {
         await this.listIncomingFriendRequests()
         await this.listOutgoingFriendRequests()
         await this.listBlockedFriends()
-        await this.listFriends()
+        if (listFriends) {
+            await this.listFriends()
+        }
     }
 
     /**
diff --git a/vite.config.js b/vite.config.js
index 7b1ac54eb..dde6302a1 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -39,8 +39,10 @@ export default defineConfig({
         nodePolyfills(),
     ],
     optimizeDeps: {
+        include: ["voice-activity-detection"],
         exclude: ["warp-wasm"],
     },
+
     css: {
         preprocessorOptions: {
             scss: {