From 7cc38d6604970fc43f83aed0e27084fd53863c68 Mon Sep 17 00:00:00 2001 From: lgmarchi Date: Tue, 15 Oct 2024 15:43:36 -0300 Subject: [PATCH 1/5] Fix callsound in general --- src/lib/components/calling/CallScreen.svelte | 12 ++++- .../components/calling/IncomingCall.svelte | 7 ++- src/lib/components/utils/SoundHandler.ts | 49 +++++++++++++------ src/routes/+layout.svelte | 27 ++++++++++ src/routes/chat/+page.svelte | 7 ++- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/src/lib/components/calling/CallScreen.svelte b/src/lib/components/calling/CallScreen.svelte index b58bab845..04b78f56d 100644 --- a/src/lib/components/calling/CallScreen.svelte +++ b/src/lib/components/calling/CallScreen.svelte @@ -12,9 +12,10 @@ import { _ } from "svelte-i18n" import type { Chat } from "$lib/types" import VolumeMixer from "./VolumeMixer.svelte" - import { onDestroy, onMount } from "svelte" + import { createEventDispatcher, onDestroy, onMount } from "svelte" import { callTimeout, TIME_TO_SHOW_CONNECTING, VoiceRTCInstance } from "$lib/media/Voice" import { log } from "$lib/utils/Logger" + import { playSound, SoundHandler, Sounds } from "../utils/SoundHandler" export let expanded: boolean = false function toggleExanded() { @@ -30,6 +31,8 @@ export let deafened: boolean = get(Store.state.devices.deafened) export let chat: Chat + let dispatch = createEventDispatcher() + function toggleFullscreen() { const elem = document.getElementById("call-screen") @@ -117,6 +120,7 @@ let showAnimation = true let message = $_("settings.calling.connecting") let timeout: NodeJS.Timeout | undefined + let callSound: SoundHandler | undefined = undefined onMount(async () => { document.addEventListener("mousedown", handleClickOutside) @@ -124,8 +128,11 @@ /// HACK: To make sure the video elements are loaded before we start the call if (VoiceRTCInstance.localVideoCurrentSrc && VoiceRTCInstance.remoteVideoCreator) { if (VoiceRTCInstance.toCall && VoiceRTCInstance.toCall.find(did => did !== "") !== undefined) { + callSound = await playSound(Sounds.OutgoingCall) await VoiceRTCInstance.makeCall() timeout = setTimeout(() => { + callSound?.stop() + callSound = undefined showAnimation = false message = $_("settings.calling.noResponse") }, TIME_TO_SHOW_CONNECTING) @@ -146,6 +153,8 @@ if (timeout) { clearTimeout(timeout) } + callSound?.stop() + callSound = undefined }) @@ -322,6 +331,7 @@ on:click={_ => { Store.endCall() VoiceRTCInstance.leaveCall() + dispatch("endCall") }}> diff --git a/src/lib/components/calling/IncomingCall.svelte b/src/lib/components/calling/IncomingCall.svelte index 6cc789ec8..80a5b7561 100644 --- a/src/lib/components/calling/IncomingCall.svelte +++ b/src/lib/components/calling/IncomingCall.svelte @@ -35,8 +35,7 @@ Store.state.pendingCall.subscribe(async _ => { if (VoiceRTCInstance.incomingCallFrom && !VoiceRTCInstance.toCall && $connectionOpened) { if (callSound === null || callSound === undefined) { - callSound = playSound(Sounds.IncomingCall) - callSound.play() + callSound = await playSound(Sounds.IncomingCall) } pending = true let chat = UIStore.getChat(VoiceRTCInstance.incomingCallFrom[1].metadata.channel) @@ -48,11 +47,11 @@ user = Store.getUser(VoiceRTCInstance.incomingCallFrom[1].metadata.did) } else if (!$connectionOpened) { cancelledCall = true + callSound?.stop() + callSound = undefined timeOutToCancel = setTimeout(() => { cancelledCall = false pending = false - callSound?.stop() - callSound = undefined }, 4000) } else { pending = false diff --git a/src/lib/components/utils/SoundHandler.ts b/src/lib/components/utils/SoundHandler.ts index 2ef6ffc06..16808f8ba 100644 --- a/src/lib/components/utils/SoundHandler.ts +++ b/src/lib/components/utils/SoundHandler.ts @@ -1,3 +1,4 @@ +import { log } from "$lib/utils/Logger" import { Howl } from "howler" export enum Sounds { @@ -23,15 +24,17 @@ export type SoundOption = { export class SoundHandler { private sound: Howl + public soundID: number = -1 private muted: boolean = false - constructor(sound: Howl) { + constructor(sound: Howl, soundID: number) { this.sound = sound + this.soundID = soundID } mute() { this.muted = !this.muted - this.sound.mute(!this.muted) + this.sound.mute(!this.muted, this.soundID) } isMuted(): boolean { @@ -39,30 +42,46 @@ export class SoundHandler { } pause() { - this.sound.pause() + this.sound.pause(this.soundID) } play() { - this.sound.play() + this.sound.play(undefined, true) } paused(): boolean { - return !this.sound.playing() + return !this.sound.playing(this.soundID) } stop() { - this.sound.stop() + this.sound.stop(0, true) this.sound.unload() } } -export function playSound(src: string | Sounds, opt?: SoundOption): SoundHandler { - var sound = new Howl({ - src: [src.toString()], - html5: opt?.large, - volume: opt?.volume ? opt?.volume : 1, - loop: opt?.loop, - }) - sound.play() - return new SoundHandler(sound) +export async function playSound(src: string | Sounds, opt?: SoundOption): Promise { + try { + // Ask for permission to play sound + if (Notification.permission !== "granted") { + const permission = await Notification.requestPermission() + if (permission !== "granted") { + log.warn("Permission to play sound not granted") + return undefined + } + } + + const sound = new Howl({ + src: [src.toString()], + html5: opt?.large || false, + volume: opt?.volume || 1, + loop: opt?.loop || false, + }) + + let soundID = sound.play(undefined, false) + + return new SoundHandler(sound, soundID) + } catch (error) { + log.error("Error to play callsound:", error) + return undefined + } } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index e1b2b149f..c3a5b5d9c 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -231,6 +231,33 @@ console.log("Arriving here on +layout") + window.addEventListener( + "click", + () => { + initializeAudioContext() + log.debug("Audio context unlocked after click.") + }, + { once: true } + ) + + window.addEventListener( + "touchstart", + () => { + initializeAudioContext() + log.debug("Audio context unlocked after touch.") + }, + { once: true } + ) + + function initializeAudioContext() { + const audioContext = new window.AudioContext() + if (audioContext.state === "suspended") { + audioContext.resume().then(() => { + log.info("Audio context unlocked.") + }) + } + } + let isLocaleSet = false $: if ($locale) { diff --git a/src/routes/chat/+page.svelte b/src/routes/chat/+page.svelte index 66fca3981..daa8ec871 100644 --- a/src/routes/chat/+page.svelte +++ b/src/routes/chat/+page.svelte @@ -616,7 +616,12 @@ {/if} {#if activeCallInProgress && activeCallDid === $activeChat.id} - + { + activeCallInProgress = false + activeCallDid = "" + }} /> {/if} Date: Tue, 15 Oct 2024 15:50:39 -0300 Subject: [PATCH 2/5] Set callTimeout as false after call is accepted --- src/lib/media/Voice.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/media/Voice.ts b/src/lib/media/Voice.ts index b8f1bfa99..8c78ce199 100644 --- a/src/lib/media/Voice.ts +++ b/src/lib/media/Voice.ts @@ -507,6 +507,7 @@ export class VoiceRTC { }) conn.once("data", d => { if (d === CALL_ACK) { + callTimeout.set(false) accepted = true } }) From b8bc491e19b9ffb97e1444fc9ab4e0e979de0290 Mon Sep 17 00:00:00 2001 From: lgmarchi Date: Wed, 16 Oct 2024 16:08:07 -0300 Subject: [PATCH 3/5] Improve calltimeout --- src/lib/components/calling/CallScreen.svelte | 4 +++- src/lib/media/Voice.ts | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/components/calling/CallScreen.svelte b/src/lib/components/calling/CallScreen.svelte index 04b78f56d..e5ce82ac6 100644 --- a/src/lib/components/calling/CallScreen.svelte +++ b/src/lib/components/calling/CallScreen.svelte @@ -13,7 +13,7 @@ import type { Chat } from "$lib/types" import VolumeMixer from "./VolumeMixer.svelte" import { createEventDispatcher, onDestroy, onMount } from "svelte" - import { callTimeout, TIME_TO_SHOW_CONNECTING, VoiceRTCInstance } from "$lib/media/Voice" + import { callTimeout, connectionOpened, TIME_TO_SHOW_CONNECTING, VoiceRTCInstance } from "$lib/media/Voice" import { log } from "$lib/utils/Logger" import { playSound, SoundHandler, Sounds } from "../utils/SoundHandler" @@ -123,6 +123,7 @@ let callSound: SoundHandler | undefined = undefined onMount(async () => { + callTimeout.set(false) document.addEventListener("mousedown", handleClickOutside) await VoiceRTCInstance.setVideoElements(localVideoCurrentSrc) /// HACK: To make sure the video elements are loaded before we start the call @@ -145,6 +146,7 @@ }) onDestroy(() => { + callTimeout.set(false) document.removeEventListener("mousedown", handleClickOutside) subscribeOne() subscribeTwo() diff --git a/src/lib/media/Voice.ts b/src/lib/media/Voice.ts index 8c78ce199..8ff39d11f 100644 --- a/src/lib/media/Voice.ts +++ b/src/lib/media/Voice.ts @@ -592,6 +592,7 @@ export class VoiceRTC { } async leaveCall(sendEndCallMessage = false) { + callTimeout.set(false) connectionOpened.set(false) timeOuts.forEach(t => clearTimeout(t)) sendEndCallMessage = sendEndCallMessage && this.channel !== undefined && this.call != null From f8893ae049eb203e7b35b0ec0fa14d8637ca1ebb Mon Sep 17 00:00:00 2001 From: lgmarchi Date: Wed, 16 Oct 2024 17:19:53 -0300 Subject: [PATCH 4/5] Stop sound when one user at least accept call --- src/lib/components/calling/CallScreen.svelte | 16 ++++++++++++++-- src/lib/lang/en.json | 1 + src/lib/media/Voice.ts | 6 ++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/lib/components/calling/CallScreen.svelte b/src/lib/components/calling/CallScreen.svelte index e5ce82ac6..6d391302d 100644 --- a/src/lib/components/calling/CallScreen.svelte +++ b/src/lib/components/calling/CallScreen.svelte @@ -13,7 +13,7 @@ import type { Chat } from "$lib/types" import VolumeMixer from "./VolumeMixer.svelte" import { createEventDispatcher, onDestroy, onMount } from "svelte" - import { callTimeout, connectionOpened, TIME_TO_SHOW_CONNECTING, VoiceRTCInstance } from "$lib/media/Voice" + import { callTimeout, connectionOpened, TIME_TO_SHOW_CONNECTING, usersAcceptedTheCall, VoiceRTCInstance } from "$lib/media/Voice" import { log } from "$lib/utils/Logger" import { playSound, SoundHandler, Sounds } from "../utils/SoundHandler" @@ -122,8 +122,15 @@ let timeout: NodeJS.Timeout | undefined let callSound: SoundHandler | undefined = undefined + $: if ($usersAcceptedTheCall.length > 0) { + console.log("Users accepted the call: ", $usersAcceptedTheCall) + callSound?.stop() + callSound = undefined + } + onMount(async () => { callTimeout.set(false) + usersAcceptedTheCall.set([]) document.addEventListener("mousedown", handleClickOutside) await VoiceRTCInstance.setVideoElements(localVideoCurrentSrc) /// HACK: To make sure the video elements are loaded before we start the call @@ -196,13 +203,18 @@ {#if user === get(Store.state.user).key && !userCallOptions.video.enabled} {:else if $userCache[user] && $userCache[user].key !== get(Store.state.user).key && VoiceRTCInstance.toCall && !$remoteStreams[user]} - {#if showAnimation} + {#if showAnimation && !$usersAcceptedTheCall.includes(user)}

{message}

+ {:else if $usersAcceptedTheCall.includes(user)} +
+ +

{$_("settings.calling.acceptedCall")}

+
{:else}
diff --git a/src/lib/lang/en.json b/src/lib/lang/en.json index c5ea93788..d4d53c904 100644 --- a/src/lib/lang/en.json +++ b/src/lib/lang/en.json @@ -347,6 +347,7 @@ "disconnecting": "Disconnecting...", "userInviteToAGroupCall": "is inviting you to join a group call", "noAnswer": "No answer, leaving the call...", + "acceptedCall": "Joined, loading...", "connecting": "Connecting...", "noResponse": "No response" }, diff --git a/src/lib/media/Voice.ts b/src/lib/media/Voice.ts index 8ff39d11f..9eda0a83f 100644 --- a/src/lib/media/Voice.ts +++ b/src/lib/media/Voice.ts @@ -1,12 +1,11 @@ import { CallDirection } from "$lib/enums" -import { SettingsStore } from "$lib/state" import { Store } from "$lib/state/Store" import { create_cancellable_handler, type Cancellable } from "$lib/utils/CancellablePromise" import { log } from "$lib/utils/Logger" import { RaygunStoreInstance } from "$lib/wasm/RaygunStore" import Peer, { DataConnection } from "peerjs" import { _ } from "svelte-i18n" -import { get, writable } from "svelte/store" +import { get, writable, type Writable } from "svelte/store" import type { Room } from "trystero" import { joinRoom } from "trystero/ipfs" @@ -17,6 +16,7 @@ const TIME_TO_SHOW_END_CALL_FEEDBACK = 3500 export const TIME_TO_SHOW_CONNECTING = 30000 let timeOuts: NodeJS.Timeout[] = [] +export const usersAcceptedTheCall: Writable = writable([]) export enum VoiceRTCMessageType { UpdateUser = "UPDATE_USER", @@ -508,6 +508,7 @@ export class VoiceRTC { conn.once("data", d => { if (d === CALL_ACK) { callTimeout.set(false) + usersAcceptedTheCall.set([...get(usersAcceptedTheCall), did]) accepted = true } }) @@ -594,6 +595,7 @@ export class VoiceRTC { async leaveCall(sendEndCallMessage = false) { callTimeout.set(false) connectionOpened.set(false) + usersAcceptedTheCall.set([]) timeOuts.forEach(t => clearTimeout(t)) sendEndCallMessage = sendEndCallMessage && this.channel !== undefined && this.call != null if (sendEndCallMessage && this.call?.start) { From 5b6b548d453e0dabbd945a2a8724602eef99b9c7 Mon Sep 17 00:00:00 2001 From: lgmarchi Date: Wed, 16 Oct 2024 17:21:52 -0300 Subject: [PATCH 5/5] Remove unnecessary log --- src/lib/components/calling/CallScreen.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/components/calling/CallScreen.svelte b/src/lib/components/calling/CallScreen.svelte index 6d391302d..bc8e433b3 100644 --- a/src/lib/components/calling/CallScreen.svelte +++ b/src/lib/components/calling/CallScreen.svelte @@ -123,7 +123,6 @@ let callSound: SoundHandler | undefined = undefined $: if ($usersAcceptedTheCall.length > 0) { - console.log("Users accepted the call: ", $usersAcceptedTheCall) callSound?.stop() callSound = undefined }