fix: reuse audio context, no audio when using krisp in some cases

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -5,6 +5,7 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSAction } from '../error/HMSAction';
 import { EventBus } from '../events/EventBus';
 import { HMSDeviceChangeEvent, HMSTrackUpdate, HMSUpdateListener } from '../interfaces';
+import { HMSAudioContextHandler } from '../internal';
 import { HMSRemoteAudioTrack } from '../media/tracks';
 import { HMSRemotePeer } from '../sdk/models/peer';
 import { Store } from '../sdk/store';
@@ -72,8 +73,9 @@ export class AudioSinkManager {
    */
   async unblockAutoplay() {
     if (this.autoPausedTracks.size > 0) {
-      this.unpauseAudioTracks();
+      await this.unpauseAudioTracks();
     }
+    await HMSAudioContextHandler.resumeContext();
   }
 
   init(elementId?: string) {
@@ -184,12 +186,12 @@ export class AudioSinkManager {
     await this.playAudioFor(track);
   };
 
-  private handleAudioDeviceChange = (event: HMSDeviceChangeEvent) => {
+  private handleAudioDeviceChange = async (event: HMSDeviceChangeEvent) => {
     // if there is no selection that means this is an init request. No need to do anything
     if (event.isUserSelection || event.error || !event.selection || event.type === 'video') {
       return;
     }
-    this.unpauseAudioTracks();
+    await this.unpauseAudioTracks();
   };
 
   /**

packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts

@@ -4,19 +4,13 @@ import AnalyticsEventFactory from '../../analytics/AnalyticsEventFactory';
 import { ErrorFactory } from '../../error/ErrorFactory';
 import { HMSAction } from '../../error/HMSAction';
 import { EventBus } from '../../events/EventBus';
+import { HMSAudioContextHandler } from '../../internal';
 import { HMSAudioTrackSettingsBuilder } from '../../media/settings';
 import { standardMediaConstraints } from '../../media/settings/constants';
 import { HMSLocalAudioTrack } from '../../media/tracks';
 import Room from '../../sdk/models/HMSRoom';
 import HMSLogger from '../../utils/logger';
 
-const DEFAULT_SAMPLE_RATE = 48000;
-
-//Handling sample rate error in case of firefox
-const checkBrowserSupport = () => {
-  return navigator.userAgent.indexOf('Firefox') !== -1;
-};
-
 /**
  * This class manages applying different plugins on a local audio track. Plugins which need to modify the audio
  * are called in the order they were added. Plugins which do not need to modify the audio are called
@@ -50,7 +44,7 @@ export class HMSAudioPluginsManager {
     this.hmsTrack = track;
     this.pluginsMap = new Map();
     this.analytics = new AudioPluginsAnalytics(eventBus);
-    this.createAudioContext();
+    this.audioContext = HMSAudioContextHandler.getAudioContext();
     this.room = room;
   }
 
@@ -234,7 +228,6 @@ export class HMSAudioPluginsManager {
 
   //Keeping it separate since we are initializing context only once
   async closeContext() {
-    this.audioContext?.close();
     this.audioContext = undefined;
   }
 
@@ -248,15 +241,14 @@ export class HMSAudioPluginsManager {
     for (const plugin of plugins) {
       await this.addPlugin(plugin);
     }
-    this.updateProcessedTrack();
+    await this.updateProcessedTrack();
   }
 
   private async initAudioNodes() {
     if (this.audioContext) {
-      if (!this.sourceNode) {
-        const audioStream = new MediaStream([this.hmsTrack.nativeTrack]);
-        this.sourceNode = this.audioContext.createMediaStreamSource(audioStream);
-      }
+      // recreate this again, irrespective of it being already there so that the latest native track is used in source node
+      const audioStream = new MediaStream([this.hmsTrack.nativeTrack]);
+      this.sourceNode = this.audioContext.createMediaStreamSource(audioStream);
       if (!this.destinationNode) {
         this.destinationNode = this.audioContext.createMediaStreamDestination();
         this.outputTrack = this.destinationNode.stream.getAudioTracks()[0];
@@ -316,19 +308,4 @@ export class HMSAudioPluginsManager {
     plugin.stop();
     this.analytics.removed(name);
   }
-
-  private createAudioContext() {
-    if (!this.audioContext) {
-      if (checkBrowserSupport()) {
-        /**
-        Not setting default sample rate for firefox since connecting
-        audio nodes from context with different sample rate is not
-        supported in firefox
- */
-        this.audioContext = new AudioContext();
-      } else {
-        this.audioContext = new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE });
-      }
-    }
-  }
 }

packages/hms-video-store/src/reactive-store/HMSSDKActions.ts

@@ -1534,24 +1534,29 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     }
   }
 
+  //eslint-disable-next-line complexity
   private async addRemoveAudioPlugin(plugin: HMSAudioPlugin, action: 'add' | 'remove') {
-    if (!plugin) {
-      HMSLogger.w('Invalid plugin received in store');
-      return;
-    }
-    const trackID = this.store.getState(selectLocalAudioTrackID);
-    if (trackID) {
-      const sdkTrack = this.getLocalTrack(trackID);
-      if (sdkTrack) {
-        if (action === 'add') {
-          await (sdkTrack as SDKHMSLocalAudioTrack).addPlugin(plugin);
-        } else if (action === 'remove') {
-          await (sdkTrack as SDKHMSLocalAudioTrack).removePlugin(plugin);
+    try {
+      if (!plugin) {
+        HMSLogger.w('Invalid plugin received in store');
+        return;
+      }
+      const trackID = this.store.getState(selectLocalAudioTrackID);
+      if (trackID) {
+        const sdkTrack = this.getLocalTrack(trackID);
+        if (sdkTrack) {
+          if (action === 'add') {
+            await (sdkTrack as SDKHMSLocalAudioTrack).addPlugin(plugin);
+          } else if (action === 'remove') {
+            await (sdkTrack as SDKHMSLocalAudioTrack).removePlugin(plugin);
+          }
+          this.syncRoomState(`${action}AudioPlugin`);
+        } else {
+          this.logPossibleInconsistency(`track ${trackID} not present, unable to ${action} plugin`);
         }
-        this.syncRoomState(`${action}AudioPlugin`);
-      } else {
-        this.logPossibleInconsistency(`track ${trackID} not present, unable to ${action} plugin`);
       }
+    } catch (err) {
+      console.error(err);
     }
   }
 

packages/hms-video-store/src/sdk/index.ts

@@ -667,6 +667,10 @@ export class HMSSdk implements HMSInterface {
       this.sdkState.isJoinInProgress = false;
       await this.publish(config.settings, previewRole);
       await this.deviceManager.autoSelectAudioOutput();
+      // Throw autoplay error even if audio context is suspended as it will be used in Audio Plugins which can lead to no audio
+      if (HMSAudioContextHandler.getAudioContext().state === 'suspended') {
+        this.listener?.onError(ErrorFactory.TracksErrors.AutoplayBlocked(HMSAction.JOIN));
+      }
     } catch (error) {
       this.analyticsTimer.end(TimedEvent.JOIN);
       this.sdkState.isJoinInProgress = false;

packages/hms-video-store/src/utils/media.ts

@@ -1,7 +1,11 @@
 import HMSLogger from './logger';
+import { isFirefox } from './support';
 import { BuildGetMediaError } from '../error/utils';
 import { HMSTrackExceptionTrackType } from '../media/tracks/HMSTrackExceptionTrackType';
 
+// discussed with krisp team and this is their recommendation for the sample rate
+const DEFAULT_SAMPLE_RATE = 32000;
+
 export async function getLocalStream(constraints: MediaStreamConstraints): Promise<MediaStream> {
   try {
     const stream = await navigator.mediaDevices.getUserMedia(constraints);
@@ -52,9 +56,10 @@ export const HMSAudioContextHandler: HMSAudioContext = {
   audioContext: null,
   getAudioContext() {
     if (!this.audioContext) {
-      this.audioContext = new AudioContext();
+      this.audioContext = isFirefox ? new AudioContext() : new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE });
     }
-    return this.audioContext;
+
+    return this.audioContext!;
   },
   async resumeContext() {
     try {

packages/roomkit-react/src/Prebuilt/components/Notifications/AutoplayBlockedModal.tsx

@@ -9,9 +9,9 @@ export function AutoplayBlockedModal() {
   return (
     <Dialog.Root
       open={!!error}
-      onOpenChange={value => {
+      onOpenChange={async value => {
         if (!value) {
-          unblockAudio();
+          await unblockAudio();
         }
         resetError();
       }}
@@ -25,8 +25,8 @@ export function AutoplayBlockedModal() {
         <DialogRow justify="end">
           <Button
             variant="primary"
-            onClick={() => {
-              unblockAudio();
+            onClick={async () => {
+              await unblockAudio();
               resetError();
             }}
           >

yarn.lock

@@ -16864,16 +16864,7 @@ string-natural-compare@^3.0.1:
   resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4"
   integrity sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -16978,14 +16969,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -18341,7 +18325,7 @@ worker-timers@^7.0.40:
     worker-timers-broker "^6.0.95"
     worker-timers-worker "^7.0.59"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -18359,15 +18343,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"