Release - 0.12.22

packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts

@@ -135,6 +135,16 @@ export default class AnalyticsEventFactory {
     });
   }
 
+  static audioRecovered(message: string) {
+    return new AnalyticsEvent({
+      name: 'audioRecovered',
+      level: AnalyticsEventLevel.VERBOSE,
+      properties: {
+        message,
+      },
+    });
+  }
+
   static deviceChange({
     isUserSelection,
     selection,

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -46,6 +46,7 @@ export class AudioSinkManager {
     this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate);
     this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange);
     this.eventBus.localVideoUnmutedNatively.subscribe(this.unpauseAudioTracks);
+    this.eventBus.localAudioUnmutedNatively.subscribe(this.unpauseAudioTracks);
   }
 
   setListener(listener?: HMSUpdateListener) {
@@ -98,6 +99,7 @@ export class AudioSinkManager {
     this.eventBus.audioTrackUpdate.unsubscribe(this.handleTrackUpdate);
     this.eventBus.deviceChange.unsubscribe(this.handleAudioDeviceChange);
     this.eventBus.localVideoUnmutedNatively.unsubscribe(this.unpauseAudioTracks);
+    this.eventBus.localAudioUnmutedNatively.unsubscribe(this.unpauseAudioTracks);
     this.autoPausedTracks = new Set();
     this.state = { ...INITIAL_STATE };
   }
@@ -136,13 +138,19 @@ export class AudioSinkManager {
       );
       this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex));
       if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) {
+        // try to wait for main execution to complete first
         this.removeAudioElement(audioEl, track);
         await sleep(500);
         await this.handleTrackAdd({ track, peer, callListener: false });
+        if (!this.state.autoplayFailed) {
+          this.eventBus.analytics.publish(
+            AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'),
+          );
+        }
       }
     };
     track.setAudioElement(audioEl);
-    track.setVolume(this.volume);
+    await track.setVolume(this.volume);
     HMSLogger.d(this.TAG, 'Audio track added', `${track}`);
     this.init(); // call to create sink element if not already created
     this.audioSink?.append(audioEl);

packages/hms-video-store/src/device-manager/DeviceManager.ts

@@ -4,7 +4,7 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSException } from '../error/HMSException';
 import { EventBus } from '../events/EventBus';
 import { DeviceMap, HMSDeviceChangeEvent, SelectedDevices } from '../interfaces';
-import { getAudioDeviceCategory, isIOS } from '../internal';
+import { getAudioDeviceCategory, HMSAudioDeviceCategory, isIOS } from '../internal';
 import { HMSAudioTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings';
 import { HMSLocalAudioTrack, HMSLocalTrack, HMSLocalVideoTrack } from '../media/tracks';
 import { Store } from '../sdk/store';
@@ -444,13 +444,13 @@ export class DeviceManager implements HMSDeviceManager {
 
     for (const device of this.audioInput) {
       const deviceCategory = getAudioDeviceCategory(device.label);
-      if (deviceCategory === 'speakerphone') {
+      if (deviceCategory === HMSAudioDeviceCategory.SPEAKERPHONE) {
         speakerPhone = device;
-      } else if (deviceCategory === 'wired') {
+      } else if (HMSAudioDeviceCategory.WIRED) {
         wired = device;
-      } else if (deviceCategory === 'bluetooth') {
+      } else if (HMSAudioDeviceCategory.BLUETOOTH) {
         bluetoothDevice = device;
-      } else if (deviceCategory === 'speakerhone') {
+      } else if (HMSAudioDeviceCategory.EARPIECE) {
         earpiece = device;
       }
     }

packages/hms-video-store/src/error/ErrorCodes.ts

@@ -76,6 +76,9 @@ export const ErrorCodes = {
 
     // Selected device not detected on change
     SELECTED_DEVICE_MISSING: 3014,
+
+    // Track is publishing with no data, can happen when a whatsapp call is ongoing before 100ms call in mweb
+    NO_DATA: 3015,
   },
 
   WebrtcErrors: {

packages/hms-video-store/src/error/ErrorFactory.ts

@@ -251,6 +251,16 @@ export const ErrorFactory = {
         false,
       );
     },
+
+    NoDataInTrack(description: string) {
+      return new HMSException(
+        ErrorCodes.TracksErrors.NO_DATA,
+        'Track does not have any data',
+        HMSAction.TRACK,
+        description,
+        'This could possibily due to another application taking priority over the access to camera or microphone or due to an incoming call',
+      );
+    },
   },
 
   WebrtcErrors: {

packages/hms-video-store/src/events/EventBus.ts

@@ -31,6 +31,7 @@ export class EventBus {
     this.eventEmitter,
   );
   readonly localVideoUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_VIDEO_UNMUTED_NATIVELY, this.eventEmitter);
+  readonly localAudioUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_AUDIO_UNMUTED_NATIVELY, this.eventEmitter);
 
   /**
    * Emitter which processes raw RTC stats from rtcStatsUpdate and calls client callback

packages/hms-video-store/src/index.ts

@@ -12,6 +12,7 @@ export * from './selectors';
 export * from './webrtc-stats';
 export {
   HMSAudioMode,
+  HMSAudioDeviceCategory,
   HMSLogLevel,
   HMSAudioPluginType,
   HMSVideoPluginType,

packages/hms-video-store/src/interfaces/config.ts

@@ -42,10 +42,6 @@ export interface HMSConfig {
   audioSinkElementId?: string;
   autoVideoSubscribe?: boolean;
   initEndpoint?: string;
-  /**
-   * Request Camera/Mic permissions irrespective of role to avoid delay in getting device list
-   */
-  alwaysRequestPermissions?: boolean;
   /**
    * Enable to get a network quality score while in preview. The score ranges from -1 to 5.
    * -1 when we are not able to connect to 100ms servers within an expected time limit

packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts

@@ -56,6 +56,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
   ) {
     super(stream, track, source);
     stream.tracks.push(this);
+    this.addTrackEventListeners(track);
 
     this.settings = settings;
     // Replace the 'default' or invalid deviceId with the actual deviceId
@@ -99,13 +100,9 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     this.manuallySelectedDeviceId = undefined;
   }
 
-  private isTrackNotPublishing = () => {
-    return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted;
-  };
-
   private handleVisibilityChange = async () => {
     // track state is fine do nothing
-    if (!this.isTrackNotPublishing()) {
+    if (!this.shouldReacquireTrack()) {
       HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`);
       return;
     }
@@ -151,16 +148,19 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
      * no audio when the above getAudioTrack throws an error. ex: DeviceInUse error
      */
     prevTrack?.stop();
+    this.removeTrackEventListeners(prevTrack);
     this.tracksCreated.forEach(track => track.stop());
     this.tracksCreated.clear();
     try {
       const newTrack = await getAudioTrack(settings);
+      this.addTrackEventListeners(newTrack);
       this.tracksCreated.add(newTrack);
       HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack);
       await this.updateTrack(newTrack);
     } catch (e) {
       // Generate a new track from previous settings so there will be audio because previous track is stopped
       const newTrack = await getAudioTrack(this.settings);
+      this.addTrackEventListeners(newTrack);
       this.tracksCreated.add(newTrack);
       await this.updateTrack(newTrack);
       if (this.isPublished) {
@@ -184,8 +184,8 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
       return;
     }
 
-    // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled.
-    if (value && (isEmptyTrack(this.nativeTrack) || this.nativeTrack.muted)) {
+    // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled or ended track or when silence is detected.
+    if (value && this.shouldReacquireTrack()) {
       await this.replaceTrackWith(this.settings);
     }
     await super.setEnabled(value);
@@ -303,6 +303,42 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     return this.processedTrack || this.nativeTrack;
   }
 
+  private addTrackEventListeners(track: MediaStreamTrack) {
+    track.addEventListener('mute', this.handleTrackMute);
+    track.addEventListener('unmute', this.handleTrackUnmute);
+  }
+
+  private removeTrackEventListeners(track: MediaStreamTrack) {
+    track.removeEventListener('mute', this.handleTrackMute);
+    track.removeEventListener('unmute', this.handleTrackUnmute);
+  }
+
+  private handleTrackMute = () => {
+    HMSLogger.d(this.TAG, 'muted natively');
+    const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call';
+    this.eventBus.analytics.publish(
+      this.sendInterruptionEvent({
+        started: true,
+        reason,
+      }),
+    );
+  };
+
+  /** @internal */
+  handleTrackUnmute = async () => {
+    HMSLogger.d(this.TAG, 'unmuted natively');
+    const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call';
+    this.eventBus.analytics.publish(
+      this.sendInterruptionEvent({
+        started: false,
+        reason,
+      }),
+    );
+    await this.setEnabled(this.enabled);
+    // whatsapp call doesn't seem to send video unmute natively, so use audio unmute to play video
+    this.eventBus.localAudioUnmutedNatively.publish();
+  };
+
   private replaceSenderTrack = async () => {
     if (!this.transceiver || this.transceiver.direction !== 'sendonly') {
       HMSLogger.d(this.TAG, `transceiver for ${this.trackId} not available or not connected yet`);
@@ -311,6 +347,12 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     await this.transceiver.sender.replaceTrack(this.processedTrack || this.nativeTrack);
   };
 
+  private shouldReacquireTrack = () => {
+    return (
+      isEmptyTrack(this.nativeTrack) || this.isTrackNotPublishing() || this.audioLevelMonitor?.isSilentThisInstant()
+    );
+  };
+
   private buildNewSettings(settings: Partial<HMSAudioTrackSettings>) {
     const { volume, codec, maxBitrate, deviceId, advanced, audioMode } = { ...this.settings, ...settings };
     const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced, audioMode);

packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts

@@ -85,6 +85,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
     this.pluginsManager = new HMSVideoPluginsManager(this, eventBus);
     this.mediaStreamPluginsManager = new HMSMediaStreamPluginsManager(eventBus, room);
     this.setFirstTrackId(this.trackId);
+    this.eventBus.localAudioUnmutedNatively.subscribe(this.handleTrackUnmute);
     if (isBrowser && source === 'regular' && isMobile()) {
       document.addEventListener('visibilitychange', this.handleVisibilityChange);
     }
@@ -131,6 +132,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
    * use this function to set the enabled state of a track. If true the track will be unmuted and muted otherwise.
    * @param value
    */
+  // eslint-disable-next-line complexity
   async setEnabled(value: boolean): Promise<void> {
     if (value === this.enabled) {
       return;
@@ -260,6 +262,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
    * @internal
    */
   async cleanup() {
+    this.eventBus.localAudioUnmutedNatively.unsubscribe(this.handleTrackUnmute);
     this.removeTrackEventListeners(this.nativeTrack);
     super.cleanup();
     this.transceiver = undefined;
@@ -513,12 +516,12 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
 
   private addTrackEventListeners(track: MediaStreamTrack) {
     track.addEventListener('mute', this.handleTrackMute);
-    track.addEventListener('unmute', this.handleTrackUnmute);
+    track.addEventListener('unmute', this.handleTrackUnmuteNatively);
   }
 
   private removeTrackEventListeners(track: MediaStreamTrack) {
     track.removeEventListener('mute', this.handleTrackMute);
-    track.removeEventListener('unmute', this.handleTrackUnmute);
+    track.removeEventListener('unmute', this.handleTrackUnmuteNatively);
   }
 
   private handleTrackMute = () => {
@@ -533,17 +536,18 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
   };
 
   /** @internal */
-  handleTrackUnmute = () => {
+  handleTrackUnmuteNatively = async () => {
     HMSLogger.d(this.TAG, 'unmuted natively');
     this.eventBus.analytics.publish(
       this.sendInterruptionEvent({
         started: false,
         reason: 'incoming-call',
       }),
     );
-    super.handleTrackUnmute();
+    this.handleTrackUnmute();
     this.eventBus.localVideoEnabled.publish({ enabled: this.enabled, track: this });
     this.eventBus.localVideoUnmutedNatively.publish();
+    await this.setEnabled(this.enabled);
   };
 
   /**

packages/hms-video-store/src/media/tracks/HMSTrack.ts

@@ -85,6 +85,11 @@ export abstract class HMSTrack {
   protected setFirstTrackId(trackId: string) {
     this.firstTrackId = trackId;
   }
+
+  isTrackNotPublishing = () => {
+    return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted;
+  };
+
   /**
    * @internal
    * It will send event to analytics when interruption start/stop

packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts

@@ -93,8 +93,8 @@ export class HMSVideoTrack extends HMSTrack {
 
   private reTriggerPlay = ({ videoElement }: { videoElement: HTMLVideoElement }) => {
     setTimeout(() => {
-      videoElement.play().catch(() => {
-        HMSLogger.w('[HMSVideoTrack]', 'failed to play');
+      videoElement.play().catch((e: Error) => {
+        HMSLogger.w('[HMSVideoTrack]', 'failed to play', e.message);
       });
     }, 0);
   };