Merge branch 'dev' into publish-alpha

.github/workflows/sync-alpha-to-main.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
     types: [closed]
     branches:
-      - main
+      - dev
 
 jobs:
   pull-request:
@@ -15,8 +15,8 @@ jobs:
         if: github.event.pull_request.merged == true
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          source_branch: 'main'
+          source_branch: 'dev'
           destination_branch: 'publish-alpha'
           pr_title: 'Update publish-alpha'
-          pr_body: ':robot: Automated PR from main to publish-alpha'
+          pr_body: ':robot: Automated PR from dev to publish-alpha'
           pr_label: 'auto-pr'

packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts

@@ -3,6 +3,7 @@ import { AdditionalAnalyticsProperties } from './AdditionalAnalyticsProperties';
 import AnalyticsEvent from './AnalyticsEvent';
 import { AnalyticsEventLevel } from './AnalyticsEventLevel';
 import { IAnalyticsPropertiesProvider } from './IAnalyticsPropertiesProvider';
+import { pluginUsageTracker } from '../common';
 import { HMSException } from '../error/HMSException';
 import { DeviceMap, SelectedDevices } from '../interfaces';
 import { HMSTrackSettings } from '../media/settings/HMSTrackSettings';
@@ -235,6 +236,23 @@ export default class AnalyticsEventFactory {
     });
   }
 
+  static getKrispUsage(sessionID: string) {
+    const duration = pluginUsageTracker.getPluginUsage('HMSKrispPlugin', sessionID);
+    return new AnalyticsEvent({
+      name: 'krisp.usage',
+      level: AnalyticsEventLevel.INFO,
+      properties: { duration },
+    });
+  }
+
+  static transportLeave() {
+    return new AnalyticsEvent({
+      name: 'transport.leave',
+      level: AnalyticsEventLevel.INFO,
+      properties: {},
+    });
+  }
+
   private static eventNameFor(name: string, ok: boolean) {
     const suffix = ok ? 'success' : 'failed';
     return `${name}.${suffix}`;

packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts

@@ -85,6 +85,7 @@ export abstract class RunningTrackAnalytics {
 
   protected samples: (LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample)[] = [];
   protected tempStats: TempPublishStats[] = [];
+  protected prevLatestStat?: TempPublishStats;
 
   constructor({
     track,
@@ -118,6 +119,7 @@ export abstract class RunningTrackAnalytics {
     }
 
     this.samples.push(this.collateSample());
+    this.prevLatestStat = this.getLatestStat();
     this.tempStats.length = 0;
   }
 
@@ -160,7 +162,7 @@ export abstract class RunningTrackAnalytics {
   }
 
   protected calculateDifferenceForSample(key: keyof TempPublishStats) {
-    const firstValue = Number(this.tempStats[0][key]) || 0;
+    const firstValue = Number(this.prevLatestStat?.[key]) || 0;
     const latestValue = Number(this.getLatestStat()[key]) || 0;
 
     return latestValue - firstValue;

packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts

@@ -92,12 +92,16 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
 
   // eslint-disable-next-line complexity
   private calculateAvSyncForStat(trackStats: HMSTrackStats, hmsStats: HMSWebrtcStats) {
-    if (!trackStats.peerID || !(trackStats.kind === 'video')) {
+    if (!trackStats.peerID || !trackStats.estimatedPlayoutTimestamp || trackStats.kind !== 'video') {
       return;
     }
     const peer = this.store.getPeerById(trackStats.peerID);
     const audioTrack = peer?.audioTrack;
     const videoTrack = peer?.videoTrack;
+    /**
+     * 1. Send value as MAX_SAFE_INTEGER when either audio or value track is muted for the entire window
+     * 2. When both audio and video are unmuted for a part of window , then divide the difference by those many number of samples only
+     */
     const areBothTracksEnabled = audioTrack && videoTrack && audioTrack.enabled && videoTrack.enabled;
     if (!areBothTracksEnabled) {
       return MAX_SAFE_INTEGER;
@@ -106,7 +110,12 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
     if (!audioStats) {
       return MAX_SAFE_INTEGER;
     }
-    return audioStats.timestamp - trackStats.timestamp;
+    if (!audioStats.estimatedPlayoutTimestamp) {
+      return;
+    }
+
+    // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp
+    return audioStats.estimatedPlayoutTimestamp - trackStats.estimatedPlayoutTimestamp;
   }
 }
 
@@ -119,14 +128,9 @@ class RunningRemoteTrackAnalytics extends RunningTrackAnalytics {
 
     const baseSample = {
       timestamp: Date.now(),
-      fec_packets_discarded: this.calculateDifferenceForSample('fecPacketsDiscarded'),
-      fec_packets_received: this.calculateDifferenceForSample('fecPacketsReceived'),
-      total_samples_duration: this.calculateDifferenceForSample('totalSamplesDuration'),
-      total_packets_received: this.calculateDifferenceForSample('packetsReceived'),
-      total_packets_lost: this.calculateDifferenceForSample('packetsLost'),
       total_pli_count: this.calculateDifferenceForSample('pliCount'),
       total_nack_count: this.calculateDifferenceForSample('nackCount'),
-      avg_jitter_buffer_delay: this.calculateAverage('calculatedJitterBufferDelay'),
+      avg_jitter_buffer_delay: this.calculateAverage('calculatedJitterBufferDelay', false),
     };
 
     if (latestStat.kind === 'video') {
@@ -155,6 +159,11 @@ class RunningRemoteTrackAnalytics extends RunningTrackAnalytics {
         audio_concealed_samples,
         audio_total_samples_received: this.calculateDifferenceForSample('totalSamplesReceived'),
         audio_concealment_events: this.calculateDifferenceForSample('concealmentEvents'),
+        fec_packets_discarded: this.calculateDifferenceForSample('fecPacketsDiscarded'),
+        fec_packets_received: this.calculateDifferenceForSample('fecPacketsReceived'),
+        total_samples_duration: this.calculateDifferenceForSample('totalSamplesDuration'),
+        total_packets_received: this.calculateDifferenceForSample('packetsReceived'),
+        total_packets_lost: this.calculateDifferenceForSample('packetsLost'),
       });
     }
   };

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -47,6 +47,7 @@ export class AudioSinkManager {
     this.eventBus.audioTrackRemoved.subscribe(this.handleTrackRemove);
     this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate);
     this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange);
+    this.startPollingForDevices();
   }
 
   setListener(listener?: HMSUpdateListener) {
@@ -266,6 +267,19 @@ export class AudioSinkManager {
     }
   };
 
+  private startPollingForDevices = () => {
+    // device change supported, no polling needed
+    if ('ondevicechange' in navigator.mediaDevices) {
+      return;
+    }
+    this.timer = setInterval(() => {
+      (async () => {
+        await this.deviceManager.init(true, false);
+        await this.autoSelectAudioOutput();
+      })();
+    }, 5000);
+  };
+
   /**
    * Mweb is not able to play via call channel by default, this is to switch from media channel to call channel
    */
@@ -294,10 +308,18 @@ export class AudioSinkManager {
       const localAudioTrack = this.store.getLocalPeer()?.audioTrack;
       if (localAudioTrack && earpiece) {
         const externalDeviceID = bluetoothDevice?.deviceId || wired?.deviceId || speakerPhone?.deviceId;
-        await localAudioTrack.setSettings({ deviceId: earpiece?.deviceId });
-        await localAudioTrack.setSettings({
-          deviceId: externalDeviceID,
-        });
+        HMSLogger.d(this.TAG, 'externalDeviceID', externalDeviceID);
+        // already selected appropriate device
+        if (localAudioTrack.settings.deviceId === externalDeviceID) {
+          return;
+        }
+        await localAudioTrack.setSettings({ deviceId: earpiece?.deviceId }, true);
+        await localAudioTrack.setSettings(
+          {
+            deviceId: externalDeviceID,
+          },
+          true,
+        );
       }
     }
   };

packages/hms-video-store/src/common/PluginUsageTracker.ts

@@ -0,0 +1,53 @@
+import AnalyticsEvent from '../analytics/AnalyticsEvent';
+
+class PluginUsageTracker {
+  private pluginUsage: Map<string, number> = new Map<string, number>();
+  private pluginLastAddedAt: Map<string, number> = new Map<string, number>();
+
+  getPluginUsage = (name: string, sessionID: string) => {
+    const pluginKey = `${sessionID}-${name}`;
+
+    if (!this.pluginUsage.has(pluginKey)) {
+      this.pluginUsage.set(pluginKey, 0);
+    }
+    if (this.pluginLastAddedAt.has(pluginKey)) {
+      const lastAddedAt = this.pluginLastAddedAt.get(pluginKey) || 0;
+      const extraDuration = lastAddedAt ? Date.now() - lastAddedAt : 0;
+      this.pluginUsage.set(pluginKey, (this.pluginUsage.get(pluginKey) || 0) + extraDuration);
+      this.pluginLastAddedAt.delete(pluginKey);
+    }
+    return this.pluginUsage.get(pluginKey);
+  };
+
+  updatePluginUsageData = (event: AnalyticsEvent, sessionID: string) => {
+    // Sent on leave, after krisp usage is sent
+    if (event.name === 'transport.leave') {
+      this.cleanup(sessionID);
+      return;
+    }
+
+    const name = event.properties.plugin_name;
+    const pluginKey = `${sessionID}-${name}`;
+    if (event.name === 'mediaPlugin.added') {
+      const addedAt = event.properties.added_at;
+      this.pluginLastAddedAt.set(pluginKey, addedAt);
+    } else if (event.name === 'mediaPlugin.stats') {
+      const duration = event.properties.duration;
+      if (duration > 0) {
+        this.pluginUsage.set(pluginKey, (this.pluginUsage.get(pluginKey) || 0) + duration * 1000);
+        this.pluginLastAddedAt.delete(pluginKey);
+      }
+    }
+  };
+
+  private cleanup = (sessionID: string) => {
+    for (const key of this.pluginUsage.keys()) {
+      if (sessionID.length && key.includes(sessionID)) {
+        this.pluginUsage.delete(key);
+        this.pluginLastAddedAt.delete(key);
+      }
+    }
+  };
+}
+
+export const pluginUsageTracker = new PluginUsageTracker();

packages/hms-video-store/src/common/index.ts

@@ -0,0 +1 @@
+export { pluginUsageTracker } from './PluginUsageTracker';

packages/hms-video-store/src/device-manager/DeviceManager.ts

@@ -192,11 +192,8 @@ export class DeviceManager implements HMSDeviceManager {
     await this.enumerateDevices();
     this.logDevices('After Device Change');
     const localPeer = this.store.getLocalPeer();
-    const audioTrack = localPeer?.audioTrack;
     await this.setOutputDevice(true);
-    if (audioTrack) {
-      await this.handleAudioInputDeviceChange(localPeer?.audioTrack);
-    }
+    await this.handleAudioInputDeviceChange(localPeer?.audioTrack);
     await this.handleVideoInputDeviceChange(localPeer?.videoTrack);
     this.eventBus.analytics.publish(
       AnalyticsEventFactory.deviceChange({

packages/hms-video-store/src/index.ts

@@ -53,6 +53,7 @@ export type {
   HMSQuizLeaderboardSummary,
 } from './internal';
 
+export { pluginUsageTracker } from './common';
 export { HMSReactiveStore } from './reactive-store/HMSReactiveStore';
 export { HMSPluginUnsupportedTypes, HMSRecordingState, HLSPlaylistType } from './internal';
 export type {

packages/hms-video-store/src/interfaces/webrtc-stats.ts

@@ -57,6 +57,7 @@ export interface MissingInboundStats extends RTCInboundRtpStreamStats, MissingCo
   totalFreezesDuration?: number;
   jitterBufferDelay?: number;
   jitterBufferEmittedCount?: number;
+  estimatedPlayoutTimestamp?: DOMHighResTimeStamp;
 }
 
 export type PeerConnectionType = 'publish' | 'subscribe';

packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts

@@ -273,6 +273,14 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
     const hasPropertyChanged = generateHasPropertyChanged(settings, this.settings);
     if (hasPropertyChanged('deviceId')) {
       this.manuallySelectedDeviceId = !internal ? settings.deviceId : this.manuallySelectedDeviceId;
+      HMSLogger.d(
+        this.TAG,
+        'device change',
+        'manual selection:',
+        this.manuallySelectedDeviceId,
+        'new device:',
+        settings.deviceId,
+      );
       await this.replaceTrackWith(settings);
       const groupId = this.nativeTrack.getSettings().groupId;
       if (!internal && settings.deviceId) {

packages/hms-video-store/src/plugins/audio/AudioPluginsAnalytics.ts

@@ -24,6 +24,7 @@ export class AudioPluginsAnalytics {
     this.addedTimestamps[name] = Date.now();
     this.initTime[name] = 0;
     this.pluginSampleRate[name] = sampleRate;
+    this.eventBus.analytics.publish(MediaPluginsAnalyticsFactory.added(name, this.addedTimestamps[name]));
   }
 
   removed(name: string) {

packages/hms-video-store/src/sdk/index.ts

@@ -12,6 +12,7 @@ import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel';
 import { AnalyticsEventsService } from '../analytics/AnalyticsEventsService';
 import { AnalyticsTimer, TimedEvent } from '../analytics/AnalyticsTimer';
 import { AudioSinkManager } from '../audio-sink-manager';
+import { pluginUsageTracker } from '../common/PluginUsageTracker';
 import { DeviceManager } from '../device-manager';
 import { AudioOutputManager } from '../device-manager/AudioOutputManager';
 import { DeviceStorageManager } from '../device-manager/DeviceStorage';
@@ -181,7 +182,6 @@ export class HMSSdk implements HMSInterface {
     );
     this.sessionStore = new SessionStore(this.transport);
     this.interactivityCenter = new InteractivityCenter(this.transport, this.store, this.listener);
-
     /**
      * Note: Subscribe to events here right after creating stores and managers
      * to not miss events that are published before the handlers are subscribed.
@@ -498,6 +498,7 @@ export class HMSSdk implements HMSInterface {
     this.errorListener?.onError(error);
   };
 
+  // eslint-disable-next-line complexity
   async join(config: HMSConfig, listener: HMSUpdateListener) {
     validateMediaDevicesExistence();
     validateRTCPeerConnection();
@@ -571,6 +572,8 @@ export class HMSSdk implements HMSInterface {
       throw error;
     }
     HMSLogger.timeEnd(`join-room-${roomId}`);
+    const sessionID = this.store.getRoom()?.sessionId || '';
+    this.eventBus.analytics.subscribe(e => pluginUsageTracker.updatePluginUsageData(e, sessionID));
   }
 
   private stringifyMetadata(config: HMSConfig) {