Release PR

packages/hms-video-store/src/analytics/stats/BaseStatsAnalytics.ts

@@ -63,15 +63,32 @@ export abstract class BaseStatsAnalytics {
     });
   }
 
+  protected cleanTrackAnalyticsAndCreateSample(shouldCreateSample: boolean) {
+    // delete track analytics if track is not present in store and no samples are present
+    this.trackAnalytics.forEach(trackAnalytic => {
+      if (!this.store.hasTrack(trackAnalytic.track) && !(trackAnalytic.samples.length > 0)) {
+        this.trackAnalytics.delete(trackAnalytic.track_id);
+      }
+    });
+
+    if (shouldCreateSample) {
+      this.trackAnalytics.forEach(trackAnalytic => {
+        trackAnalytic.createSample();
+      });
+    }
+  }
+
   protected abstract toAnalytics(): PublishAnalyticPayload | SubscribeAnalyticPayload;
 
   protected abstract handleStatsUpdate(hmsStats: HMSWebrtcStats): void;
 }
 
-type TempPublishStats = HMSTrackStats & {
+export type TempStats = HMSTrackStats & {
   availableOutgoingBitrate?: number;
   calculatedJitterBufferDelay?: number;
   avSync?: number;
+  expectedFrameHeight?: number;
+  expectedFrameWidth?: number;
 };
 
 export abstract class RunningTrackAnalytics {
@@ -83,9 +100,9 @@ export abstract class RunningTrackAnalytics {
   kind: string;
   rid?: string;
 
-  protected samples: (LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample)[] = [];
-  protected tempStats: TempPublishStats[] = [];
-  protected prevLatestStat?: TempPublishStats;
+  samples: (LocalBaseSample | LocalVideoSample | RemoteAudioSample | RemoteVideoSample)[] = [];
+  protected tempStats: TempStats[] = [];
+  protected prevLatestStat?: TempStats;
 
   constructor({
     track,
@@ -109,7 +126,7 @@ export abstract class RunningTrackAnalytics {
     this.sampleWindowSize = sampleWindowSize;
   }
 
-  pushTempStat(stat: TempPublishStats) {
+  pushTempStat(stat: TempStats) {
     this.tempStats.push(stat);
   }
 
@@ -145,7 +162,7 @@ export abstract class RunningTrackAnalytics {
     return this.tempStats[0];
   }
 
-  protected calculateSum(key: keyof TempPublishStats) {
+  protected calculateSum(key: keyof TempStats) {
     const checkStat = this.getLatestStat()[key];
     if (typeof checkStat !== 'number') {
       return;
@@ -155,25 +172,25 @@ export abstract class RunningTrackAnalytics {
     }, 0);
   }
 
-  protected calculateAverage(key: keyof TempPublishStats, round = true) {
+  protected calculateAverage(key: keyof TempStats, round = true) {
     const sum = this.calculateSum(key);
     const avg = sum !== undefined ? sum / this.tempStats.length : undefined;
     return avg ? (round ? Math.round(avg) : avg) : undefined;
   }
 
-  protected calculateDifferenceForSample(key: keyof TempPublishStats) {
+  protected calculateDifferenceForSample(key: keyof TempStats) {
     const firstValue = Number(this.prevLatestStat?.[key]) || 0;
     const latestValue = Number(this.getLatestStat()[key]) || 0;
 
     return latestValue - firstValue;
   }
 
-  protected calculateDifferenceAverage(key: keyof TempPublishStats, round = true) {
+  protected calculateDifferenceAverage(key: keyof TempStats, round = true) {
     const avg = this.calculateDifferenceForSample(key) / this.tempStats.length;
     return round ? Math.round(avg) : avg;
   }
 
-  protected calculateInstancesOfHigh(key: keyof TempPublishStats, threshold: number) {
+  protected calculateInstancesOfHigh(key: keyof TempStats, threshold: number) {
     const checkStat = this.getLatestStat()[key];
     if (typeof checkStat !== 'number') {
       return;
@@ -185,10 +202,10 @@ export abstract class RunningTrackAnalytics {
   }
 }
 
-export const hasResolutionChanged = (newStat: TempPublishStats, prevStat: TempPublishStats) =>
+export const hasResolutionChanged = (newStat: TempStats, prevStat: TempStats) =>
   newStat && prevStat && (newStat.frameWidth !== prevStat.frameWidth || newStat.frameHeight !== prevStat.frameHeight);
 
-export const hasEnabledStateChanged = (newStat: TempPublishStats, prevStat: TempPublishStats) =>
+export const hasEnabledStateChanged = (newStat: TempStats, prevStat: TempStats) =>
   newStat && prevStat && newStat.enabled !== prevStat.enabled;
 
 export const removeUndefinedFromObject = <T extends Record<string, any>>(data: T) => {

packages/hms-video-store/src/analytics/stats/PublishStatsAnalytics.ts

@@ -84,18 +84,7 @@ export class PublishStatsAnalytics extends BaseStatsAnalytics {
       });
     });
 
-    // delete track analytics if track is not present in store and no samples are present
-    this.trackAnalytics.forEach(trackAnalytic => {
-      if (!this.store.hasTrack(trackAnalytic.track) && !(trackAnalytic.samples.length > 0)) {
-        this.trackAnalytics.delete(trackAnalytic.track_id);
-      }
-    });
-
-    if (shouldCreateSample) {
-      this.trackAnalytics.forEach(trackAnalytic => {
-        trackAnalytic.createSample();
-      });
-    }
+    this.cleanTrackAnalyticsAndCreateSample(shouldCreateSample);
   }
 
   private getTrackIdentifier(trackId: string, stats: HMSTrackStats) {

packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts

@@ -4,6 +4,7 @@ import {
   hasResolutionChanged,
   removeUndefinedFromObject,
   RunningTrackAnalytics,
+  TempStats,
 } from './BaseStatsAnalytics';
 import {
   RemoteAudioSample,
@@ -13,6 +14,7 @@ import {
   SubscribeAnalyticPayload,
 } from './interfaces';
 import { HMSTrackStats } from '../../interfaces';
+import { HMSRemoteVideoTrack } from '../../internal';
 import { HMSWebrtcStats } from '../../rtc-stats';
 import { MAX_SAFE_INTEGER, SUBSCRIBE_STATS_SAMPLE_WINDOW } from '../../utils/constants';
 import AnalyticsEventFactory from '../AnalyticsEventFactory';
@@ -50,14 +52,23 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
     Object.keys(remoteTracksStats).forEach(trackID => {
       const trackStats = remoteTracksStats[trackID];
       const track = this.store.getTrackById(trackID);
-      const calculatedJitterBufferDelay =
+
+      const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats) =>
         trackStats.jitterBufferDelay &&
         trackStats.jitterBufferEmittedCount &&
         trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount;
 
+      const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats);
+
       const avSync = this.calculateAvSyncForStat(trackStats, hmsStats);
+      const newTempStat: TempStats = { ...trackStats, calculatedJitterBufferDelay, avSync };
+      if (trackStats.kind === 'video') {
+        const definition = (track as HMSRemoteVideoTrack).getPreferredLayerDefinition();
+        newTempStat.expectedFrameHeight = definition?.resolution.height;
+        newTempStat.expectedFrameWidth = definition?.resolution.width;
+      }
       if (this.trackAnalytics.has(trackID)) {
-        this.trackAnalytics.get(trackID)?.pushTempStat({ ...trackStats, calculatedJitterBufferDelay, avSync });
+        this.trackAnalytics.get(trackID)?.pushTempStat(newTempStat);
       } else {
         if (track) {
           const trackAnalytics = new RunningRemoteTrackAnalytics({
@@ -66,7 +77,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
             ssrc: trackStats.ssrc.toString(),
             kind: trackStats.kind,
           });
-          trackAnalytics.pushTempStat({ ...trackStats, calculatedJitterBufferDelay, avSync });
+          trackAnalytics.pushTempStat(newTempStat);
           this.trackAnalytics.set(trackID, trackAnalytics);
         }
       }
@@ -76,18 +87,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
       }
     });
 
-    // delete track analytics if track is not present in store and no samples are present
-    this.trackAnalytics.forEach(trackAnalytic => {
-      if (!this.store.hasTrack(trackAnalytic.track) && !(trackAnalytic.samples.length > 0)) {
-        this.trackAnalytics.delete(trackAnalytic.track_id);
-      }
-    });
-
-    if (shouldCreateSample) {
-      this.trackAnalytics.forEach(trackAnalytic => {
-        trackAnalytic.createSample();
-      });
-    }
+    this.cleanTrackAnalyticsAndCreateSample(shouldCreateSample);
   }
 
   // eslint-disable-next-line complexity
@@ -142,6 +142,8 @@ class RunningRemoteTrackAnalytics extends RunningTrackAnalytics {
         avg_frames_decoded_per_sec: this.calculateDifferenceAverage('framesDecoded'),
         frame_width: this.calculateAverage('frameWidth'),
         frame_height: this.calculateAverage('frameHeight'),
+        expected_frame_width: this.calculateAverage('expectedFrameWidth'),
+        expected_frame_height: this.calculateAverage('expectedFrameHeight'),
         pause_count: this.calculateDifferenceForSample('pauseCount'),
         pause_duration_seconds: this.calculateDifferenceForSample('totalPausesDuration'),
         freeze_count: this.calculateDifferenceForSample('freezeCount'),

packages/hms-video-store/src/analytics/stats/interfaces.ts

@@ -90,6 +90,8 @@ export interface RemoteVideoSample extends RemoteBaseSample {
   avg_av_sync_ms?: number;
   frame_width?: number;
   frame_height?: number;
+  expected_frame_width?: number;
+  expected_frame_height?: number;
   pause_count?: number;
   pause_duration_seconds?: number;
   freeze_count?: number;

packages/hms-video-store/src/index.ts

@@ -52,11 +52,19 @@ export type {
   HMSPollQuestionOption,
   HMSQuizLeaderboardResponse,
   HMSQuizLeaderboardSummary,
+  HMSTranscriptionInfo,
 } from './internal';
 
 export { EventBus } from './events/EventBus';
 export { HMSReactiveStore } from './reactive-store/HMSReactiveStore';
-export { HMSPluginUnsupportedTypes, HMSRecordingState, HLSPlaylistType, HLSStreamType } from './internal';
+export {
+  HMSPluginUnsupportedTypes,
+  HMSRecordingState,
+  HLSPlaylistType,
+  HLSStreamType,
+  HMSTranscriptionMode,
+  HMSTranscriptionState,
+} from './internal';
 export type {
   HMSVideoPlugin,
   HMSAudioPlugin,

packages/hms-video-store/src/interfaces/hms.ts

@@ -8,7 +8,7 @@ import { HMSPlaylistManager, HMSPlaylistSettings } from './playlist';
 import { HMSPreviewListener } from './preview-listener';
 import { HMSRole } from './role';
 import { HMSRoleChangeRequest } from './role-change-request';
-import { HMSHLS, HMSRecording, HMSRTMP } from './room';
+import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room';
 import { RTMPRecordingConfig } from './rtmp-recording-config';
 import { HMSInteractivityCenter, HMSSessionStore } from './session-store';
 import { HMSScreenShareConfig } from './track-settings';
@@ -64,6 +64,7 @@ export interface HMSInterface {
   getRecordingState(): HMSRecording | undefined;
   getRTMPState(): HMSRTMP | undefined;
   getHLSState(): HMSHLS | undefined;
+  getTranscriptionState(): HMSTranscriptionInfo[] | undefined;
   changeName(name: string): Promise<void>;
   changeMetadata(metadata: string): Promise<void>;
 

packages/hms-video-store/src/interfaces/room.ts

@@ -43,6 +43,7 @@ export interface HMSRoom {
   effectsKey?: string;
   isHipaaEnabled?: boolean;
   isNoiseCancellationEnabled?: boolean;
+  transcriptions?: HMSTranscriptionInfo[];
 }
 
 export interface HMSRecording {
@@ -117,3 +118,19 @@ export interface HLSVariant {
   state?: HMSStreamingState;
   stream_type?: HLSStreamType;
 }
+
+/*
+Transcription related details
+*/
+export enum HMSTranscriptionState {
+  STARTED = 'started',
+  STOPPED = 'stopped',
+  FAILED = 'failed',
+}
+export enum HMSTranscriptionMode {
+  CAPTION = 'caption',
+}
+export interface HMSTranscriptionInfo {
+  state?: HMSTranscriptionState;
+  mode?: HMSTranscriptionMode;
+}

packages/hms-video-store/src/media/tracks/HMSRemoteVideoTrack.ts

@@ -89,6 +89,10 @@ export class HMSRemoteVideoTrack extends HMSVideoTrack {
     return this.preferredLayer;
   }
 
+  getPreferredLayerDefinition() {
+    return this._layerDefinitions.find(layer => layer.layer === this.preferredLayer);
+  }
+
   replaceTrack(track: HMSRemoteVideoTrack) {
     this.nativeTrack = track.nativeTrack;
     if (track.transceiver) {

packages/hms-video-store/src/notification-manager/HMSNotifications.ts

@@ -61,6 +61,14 @@ export enum HMSStreamingState {
   FAILED = 'failed',
 }
 
+export enum HMSTranscriptionState {
+  STARTED = 'started',
+  STOPPED = 'stopped',
+  FAILED = 'failed',
+}
+export enum HMSTranscriptionMode {
+  CAPTION = 'caption',
+}
 interface PluginPermissions {
   permissions?: {
     // list of roles
@@ -122,6 +130,11 @@ export interface PeerNotification {
   is_from_room_state?: boolean;
 }
 
+export interface TranscriptionNotification {
+  state?: HMSTranscriptionState;
+  mode?: HMSTranscriptionMode;
+}
+
 export interface RoomState {
   name: string;
   session_id?: string;
@@ -153,6 +166,7 @@ export interface RoomState {
     rtmp: { enabled: boolean; started_at?: number; state?: HMSStreamingState };
     hls: HLSNotification;
   };
+  transcriptions?: TranscriptionNotification[];
 }
 
 export interface PeerListNotification {

packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts

@@ -7,6 +7,7 @@ import {
   HMSHLSRecording,
   HMSRoomUpdate,
   HMSSFURecording,
+  HMSTranscriptionInfo,
   HMSUpdateListener,
 } from '../../interfaces';
 import { ServerError } from '../../interfaces/internal';
@@ -26,6 +27,7 @@ import {
   RoomState,
   RTMPNotification,
   SessionInfo,
+  TranscriptionNotification,
 } from '../HMSNotifications';
 
 export class RoomUpdateManager {
@@ -93,7 +95,7 @@ export class RoomUpdateManager {
   }
 
   private onRoomState(roomNotification: RoomState) {
-    const { recording, streaming, session_id, started_at, name } = roomNotification;
+    const { recording, streaming, transcriptions, session_id, started_at, name } = roomNotification;
     const room = this.store.getRoom();
     if (!room) {
       HMSLogger.w(this.TAG, 'on room state - room not present');
@@ -112,11 +114,24 @@ export class RoomUpdateManager {
 
     room.hls = this.convertHls(streaming?.hls);
 
+    room.transcriptions = this.addTranscriptionDetail(transcriptions);
+
     room.sessionId = session_id;
     room.startedAt = convertDateNumToDate(started_at);
     this.listener?.onRoomUpdate(HMSRoomUpdate.RECORDING_STATE_UPDATED, room);
   }
 
+  private addTranscriptionDetail(transcriptions?: TranscriptionNotification[]): HMSTranscriptionInfo[] {
+    if (!transcriptions) {
+      return [];
+    }
+    return transcriptions.map((transcription: TranscriptionNotification) => {
+      return {
+        state: transcription.state,
+        mode: transcription.mode,
+      };
+    });
+  }
   private isRecordingRunning(state?: HMSRecordingState): boolean {
     if (!state) {
       return false;