Merge branch 'WEB-2809-start-stop-transcription' into WEB-2812-ui-sta…

…rt-stop-transcription

packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts

@@ -56,7 +56,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
       const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats) =>
         trackStats.jitterBufferDelay &&
         trackStats.jitterBufferEmittedCount &&
-        trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount;
+        (trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount) * 1000;
 
       const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats);
 

packages/hms-video-store/src/index.ts

@@ -53,6 +53,7 @@ export type {
   HMSQuizLeaderboardResponse,
   HMSQuizLeaderboardSummary,
   HMSTranscriptionInfo,
+  HMSICEServer,
 } from './internal';
 
 export { EventBus } from './events/EventBus';

packages/hms-video-store/src/interfaces/config.ts

@@ -5,6 +5,13 @@ import InitialSettings from './settings';
  * @link https://docs.100ms.live/javascript/v2/features/preview
  * @link https://docs.100ms.live/javascript/v2/features/join
  */
+
+export type HMSICEServer = {
+  urls: string[];
+  userName?: string;
+  password?: string;
+};
+
 export interface HMSConfig {
   /**
    * the name of the peer, can be later accessed via peer.name and can also be changed mid call.
@@ -53,10 +60,14 @@ export interface HMSConfig {
    */
   autoManageVideo?: boolean;
   /**
-   * if this flag is enabled, wake lock will be acquired automatically(if supported) when joining the room, so the device
+   * if this flag is enabled, wake lock will be acquired automatically (if supported) when joining the room, so the device
    * will be kept awake.
    */
   autoManageWakeLock?: boolean;
+  /**
+   * use custom STUN/TURN servers for media connection (advanced)
+   */
+  iceServers?: HMSICEServer[];
 }
 
 export interface HMSMidCallPreviewConfig {

packages/hms-video-store/src/interfaces/room.ts

@@ -123,6 +123,7 @@ export interface HLSVariant {
 Transcription related details
 */
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',

packages/hms-video-store/src/notification-manager/HMSNotifications.ts

@@ -62,6 +62,7 @@ export enum HMSStreamingState {
 }
 
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',

packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts

@@ -128,7 +128,7 @@ export class RoomUpdateManager {
     if (!transcriptions) {
       return [];
     }
-    const output = transcriptions.map((transcription: TranscriptionNotification) => {
+    return transcriptions.map((transcription: TranscriptionNotification) => {
       return {
         state: transcription.state,
         mode: transcription.mode,
@@ -139,7 +139,6 @@ export class RoomUpdateManager {
         error: this.toSdkError(transcription?.error),
       };
     });
-    return output;
   }
   private isRecordingRunning(state?: HMSRecordingState): boolean {
     if (!state) {

packages/hms-video-store/src/reactive-store/HMSSDKActions.ts

@@ -701,11 +701,11 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     await this.sdk.stopHLSStreaming(params);
   }
 
-  async startTranscription(params?: sdkTypes.TranscriptionConfig) {
+  async startTranscription(params: sdkTypes.TranscriptionConfig) {
     await this.sdk.startTranscription(params);
   }
 
-  async stopTranscription(params?: sdkTypes.TranscriptionConfig): Promise<void> {
+  async stopTranscription(params: sdkTypes.TranscriptionConfig): Promise<void> {
     await this.sdk.stopTranscription(params);
   }
 

packages/hms-video-store/src/sdk/index.ts

@@ -425,6 +425,7 @@ export class HMSSdk implements HMSInterface {
           this.localPeer!.peerId,
           { name: config.userName, metaData: config.metaData || '' },
           config.autoVideoSubscribe,
+          config.iceServers,
         )
         .then((initConfig: InitConfig | void) => {
           initSuccessful = true;
@@ -566,6 +567,7 @@ export class HMSSdk implements HMSInterface {
         { name: config.userName, metaData: config.metaData! },
         config.initEndpoint!,
         config.autoVideoSubscribe,
+        config.iceServers,
       );
       HMSLogger.d(this.TAG, `✅ Joined room ${roomId}`);
       this.analyticsTimer.start(TimedEvent.PEER_LIST);
@@ -1011,27 +1013,24 @@ export class HMSSdk implements HMSInterface {
     await this.transport?.signal.stopHLSStreaming();
   }
 
-  async startTranscription(params?: TranscriptionConfig) {
+  async startTranscription(params: TranscriptionConfig) {
     if (!this.localPeer) {
       throw ErrorFactory.GenericErrors.NotConnected(
         HMSAction.VALIDATION,
-        'No local peer present, cannot start HLS streaming',
+        'No local peer present, cannot start transcriptions',
       );
     }
-    if (!params) {
-      throw ErrorFactory.GenericErrors.Signalling(HMSAction.VALIDATION, 'No mode is passed to start the transcription');
-    }
     const transcriptionParams: StartTranscriptionRequestParams = {
       mode: params.mode,
     };
     await this.transport?.signal.startTranscription(transcriptionParams);
   }
 
-  async stopTranscription(params?: TranscriptionConfig) {
+  async stopTranscription(params: TranscriptionConfig) {
     if (!this.localPeer) {
       throw ErrorFactory.GenericErrors.NotConnected(
         HMSAction.VALIDATION,
-        'No local peer present, cannot stop HLS streaming',
+        'No local peer present, cannot stop transcriptions',
       );
     }
     if (!params) {

packages/hms-video-store/src/signal/init/index.ts

@@ -1,6 +1,8 @@
 import { InitConfig } from './models';
 import { ErrorFactory } from '../../error/ErrorFactory';
 import { HMSAction } from '../../error/HMSAction';
+import { HMSICEServer } from '../../interfaces';
+import { transformIceServerConfig } from '../../utils/ice-server-config';
 import HMSLogger from '../../utils/logger';
 
 const TAG = '[InitService]';
@@ -26,26 +28,26 @@ export default class InitService {
     userAgent,
     initEndpoint = 'https://prod-init.100ms.live',
     region = '',
+    iceServers,
   }: {
     token: string;
     peerId: string;
     userAgent: string;
     initEndpoint?: string;
     region?: string;
+    iceServers?: HMSICEServer[];
   }): Promise<InitConfig> {
     HMSLogger.d(TAG, `fetchInitConfig: initEndpoint=${initEndpoint} token=${token} peerId=${peerId} region=${region} `);
     const url = getUrl(initEndpoint, peerId, userAgent, region);
     try {
       const response = await fetch(url, {
-        headers: {
-          Authorization: `Bearer ${token}`,
-        },
+        headers: { Authorization: `Bearer ${token}` },
       });
       try {
         const config = await response.clone().json();
         this.handleError(response, config);
         HMSLogger.d(TAG, `config is ${JSON.stringify(config, null, 2)}`);
-        return transformInitConfig(config);
+        return transformInitConfig(config, iceServers);
       } catch (err) {
         const text = await response.text();
         HMSLogger.e(TAG, 'json error', (err as Error).message, text);
@@ -78,9 +80,12 @@ export function getUrl(endpoint: string, peerId: string, userAgent: string, regi
   }
 }
 
-export function transformInitConfig(config: any): InitConfig {
+export function transformInitConfig(config: any, iceServers?: HMSICEServer[]): InitConfig {
   return {
     ...config,
-    rtcConfiguration: { ...config.rtcConfiguration, iceServers: config.rtcConfiguration?.ice_servers },
+    rtcConfiguration: {
+      ...config.rtcConfiguration,
+      iceServers: transformIceServerConfig(config.rtcConfiguration?.ice_servers, iceServers),
+    },
   };
 }

packages/hms-video-store/src/transport/index.ts

@@ -23,7 +23,7 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSAction } from '../error/HMSAction';
 import { HMSException } from '../error/HMSException';
 import { EventBus } from '../events/EventBus';
-import { HMSRole } from '../interfaces';
+import { HMSICEServer, HMSRole } from '../interfaces';
 import { HMSLocalStream } from '../media/streams/HMSLocalStream';
 import { HMSLocalTrack, HMSLocalVideoTrack, HMSTrack } from '../media/tracks';
 import { TrackState } from '../notification-manager';
@@ -397,8 +397,9 @@ export default class HMSTransport {
     peerId: string,
     customData: { name: string; metaData: string },
     autoSubscribeVideo = false,
+    iceServers?: HMSICEServer[],
   ): Promise<InitConfig | void> {
-    const initConfig = await this.connect(token, endpoint, peerId, customData, autoSubscribeVideo);
+    const initConfig = await this.connect(token, endpoint, peerId, customData, autoSubscribeVideo, iceServers);
     this.state = TransportState.Preview;
     this.observer.onStateChange(this.state);
     return initConfig;
@@ -410,11 +411,12 @@ export default class HMSTransport {
     customData: { name: string; metaData: string },
     initEndpoint: string,
     autoSubscribeVideo = false,
+    iceServers?: HMSICEServer[],
   ): Promise<void> {
     HMSLogger.d(TAG, 'join: started ⏰');
     try {
       if (!this.signal.isConnected || !this.initConfig) {
-        await this.connect(authToken, initEndpoint, peerId, customData, autoSubscribeVideo);
+        await this.connect(authToken, initEndpoint, peerId, customData, autoSubscribeVideo, iceServers);
       }
 
       this.validateNotDisconnected('connect');
@@ -447,6 +449,7 @@ export default class HMSTransport {
     peerId: string,
     customData: { name: string; metaData: string },
     autoSubscribeVideo = false,
+    iceServers?: HMSICEServer[],
   ): Promise<InitConfig | void> {
     this.setTransportStateForConnect();
     this.joinParameters = new JoinParameters(
@@ -456,9 +459,10 @@ export default class HMSTransport {
       customData.metaData,
       endpoint,
       autoSubscribeVideo,
+      iceServers,
     );
     try {
-      const response = await this.internalConnect(token, endpoint, peerId);
+      const response = await this.internalConnect(token, endpoint, peerId, iceServers);
       return response;
     } catch (error) {
       const shouldRetry =
@@ -474,7 +478,7 @@ export default class HMSTransport {
 
       if (shouldRetry) {
         const task = async () => {
-          await this.internalConnect(token, endpoint, peerId);
+          await this.internalConnect(token, endpoint, peerId, iceServers);
           return Boolean(this.initConfig && this.initConfig.endpoint);
         };
 
@@ -898,7 +902,7 @@ export default class HMSTransport {
     }
   }
 
-  private async internalConnect(token: string, initEndpoint: string, peerId: string) {
+  private async internalConnect(token: string, initEndpoint: string, peerId: string, iceServers?: HMSICEServer[]) {
     HMSLogger.d(TAG, 'connect: started ⏰');
     const connectRequestedAt = new Date();
     try {
@@ -908,6 +912,7 @@ export default class HMSTransport {
         peerId,
         userAgent: this.store.getUserAgent(),
         initEndpoint,
+        iceServers,
       });
       const room = this.store.getRoom();
       if (room) {
@@ -1093,6 +1098,7 @@ export default class HMSTransport {
         this.joinParameters!.authToken,
         this.joinParameters!.endpoint,
         this.joinParameters!.peerId,
+        this.joinParameters!.iceServers,
       );
     }
 

packages/hms-video-store/src/transport/models/JoinParameters.ts

@@ -1,3 +1,5 @@
+import { HMSICEServer } from '../../interfaces';
+
 export class JoinParameters {
   constructor(
     public authToken: string,
@@ -6,5 +8,6 @@ export class JoinParameters {
     public data: string = '',
     public endpoint: string = 'https://prod-init.100ms.live/init',
     public autoSubscribeVideo: boolean = false,
+    public iceServers?: HMSICEServer[],
   ) {}
 }

packages/hms-video-store/src/utils/ice-server-config.ts

@@ -0,0 +1,11 @@
+import { HMSICEServer } from '../interfaces';
+
+export const transformIceServerConfig = (defaultConfig?: RTCIceServer[], iceServers?: HMSICEServer[]) => {
+  if (!iceServers || iceServers.length === 0) {
+    return defaultConfig;
+  }
+  const transformedIceServers = iceServers.map(server => {
+    return { urls: server.urls, credentialType: 'password', credential: server.password, username: server.userName };
+  });
+  return transformedIceServers;
+};

packages/hms-whiteboard/src/hooks/StoreClient.ts

@@ -1,6 +1,7 @@
 import { GrpcWebFetchTransport } from '@protobuf-ts/grpcweb-transport';
 import { Value_Type } from '../grpc/sessionstore';
 import { StoreClient } from '../grpc/sessionstore.client';
+import { OPEN_WAIT_TIMEOUT } from '../utils';
 
 interface OpenCallbacks<T> {
   handleOpen: (values: T[]) => void;
@@ -31,28 +32,23 @@ export class SessionStore<T> {
       },
       { abort: this.abortController.signal },
     );
-    /**
-     * on open, get key count to call handleOpen with the pre-existing values from the store
-     * retry if getKeysCount is called before open call is completed
-     */
-    const keyCount = await this.retryForOpen(this.getKeysCount.bind(this));
     const initialValues: T[] = [];
+    let initialised = false;
 
-    if (!keyCount) {
-      handleOpen([]);
-    }
+    // on open, wait to call handleOpen with the pre-existing values from the store
+    setTimeout(() => {
+      handleOpen(initialValues);
+      initialised = true;
+    }, OPEN_WAIT_TIMEOUT);
 
     call.responses.onMessage(message => {
       if (message.value) {
         if (message.value?.data.oneofKind === 'str') {
           const record = JSON.parse(message.value.data.str) as T;
-          if (initialValues.length === keyCount) {
+          if (initialised) {
             handleChange(message.key, record);
           } else {
             initialValues.push(record);
-            if (initialValues.length === keyCount) {
-              handleOpen(initialValues);
-            }
           }
         }
       } else {
@@ -103,16 +99,4 @@ export class SessionStore<T> {
   delete(key: string) {
     return this.storeClient.delete({ key });
   }
-
-  private async retryForOpen<T>(fn: () => Promise<T>, retries = 3): Promise<T> {
-    try {
-      return await fn();
-    } catch (error) {
-      const shouldRetry = (error as Error).message.includes('peer not found') && retries > 0;
-      if (!shouldRetry) {
-        return Promise.reject(error);
-      }
-      return await this.retryForOpen(fn, retries - 1);
-    }
-  }
 }