Merge branch 'dev' into test/video-pause

100mslive · May 21, 2024 · 208298c · 208298c
2 parents 9456aa7 + 7b4ebde
commit 208298c
Show file tree

Hide file tree

Showing 18 changed files with 142 additions and 14 deletions.
diff --git a/packages/hms-video-store/src/IHMSActions.ts b/packages/hms-video-store/src/IHMSActions.ts
@@ -1,3 +1,4 @@
+import { TranscriptionConfig } from './interfaces/transcription-config';
 import {
   HLSConfig,
   HLSTimedMetadata,
@@ -371,6 +372,18 @@ export interface IHMSActions<T extends HMSGenericTypes = { sessionStore: Record<
    */
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
 
+  /**
+   * If you want to start transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription. Currently we have Caption mode only
+   */
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+
+  /**
+   * If you want to stop transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription you want to stop. Currently we have Caption mode only
+   */
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
+
   /**
    * @alpha
    * Used to define date range metadata in a media playlist.

diff --git a/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts b/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts
@@ -56,7 +56,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
       const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats) =>
         trackStats.jitterBufferDelay &&
         trackStats.jitterBufferEmittedCount &&
-        trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount;
+        (trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount) * 1000;
 
       const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats);
 

diff --git a/packages/hms-video-store/src/interfaces/hms.ts b/packages/hms-video-store/src/interfaces/hms.ts
@@ -12,6 +12,7 @@ import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room';
 import { RTMPRecordingConfig } from './rtmp-recording-config';
 import { HMSInteractivityCenter, HMSSessionStore } from './session-store';
 import { HMSScreenShareConfig } from './track-settings';
+import { TranscriptionConfig } from './transcription-config';
 import { HMSAudioListener, HMSConnectionQualityListener, HMSUpdateListener } from './update-listener';
 import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel';
 import { IAudioOutputManager } from '../device-manager/AudioOutputManager';
@@ -61,6 +62,8 @@ export interface HMSInterface {
    */
   startHLSStreaming(params?: HLSConfig): Promise<void>;
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
   getRecordingState(): HMSRecording | undefined;
   getRTMPState(): HMSRTMP | undefined;
   getHLSState(): HMSHLS | undefined;

diff --git a/packages/hms-video-store/src/interfaces/index.ts b/packages/hms-video-store/src/interfaces/index.ts
@@ -19,3 +19,4 @@ export * from './webrtc-stats';
 export * from './framework-info';
 export * from './get-token';
 export * from './session-store';
+export * from './transcription-config';
diff --git a/packages/hms-video-store/src/interfaces/room.ts b/packages/hms-video-store/src/interfaces/room.ts
@@ -123,6 +123,7 @@ export interface HLSVariant {
 Transcription related details
 */
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,4 +134,9 @@ export enum HMSTranscriptionMode {
 export interface HMSTranscriptionInfo {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: Date;
+  started_at?: Date;
+  updated_at?: Date;
+  stopped_at?: Date;
+  error?: HMSException;
 }
diff --git a/packages/hms-video-store/src/interfaces/transcription-config.ts b/packages/hms-video-store/src/interfaces/transcription-config.ts
@@ -0,0 +1,5 @@
+import { HMSTranscriptionMode } from './room';
+
+export interface TranscriptionConfig {
+  mode: HMSTranscriptionMode;
+}
diff --git a/packages/hms-video-store/src/interfaces/update-listener.ts b/packages/hms-video-store/src/interfaces/update-listener.ts
@@ -17,6 +17,7 @@ export enum HMSRoomUpdate {
   SERVER_RECORDING_STATE_UPDATED = 'SERVER_RECORDING_STATE_UPDATED',
   RTMP_STREAMING_STATE_UPDATED = 'RTMP_STREAMING_STATE_UPDATED',
   HLS_STREAMING_STATE_UPDATED = 'HLS_STREAMING_STATE_UPDATED',
+  TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED',
   ROOM_PEER_COUNT_UPDATED = 'ROOM_PEER_COUNT_UPDATED',
 }
 

diff --git a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts
@@ -28,6 +28,7 @@ export enum HMSNotificationMethod {
   RTMP_UPDATE = 'on-rtmp-update',
   RECORDING_UPDATE = 'on-record-update',
   HLS_UPDATE = 'on-hls-update',
+  TRANSCRIPTION_UPDATE = 'on-transcription-update',
   METADATA_CHANGE = 'on-metadata-change',
   POLL_START = 'on-poll-start',
   POLL_STOP = 'on-poll-stop',

diff --git a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts
@@ -62,6 +62,7 @@ export enum HMSStreamingState {
 }
 
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,6 +134,12 @@ export interface PeerNotification {
 export interface TranscriptionNotification {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: number;
+  started_at?: number;
+  updated_at?: number;
+  stopped_at?: number;
+  peer?: PeerNotificationInfo;
+  error?: ServerError;
 }
 
 export interface RoomState {

diff --git a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts
@@ -59,6 +59,9 @@ export class RoomUpdateManager {
       case HMSNotificationMethod.HLS_UPDATE:
         this.updateHLSStatus(notification as HLSNotification);
         break;
+      case HMSNotificationMethod.TRANSCRIPTION_UPDATE:
+        this.handleTranscriptionStatus([notification as TranscriptionNotification]);
+        break;
       default:
         break;
     }
@@ -129,6 +132,11 @@ export class RoomUpdateManager {
       return {
         state: transcription.state,
         mode: transcription.mode,
+        initialised_at: convertDateNumToDate(transcription.initialised_at),
+        started_at: convertDateNumToDate(transcription.started_at),
+        stopped_at: convertDateNumToDate(transcription.stopped_at),
+        updated_at: convertDateNumToDate(transcription.updated_at),
+        error: this.toSdkError(transcription?.error),
       };
     });
   }
@@ -200,6 +208,15 @@ export class RoomUpdateManager {
     this.listener?.onRoomUpdate(HMSRoomUpdate.HLS_STREAMING_STATE_UPDATED, room);
   }
 
+  private handleTranscriptionStatus(notification: TranscriptionNotification[]) {
+    const room = this.store.getRoom();
+    if (!room) {
+      HMSLogger.w(this.TAG, 'on transcription - room not present');
+      return;
+    }
+    room.transcriptions = this.addTranscriptionDetail(notification) || [];
+    this.listener?.onRoomUpdate(HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED, room);
+  }
   private convertHls(hlsNotification?: HLSNotification) {
     const isInitialised =
       hlsNotification?.variants && hlsNotification.variants.length > 0

diff --git a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts
@@ -701,6 +701,14 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     await this.sdk.stopHLSStreaming(params);
   }
 
+  async startTranscription(params: sdkTypes.TranscriptionConfig) {
+    await this.sdk.startTranscription(params);
+  }
+
+  async stopTranscription(params: sdkTypes.TranscriptionConfig): Promise<void> {
+    await this.sdk.stopTranscription(params);
+  }
+
   async sendHLSTimedMetadata(metadataList: sdkTypes.HLSTimedMetadata[]): Promise<void> {
     await this.sdk.sendHLSTimedMetadata(metadataList);
   }

diff --git a/packages/hms-video-store/src/reactive-store/adapter.ts b/packages/hms-video-store/src/reactive-store/adapter.ts
@@ -139,10 +139,11 @@ export class SDKToHMS {
   }
 
   static convertRoom(sdkRoom: sdkTypes.HMSRoom, sdkLocalPeerId?: string): Partial<HMSRoom> {
-    const { recording, rtmp, hls } = SDKToHMS.convertRecordingStreamingState(
-      sdkRoom?.recording,
-      sdkRoom?.rtmp,
-      sdkRoom?.hls,
+    const { recording, rtmp, hls, transcriptions } = SDKToHMS.convertRecordingStreamingState(
+      sdkRoom.recording,
+      sdkRoom.rtmp,
+      sdkRoom.hls,
+      sdkRoom.transcriptions,
     );
     return {
       id: sdkRoom.id,
@@ -151,6 +152,7 @@ export class SDKToHMS {
       recording,
       rtmp,
       hls,
+      transcriptions,
       sessionId: sdkRoom.sessionId,
       startedAt: sdkRoom.startedAt,
       joinedAt: sdkRoom.joinedAt,

diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts
@@ -48,7 +48,7 @@ import { HMSPreviewListener } from '../interfaces/preview-listener';
 import { RTMPRecordingConfig } from '../interfaces/rtmp-recording-config';
 import InitialSettings from '../interfaces/settings';
 import { HMSAudioListener, HMSPeerUpdate, HMSTrackUpdate, HMSUpdateListener } from '../interfaces/update-listener';
-import { PlaylistManager } from '../internal';
+import { PlaylistManager, TranscriptionConfig } from '../internal';
 import { HMSLocalStream } from '../media/streams/HMSLocalStream';
 import {
   HMSLocalAudioTrack,
@@ -70,6 +70,7 @@ import {
   HLSTimedMetadataParams,
   HLSVariant,
   StartRTMPOrRecordingRequestParams,
+  StartTranscriptionRequestParams,
 } from '../signal/interfaces';
 import HMSTransport from '../transport';
 import ITransportObserver from '../transport/ITransportObserver';
@@ -1012,6 +1013,35 @@ export class HMSSdk implements HMSInterface {
     await this.transport?.signal.stopHLSStreaming();
   }
 
+  async startTranscription(params: TranscriptionConfig) {
+    if (!this.localPeer) {
+      throw ErrorFactory.GenericErrors.NotConnected(
+        HMSAction.VALIDATION,
+        'No local peer present, cannot start transcriptions',
+      );
+    }
+    const transcriptionParams: StartTranscriptionRequestParams = {
+      mode: params.mode,
+    };
+    await this.transport?.signal.startTranscription(transcriptionParams);
+  }
+
+  async stopTranscription(params: TranscriptionConfig) {
+    if (!this.localPeer) {
+      throw ErrorFactory.GenericErrors.NotConnected(
+        HMSAction.VALIDATION,
+        'No local peer present, cannot stop transcriptions',
+      );
+    }
+    if (!params) {
+      throw ErrorFactory.GenericErrors.Signalling(HMSAction.VALIDATION, 'No mode is passed to stop the transcription');
+    }
+    const transcriptionParams: StartTranscriptionRequestParams = {
+      mode: params.mode,
+    };
+    await this.transport?.signal.stopTranscription(transcriptionParams);
+  }
+
   async sendHLSTimedMetadata(metadataList: HLSTimedMetadata[]) {
     this.validateJoined('sendHLSTimedMetadata');
     if (metadataList.length > 0) {

diff --git a/packages/hms-video-store/src/signal/interfaces/superpowers.ts b/packages/hms-video-store/src/signal/interfaces/superpowers.ts
@@ -1,4 +1,4 @@
-import { HMSTrackSource } from '../..';
+import { HMSTrackSource, HMSTranscriptionMode } from '../..';
 import { HLSTimedMetadata, RTMPRecordingResolution } from '../../interfaces';
 
 /**
@@ -57,6 +57,9 @@ export interface UpdatePeerRequestParams {
   data?: string;
 }
 
+export interface StartTranscriptionRequestParams {
+  mode: HMSTranscriptionMode;
+}
 export interface SetSessionMetadataParams {
   key?: string;
   data: any;

diff --git a/packages/hms-video-store/src/signal/jsonrpc/index.ts b/packages/hms-video-store/src/signal/jsonrpc/index.ts
@@ -60,6 +60,7 @@ import {
   SetSessionMetadataParams,
   SetSessionMetadataResponse,
   StartRTMPOrRecordingRequestParams,
+  StartTranscriptionRequestParams,
   Track,
   TrackUpdateRequestParams,
   UpdatePeerRequestParams,
@@ -364,6 +365,14 @@ export default class JsonRpcSignal {
     await this.call(HMSSignalMethod.STOP_HLS_STREAMING, { ...params });
   }
 
+  async startTranscription(params: StartTranscriptionRequestParams) {
+    await this.call(HMSSignalMethod.START_TRANSCRIPTION, { ...params });
+  }
+
+  async stopTranscription(params: StartTranscriptionRequestParams) {
+    await this.call(HMSSignalMethod.STOP_TRANSCRIPTION, { ...params });
+  }
+
   async sendHLSTimedMetadata(params?: HLSTimedMetadataParams): Promise<void> {
     await this.call(HMSSignalMethod.HLS_TIMED_METADATA, { ...params });
   }

diff --git a/packages/hms-video-store/src/signal/jsonrpc/models.ts b/packages/hms-video-store/src/signal/jsonrpc/models.ts
@@ -39,6 +39,8 @@ export enum HMSSignalMethod {
   UPDATE_PEER_METADATA = 'peer-update',
   START_HLS_STREAMING = 'hls-start',
   STOP_HLS_STREAMING = 'hls-stop',
+  START_TRANSCRIPTION = 'transcription-start',
+  STOP_TRANSCRIPTION = 'transcription-stop',
   HLS_TIMED_METADATA = 'hls-timed-metadata',
   SET_METADATA = 'set-metadata',
   GET_METADATA = 'get-metadata',

diff --git a/packages/roomkit-react/src/Prebuilt/components/Chat/Chat.tsx b/packages/roomkit-react/src/Prebuilt/components/Chat/Chat.tsx
@@ -10,17 +10,20 @@ import { Box, Flex } from '../../../Layout';
 import { config as cssConfig } from '../../../Theme';
 // @ts-ignore: No implicit any
 import { EmojiReaction } from '../EmojiReaction';
+import { MoreSettings } from '../MoreSettings/MoreSettings';
+import { RaiseHand } from '../RaiseHand';
 import { ChatBody } from './ChatBody';
 import { ChatFooter } from './ChatFooter';
 import { ChatBlocked, ChatPaused } from './ChatStates';
 import { PinnedMessage } from './PinnedMessage';
 import { useRoomLayoutConferencingScreen } from '../../provider/roomLayoutProvider/hooks/useRoomLayoutScreen';
 import { useSidepaneResetOnLayoutUpdate } from '../AppData/useSidepaneResetOnLayoutUpdate';
+import { useIsPeerBlacklisted } from '../hooks/useChatBlacklist';
 import { useLandscapeHLSStream, useMobileHLSStream } from '../../common/hooks';
 import { SESSION_STORE_KEY, SIDE_PANE_OPTIONS } from '../../common/constants';
 
 export const Chat = () => {
-  const { elements } = useRoomLayoutConferencingScreen();
+  const { elements, screenType } = useRoomLayoutConferencingScreen();
   const listRef = useRef<VariableSizeList | null>(null);
   const hmsActions = useHMSActions();
   const vanillaStore = useHMSVanillaStore();
@@ -29,6 +32,7 @@ export const Chat = () => {
   const isMobileHLSStream = useMobileHLSStream();
   const isLandscapeStream = useLandscapeHLSStream();
   useSidepaneResetOnLayoutUpdate('chat', SIDE_PANE_OPTIONS.CHAT);
+  const isLocalPeerBlacklisted = useIsPeerBlacklisted({ local: true });
 
   const scrollToBottom = useCallback(
     (unreadCount = 0) => {
@@ -57,20 +61,27 @@ export const Chat = () => {
     >
       {isMobile && elements?.chat?.is_overlay && !streaming ? null : <PinnedMessage />}
       <ChatBody ref={listRef} scrollToBottom={scrollToBottom} />
-
-      <ChatPaused />
-      <ChatBlocked />
+      <Flex align="center" css={{ w: '100%', gap: '$2' }}>
+        <ChatPaused />
+        <ChatBlocked />
+        {streaming && (!isChatEnabled || isLocalPeerBlacklisted) && (
+          <>
+            <RaiseHand css={{ bg: '$surface_default' }} />
+            <MoreSettings elements={elements} screenType={screenType} />
+          </>
+        )}
+      </Flex>
       {isMobile && elements?.chat?.is_overlay && !streaming ? <PinnedMessage /> : null}
       {isChatEnabled ? (
         <ChatFooter onSend={scrollToBottom}>
           <NewMessageIndicator scrollToBottom={scrollToBottom} listRef={listRef} />
         </ChatFooter>
       ) : null}
-      {(isMobileHLSStream || isLandscapeStream) && (
+      {streaming && (
         <Box
           css={{
             position: 'absolute',
-            ...match({ isLandscapeStream, isMobileHLSStream, isChatEnabled })
+            ...match({ isLandscapeStream, isMobileHLSStream, isChatEnabled, isLocalPeerBlacklisted })
               .with(
                 {
                   isLandscapeStream: true,
@@ -96,13 +107,22 @@ export const Chat = () => {
                 {
                   isMobileHLSStream: true,
                   isChatEnabled: true,
+                  isLocalPeerBlacklisted: false,
                 },
                 () => ({ bottom: '$17', right: '$8' }),
               )
               .with(
                 {
                   isLandscapeStream: false,
                   isChatEnabled: true,
+                  isLocalPeerBlacklisted: true,
+                },
+                () => ({ bottom: '$18', right: '$8' }),
+              )
+              .with(
+                {
+                  isMobileHLSStream: true,
+                  isLocalPeerBlacklisted: true,
                 },
                 () => ({ bottom: '$20', right: '$8' }),
               )

diff --git a/packages/roomkit-react/src/Prebuilt/components/Chat/ChatStates.tsx b/packages/roomkit-react/src/Prebuilt/components/Chat/ChatStates.tsx
@@ -30,7 +30,7 @@ export const ChatPaused = () => {
     <Flex
       align="center"
       justify="between"
-      css={{ borderRadius: '$1', bg: '$surface_default', p: '$4 $4 $4 $8', w: '100%' }}
+      css={{ borderRadius: '$1', bg: '$surface_default', p: '$2 $4 $2 $8', w: '100%' }}
     >
       <Box>
         <Text variant="sm" css={{ fontWeight: '$semiBold', color: '$on_surface_high' }}>