From af3c5ceb7790940bd216f786c616826c44e78b73 Mon Sep 17 00:00:00 2001 From: Amar Bathwal <110378139+amar-1995@users.noreply.github.com> Date: Tue, 21 May 2024 15:22:14 +0530 Subject: [PATCH] feat: start/stop events for transcriptions in sdk --- packages/hms-video-store/src/IHMSActions.ts | 13 ++++++++ .../hms-video-store/src/interfaces/hms.ts | 3 ++ .../hms-video-store/src/interfaces/index.ts | 1 + .../hms-video-store/src/interfaces/room.ts | 6 ++++ .../src/interfaces/transcription-config.ts | 5 +++ .../src/interfaces/update-listener.ts | 1 + .../HMSNotificationMethod.ts | 1 + .../notification-manager/HMSNotifications.ts | 7 ++++ .../managers/RoomUpdateManager.ts | 17 ++++++++++ .../src/reactive-store/HMSSDKActions.ts | 8 +++++ .../src/reactive-store/adapter.ts | 10 +++--- packages/hms-video-store/src/sdk/index.ts | 32 ++++++++++++++++++- .../src/signal/interfaces/superpowers.ts | 5 ++- .../src/signal/jsonrpc/index.ts | 9 ++++++ .../src/signal/jsonrpc/models.ts | 2 ++ 15 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 packages/hms-video-store/src/interfaces/transcription-config.ts diff --git a/packages/hms-video-store/src/IHMSActions.ts b/packages/hms-video-store/src/IHMSActions.ts index ccdd3635f4..625903f109 100644 --- a/packages/hms-video-store/src/IHMSActions.ts +++ b/packages/hms-video-store/src/IHMSActions.ts @@ -1,3 +1,4 @@ +import { TranscriptionConfig } from './interfaces/transcription-config'; import { HLSConfig, HLSTimedMetadata, @@ -371,6 +372,18 @@ export interface IHMSActions; + /** + * If you want to start transcriptions(Closed Caption). + * @param params.mode This is the mode which represent the type of transcription. Currently we have Caption mode only + */ + startTranscription(params: TranscriptionConfig): Promise; + + /** + * If you want to stop transcriptions(Closed Caption). + * @param params.mode This is the mode which represent the type of transcription you want to stop. Currently we have Caption mode only + */ + stopTranscription(params: TranscriptionConfig): Promise; + /** * @alpha * Used to define date range metadata in a media playlist. diff --git a/packages/hms-video-store/src/interfaces/hms.ts b/packages/hms-video-store/src/interfaces/hms.ts index 22bfe25ebc..14b983ab6c 100644 --- a/packages/hms-video-store/src/interfaces/hms.ts +++ b/packages/hms-video-store/src/interfaces/hms.ts @@ -12,6 +12,7 @@ import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room'; import { RTMPRecordingConfig } from './rtmp-recording-config'; import { HMSInteractivityCenter, HMSSessionStore } from './session-store'; import { HMSScreenShareConfig } from './track-settings'; +import { TranscriptionConfig } from './transcription-config'; import { HMSAudioListener, HMSConnectionQualityListener, HMSUpdateListener } from './update-listener'; import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel'; import { IAudioOutputManager } from '../device-manager/AudioOutputManager'; @@ -61,6 +62,8 @@ export interface HMSInterface { */ startHLSStreaming(params?: HLSConfig): Promise; stopHLSStreaming(params?: HLSConfig): Promise; + startTranscription(params: TranscriptionConfig): Promise; + stopTranscription(params: TranscriptionConfig): Promise; getRecordingState(): HMSRecording | undefined; getRTMPState(): HMSRTMP | undefined; getHLSState(): HMSHLS | undefined; diff --git a/packages/hms-video-store/src/interfaces/index.ts b/packages/hms-video-store/src/interfaces/index.ts index 8bfb7576e5..f18940854f 100644 --- a/packages/hms-video-store/src/interfaces/index.ts +++ b/packages/hms-video-store/src/interfaces/index.ts @@ -19,3 +19,4 @@ export * from './webrtc-stats'; export * from './framework-info'; export * from './get-token'; export * from './session-store'; +export * from './transcription-config'; diff --git a/packages/hms-video-store/src/interfaces/room.ts b/packages/hms-video-store/src/interfaces/room.ts index 66c06dc19b..db861e0725 100644 --- a/packages/hms-video-store/src/interfaces/room.ts +++ b/packages/hms-video-store/src/interfaces/room.ts @@ -123,6 +123,7 @@ export interface HLSVariant { Transcription related details */ export enum HMSTranscriptionState { + INITIALISED = 'initialised', STARTED = 'started', STOPPED = 'stopped', FAILED = 'failed', @@ -133,4 +134,9 @@ export enum HMSTranscriptionMode { export interface HMSTranscriptionInfo { state?: HMSTranscriptionState; mode?: HMSTranscriptionMode; + initialised_at?: Date; + started_at?: Date; + updated_at?: Date; + stopped_at?: Date; + error?: HMSException; } diff --git a/packages/hms-video-store/src/interfaces/transcription-config.ts b/packages/hms-video-store/src/interfaces/transcription-config.ts new file mode 100644 index 0000000000..9294bfc851 --- /dev/null +++ b/packages/hms-video-store/src/interfaces/transcription-config.ts @@ -0,0 +1,5 @@ +import { HMSTranscriptionMode } from './room'; + +export interface TranscriptionConfig { + mode: HMSTranscriptionMode; +} diff --git a/packages/hms-video-store/src/interfaces/update-listener.ts b/packages/hms-video-store/src/interfaces/update-listener.ts index c870e29fec..ef1b385dc2 100644 --- a/packages/hms-video-store/src/interfaces/update-listener.ts +++ b/packages/hms-video-store/src/interfaces/update-listener.ts @@ -17,6 +17,7 @@ export enum HMSRoomUpdate { SERVER_RECORDING_STATE_UPDATED = 'SERVER_RECORDING_STATE_UPDATED', RTMP_STREAMING_STATE_UPDATED = 'RTMP_STREAMING_STATE_UPDATED', HLS_STREAMING_STATE_UPDATED = 'HLS_STREAMING_STATE_UPDATED', + TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED', ROOM_PEER_COUNT_UPDATED = 'ROOM_PEER_COUNT_UPDATED', } diff --git a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts index 72c5d06921..28616e29da 100644 --- a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts +++ b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts @@ -28,6 +28,7 @@ export enum HMSNotificationMethod { RTMP_UPDATE = 'on-rtmp-update', RECORDING_UPDATE = 'on-record-update', HLS_UPDATE = 'on-hls-update', + TRANSCRIPTION_UPDATE = 'on-transcription-update', METADATA_CHANGE = 'on-metadata-change', POLL_START = 'on-poll-start', POLL_STOP = 'on-poll-stop', diff --git a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts index 8130e93e2d..34818e72e3 100644 --- a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts +++ b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts @@ -62,6 +62,7 @@ export enum HMSStreamingState { } export enum HMSTranscriptionState { + INITIALISED = 'initialised', STARTED = 'started', STOPPED = 'stopped', FAILED = 'failed', @@ -133,6 +134,12 @@ export interface PeerNotification { export interface TranscriptionNotification { state?: HMSTranscriptionState; mode?: HMSTranscriptionMode; + initialised_at?: number; + started_at?: number; + updated_at?: number; + stopped_at?: number; + peer?: PeerNotificationInfo; + error?: ServerError; } export interface RoomState { diff --git a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts index 7f4c42b86b..84ec21d0ac 100644 --- a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts +++ b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts @@ -59,6 +59,9 @@ export class RoomUpdateManager { case HMSNotificationMethod.HLS_UPDATE: this.updateHLSStatus(notification as HLSNotification); break; + case HMSNotificationMethod.TRANSCRIPTION_UPDATE: + this.handleTranscriptionStatus([notification as TranscriptionNotification]); + break; default: break; } @@ -129,6 +132,11 @@ export class RoomUpdateManager { return { state: transcription.state, mode: transcription.mode, + initialised_at: convertDateNumToDate(transcription.initialised_at), + started_at: convertDateNumToDate(transcription.started_at), + stopped_at: convertDateNumToDate(transcription.stopped_at), + updated_at: convertDateNumToDate(transcription.updated_at), + error: this.toSdkError(transcription?.error), }; }); } @@ -200,6 +208,15 @@ export class RoomUpdateManager { this.listener?.onRoomUpdate(HMSRoomUpdate.HLS_STREAMING_STATE_UPDATED, room); } + private handleTranscriptionStatus(notification: TranscriptionNotification[]) { + const room = this.store.getRoom(); + if (!room) { + HMSLogger.w(this.TAG, 'on transcription - room not present'); + return; + } + room.transcriptions = this.addTranscriptionDetail(notification) || []; + this.listener?.onRoomUpdate(HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED, room); + } private convertHls(hlsNotification?: HLSNotification) { const isInitialised = hlsNotification?.variants && hlsNotification.variants.length > 0 diff --git a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts index 57db483bad..2fbff74954 100644 --- a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts +++ b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts @@ -701,6 +701,14 @@ export class HMSSDKActions { + await this.sdk.stopTranscription(params); + } + async sendHLSTimedMetadata(metadataList: sdkTypes.HLSTimedMetadata[]): Promise { await this.sdk.sendHLSTimedMetadata(metadataList); } diff --git a/packages/hms-video-store/src/reactive-store/adapter.ts b/packages/hms-video-store/src/reactive-store/adapter.ts index d16e75e830..717b93bb63 100644 --- a/packages/hms-video-store/src/reactive-store/adapter.ts +++ b/packages/hms-video-store/src/reactive-store/adapter.ts @@ -139,10 +139,11 @@ export class SDKToHMS { } static convertRoom(sdkRoom: sdkTypes.HMSRoom, sdkLocalPeerId?: string): Partial { - const { recording, rtmp, hls } = SDKToHMS.convertRecordingStreamingState( - sdkRoom?.recording, - sdkRoom?.rtmp, - sdkRoom?.hls, + const { recording, rtmp, hls, transcriptions } = SDKToHMS.convertRecordingStreamingState( + sdkRoom.recording, + sdkRoom.rtmp, + sdkRoom.hls, + sdkRoom.transcriptions, ); return { id: sdkRoom.id, @@ -151,6 +152,7 @@ export class SDKToHMS { recording, rtmp, hls, + transcriptions, sessionId: sdkRoom.sessionId, startedAt: sdkRoom.startedAt, joinedAt: sdkRoom.joinedAt, diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts index 19d0ef83af..2e4679323f 100644 --- a/packages/hms-video-store/src/sdk/index.ts +++ b/packages/hms-video-store/src/sdk/index.ts @@ -48,7 +48,7 @@ import { HMSPreviewListener } from '../interfaces/preview-listener'; import { RTMPRecordingConfig } from '../interfaces/rtmp-recording-config'; import InitialSettings from '../interfaces/settings'; import { HMSAudioListener, HMSPeerUpdate, HMSTrackUpdate, HMSUpdateListener } from '../interfaces/update-listener'; -import { PlaylistManager } from '../internal'; +import { PlaylistManager, TranscriptionConfig } from '../internal'; import { HMSLocalStream } from '../media/streams/HMSLocalStream'; import { HMSLocalAudioTrack, @@ -70,6 +70,7 @@ import { HLSTimedMetadataParams, HLSVariant, StartRTMPOrRecordingRequestParams, + StartTranscriptionRequestParams, } from '../signal/interfaces'; import HMSTransport from '../transport'; import ITransportObserver from '../transport/ITransportObserver'; @@ -1012,6 +1013,35 @@ export class HMSSdk implements HMSInterface { await this.transport?.signal.stopHLSStreaming(); } + async startTranscription(params: TranscriptionConfig) { + if (!this.localPeer) { + throw ErrorFactory.GenericErrors.NotConnected( + HMSAction.VALIDATION, + 'No local peer present, cannot start transcriptions', + ); + } + const transcriptionParams: StartTranscriptionRequestParams = { + mode: params.mode, + }; + await this.transport?.signal.startTranscription(transcriptionParams); + } + + async stopTranscription(params: TranscriptionConfig) { + if (!this.localPeer) { + throw ErrorFactory.GenericErrors.NotConnected( + HMSAction.VALIDATION, + 'No local peer present, cannot stop transcriptions', + ); + } + if (!params) { + throw ErrorFactory.GenericErrors.Signalling(HMSAction.VALIDATION, 'No mode is passed to stop the transcription'); + } + const transcriptionParams: StartTranscriptionRequestParams = { + mode: params.mode, + }; + await this.transport?.signal.stopTranscription(transcriptionParams); + } + async sendHLSTimedMetadata(metadataList: HLSTimedMetadata[]) { this.validateJoined('sendHLSTimedMetadata'); if (metadataList.length > 0) { diff --git a/packages/hms-video-store/src/signal/interfaces/superpowers.ts b/packages/hms-video-store/src/signal/interfaces/superpowers.ts index b96e5661bb..a72c602f0a 100644 --- a/packages/hms-video-store/src/signal/interfaces/superpowers.ts +++ b/packages/hms-video-store/src/signal/interfaces/superpowers.ts @@ -1,4 +1,4 @@ -import { HMSTrackSource } from '../..'; +import { HMSTrackSource, HMSTranscriptionMode } from '../..'; import { HLSTimedMetadata, RTMPRecordingResolution } from '../../interfaces'; /** @@ -57,6 +57,9 @@ export interface UpdatePeerRequestParams { data?: string; } +export interface StartTranscriptionRequestParams { + mode: HMSTranscriptionMode; +} export interface SetSessionMetadataParams { key?: string; data: any; diff --git a/packages/hms-video-store/src/signal/jsonrpc/index.ts b/packages/hms-video-store/src/signal/jsonrpc/index.ts index 014f725e43..a96909343d 100644 --- a/packages/hms-video-store/src/signal/jsonrpc/index.ts +++ b/packages/hms-video-store/src/signal/jsonrpc/index.ts @@ -60,6 +60,7 @@ import { SetSessionMetadataParams, SetSessionMetadataResponse, StartRTMPOrRecordingRequestParams, + StartTranscriptionRequestParams, Track, TrackUpdateRequestParams, UpdatePeerRequestParams, @@ -364,6 +365,14 @@ export default class JsonRpcSignal { await this.call(HMSSignalMethod.STOP_HLS_STREAMING, { ...params }); } + async startTranscription(params: StartTranscriptionRequestParams) { + await this.call(HMSSignalMethod.START_TRANSCRIPTION, { ...params }); + } + + async stopTranscription(params: StartTranscriptionRequestParams) { + await this.call(HMSSignalMethod.STOP_TRANSCRIPTION, { ...params }); + } + async sendHLSTimedMetadata(params?: HLSTimedMetadataParams): Promise { await this.call(HMSSignalMethod.HLS_TIMED_METADATA, { ...params }); } diff --git a/packages/hms-video-store/src/signal/jsonrpc/models.ts b/packages/hms-video-store/src/signal/jsonrpc/models.ts index 2344ec3e70..51f01d828a 100644 --- a/packages/hms-video-store/src/signal/jsonrpc/models.ts +++ b/packages/hms-video-store/src/signal/jsonrpc/models.ts @@ -39,6 +39,8 @@ export enum HMSSignalMethod { UPDATE_PEER_METADATA = 'peer-update', START_HLS_STREAMING = 'hls-start', STOP_HLS_STREAMING = 'hls-stop', + START_TRANSCRIPTION = 'transcription-start', + STOP_TRANSCRIPTION = 'transcription-stop', HLS_TIMED_METADATA = 'hls-timed-metadata', SET_METADATA = 'set-metadata', GET_METADATA = 'get-metadata',