diff --git a/examples/prebuilt-react-integration/package.json b/examples/prebuilt-react-integration/package.json index 59126e6832..257288ca4b 100644 --- a/examples/prebuilt-react-integration/package.json +++ b/examples/prebuilt-react-integration/package.json @@ -10,7 +10,7 @@ "preview": "vite preview" }, "dependencies": { - "@100mslive/roomkit-react": "0.3.21", + "@100mslive/roomkit-react": "0.3.22", "react": "^18.2.0", "react-dom": "^18.2.0" }, diff --git a/packages/hls-player/package.json b/packages/hls-player/package.json index eb5ae37e98..5ab3206f5d 100644 --- a/packages/hls-player/package.json +++ b/packages/hls-player/package.json @@ -1,6 +1,6 @@ { "name": "@100mslive/hls-player", - "version": "0.3.21", + "version": "0.3.22", "description": "HLS client library which uses HTML5 Video element and Media Source Extension for playback", "main": "dist/index.cjs.js", "module": "dist/index.js", @@ -36,7 +36,7 @@ "author": "100ms", "license": "MIT", "dependencies": { - "@100mslive/hls-stats": "0.4.21", + "@100mslive/hls-stats": "0.4.22", "eventemitter2": "^6.4.9", "hls.js": "1.4.12" } diff --git a/packages/hls-stats/package.json b/packages/hls-stats/package.json index b078a76ac8..a7301ef574 100644 --- a/packages/hls-stats/package.json +++ b/packages/hls-stats/package.json @@ -1,6 +1,6 @@ { "name": "@100mslive/hls-stats", - "version": "0.4.21", + "version": "0.4.22", "description": "A simple library that provides stats for your hls stream", "main": "dist/index.cjs.js", "module": "dist/index.js", diff --git a/packages/hms-video-store/package.json b/packages/hms-video-store/package.json index ccc9423223..4b7ea5597c 100644 --- a/packages/hms-video-store/package.json +++ b/packages/hms-video-store/package.json @@ -1,5 +1,5 @@ { - "version": "0.12.21", + "version": "0.12.22", "license": "MIT", "repository": { "type": "git", diff --git a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts index 2053784eaa..4197182456 100644 --- a/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts +++ b/packages/hms-video-store/src/analytics/AnalyticsEventFactory.ts @@ -135,6 +135,16 @@ export default class AnalyticsEventFactory { }); } + static audioRecovered(message: string) { + return new AnalyticsEvent({ + name: 'audioRecovered', + level: AnalyticsEventLevel.VERBOSE, + properties: { + message, + }, + }); + } + static deviceChange({ isUserSelection, selection, diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts index 08b0b7e0df..9359b0b88e 100644 --- a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts +++ b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts @@ -46,6 +46,7 @@ export class AudioSinkManager { this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate); this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange); this.eventBus.localVideoUnmutedNatively.subscribe(this.unpauseAudioTracks); + this.eventBus.localAudioUnmutedNatively.subscribe(this.unpauseAudioTracks); } setListener(listener?: HMSUpdateListener) { @@ -98,6 +99,7 @@ export class AudioSinkManager { this.eventBus.audioTrackUpdate.unsubscribe(this.handleTrackUpdate); this.eventBus.deviceChange.unsubscribe(this.handleAudioDeviceChange); this.eventBus.localVideoUnmutedNatively.unsubscribe(this.unpauseAudioTracks); + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.unpauseAudioTracks); this.autoPausedTracks = new Set(); this.state = { ...INITIAL_STATE }; } @@ -136,13 +138,19 @@ export class AudioSinkManager { ); this.eventBus.analytics.publish(AnalyticsEventFactory.audioPlaybackError(ex)); if (audioEl?.error?.code === MediaError.MEDIA_ERR_DECODE) { + // try to wait for main execution to complete first this.removeAudioElement(audioEl, track); await sleep(500); await this.handleTrackAdd({ track, peer, callListener: false }); + if (!this.state.autoplayFailed) { + this.eventBus.analytics.publish( + AnalyticsEventFactory.audioRecovered('Audio recovered after media decode error'), + ); + } } }; track.setAudioElement(audioEl); - track.setVolume(this.volume); + await track.setVolume(this.volume); HMSLogger.d(this.TAG, 'Audio track added', `${track}`); this.init(); // call to create sink element if not already created this.audioSink?.append(audioEl); diff --git a/packages/hms-video-store/src/device-manager/DeviceManager.ts b/packages/hms-video-store/src/device-manager/DeviceManager.ts index 629dd36bc5..d6d4e5281f 100644 --- a/packages/hms-video-store/src/device-manager/DeviceManager.ts +++ b/packages/hms-video-store/src/device-manager/DeviceManager.ts @@ -4,7 +4,7 @@ import { ErrorFactory } from '../error/ErrorFactory'; import { HMSException } from '../error/HMSException'; import { EventBus } from '../events/EventBus'; import { DeviceMap, HMSDeviceChangeEvent, SelectedDevices } from '../interfaces'; -import { getAudioDeviceCategory, isIOS } from '../internal'; +import { getAudioDeviceCategory, HMSAudioDeviceCategory, isIOS } from '../internal'; import { HMSAudioTrackSettingsBuilder, HMSVideoTrackSettingsBuilder } from '../media/settings'; import { HMSLocalAudioTrack, HMSLocalTrack, HMSLocalVideoTrack } from '../media/tracks'; import { Store } from '../sdk/store'; @@ -444,13 +444,13 @@ export class DeviceManager implements HMSDeviceManager { for (const device of this.audioInput) { const deviceCategory = getAudioDeviceCategory(device.label); - if (deviceCategory === 'speakerphone') { + if (deviceCategory === HMSAudioDeviceCategory.SPEAKERPHONE) { speakerPhone = device; - } else if (deviceCategory === 'wired') { + } else if (HMSAudioDeviceCategory.WIRED) { wired = device; - } else if (deviceCategory === 'bluetooth') { + } else if (HMSAudioDeviceCategory.BLUETOOTH) { bluetoothDevice = device; - } else if (deviceCategory === 'speakerhone') { + } else if (HMSAudioDeviceCategory.EARPIECE) { earpiece = device; } } diff --git a/packages/hms-video-store/src/error/ErrorCodes.ts b/packages/hms-video-store/src/error/ErrorCodes.ts index 24e00e6419..dccd5b6286 100644 --- a/packages/hms-video-store/src/error/ErrorCodes.ts +++ b/packages/hms-video-store/src/error/ErrorCodes.ts @@ -76,6 +76,9 @@ export const ErrorCodes = { // Selected device not detected on change SELECTED_DEVICE_MISSING: 3014, + + // Track is publishing with no data, can happen when a whatsapp call is ongoing before 100ms call in mweb + NO_DATA: 3015, }, WebrtcErrors: { diff --git a/packages/hms-video-store/src/error/ErrorFactory.ts b/packages/hms-video-store/src/error/ErrorFactory.ts index e0655a225f..7345d26749 100644 --- a/packages/hms-video-store/src/error/ErrorFactory.ts +++ b/packages/hms-video-store/src/error/ErrorFactory.ts @@ -251,6 +251,16 @@ export const ErrorFactory = { false, ); }, + + NoDataInTrack(description: string) { + return new HMSException( + ErrorCodes.TracksErrors.NO_DATA, + 'Track does not have any data', + HMSAction.TRACK, + description, + 'This could possibily due to another application taking priority over the access to camera or microphone or due to an incoming call', + ); + }, }, WebrtcErrors: { diff --git a/packages/hms-video-store/src/events/EventBus.ts b/packages/hms-video-store/src/events/EventBus.ts index 0bd6f2e263..096b4bba32 100644 --- a/packages/hms-video-store/src/events/EventBus.ts +++ b/packages/hms-video-store/src/events/EventBus.ts @@ -31,6 +31,7 @@ export class EventBus { this.eventEmitter, ); readonly localVideoUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_VIDEO_UNMUTED_NATIVELY, this.eventEmitter); + readonly localAudioUnmutedNatively = new HMSInternalEvent(HMSEvents.LOCAL_AUDIO_UNMUTED_NATIVELY, this.eventEmitter); /** * Emitter which processes raw RTC stats from rtcStatsUpdate and calls client callback diff --git a/packages/hms-video-store/src/index.ts b/packages/hms-video-store/src/index.ts index aa36a66403..e26beb60ef 100644 --- a/packages/hms-video-store/src/index.ts +++ b/packages/hms-video-store/src/index.ts @@ -12,6 +12,7 @@ export * from './selectors'; export * from './webrtc-stats'; export { HMSAudioMode, + HMSAudioDeviceCategory, HMSLogLevel, HMSAudioPluginType, HMSVideoPluginType, diff --git a/packages/hms-video-store/src/interfaces/config.ts b/packages/hms-video-store/src/interfaces/config.ts index 07d75e21a4..23b88c4b58 100644 --- a/packages/hms-video-store/src/interfaces/config.ts +++ b/packages/hms-video-store/src/interfaces/config.ts @@ -42,10 +42,6 @@ export interface HMSConfig { audioSinkElementId?: string; autoVideoSubscribe?: boolean; initEndpoint?: string; - /** - * Request Camera/Mic permissions irrespective of role to avoid delay in getting device list - */ - alwaysRequestPermissions?: boolean; /** * Enable to get a network quality score while in preview. The score ranges from -1 to 5. * -1 when we are not able to connect to 100ms servers within an expected time limit diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts index b97efa49cd..69fd62e186 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts @@ -56,6 +56,7 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { ) { super(stream, track, source); stream.tracks.push(this); + this.addTrackEventListeners(track); this.settings = settings; // Replace the 'default' or invalid deviceId with the actual deviceId @@ -99,13 +100,9 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { this.manuallySelectedDeviceId = undefined; } - private isTrackNotPublishing = () => { - return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted; - }; - private handleVisibilityChange = async () => { // track state is fine do nothing - if (!this.isTrackNotPublishing()) { + if (!this.shouldReacquireTrack()) { HMSLogger.d(this.TAG, `visibiltiy: ${document.visibilityState}`, `${this}`); return; } @@ -151,16 +148,19 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { * no audio when the above getAudioTrack throws an error. ex: DeviceInUse error */ prevTrack?.stop(); + this.removeTrackEventListeners(prevTrack); this.tracksCreated.forEach(track => track.stop()); this.tracksCreated.clear(); try { const newTrack = await getAudioTrack(settings); + this.addTrackEventListeners(newTrack); this.tracksCreated.add(newTrack); HMSLogger.d(this.TAG, 'replaceTrack, Previous track stopped', prevTrack, 'newTrack', newTrack); await this.updateTrack(newTrack); } catch (e) { // Generate a new track from previous settings so there will be audio because previous track is stopped const newTrack = await getAudioTrack(this.settings); + this.addTrackEventListeners(newTrack); this.tracksCreated.add(newTrack); await this.updateTrack(newTrack); if (this.isPublished) { @@ -184,8 +184,8 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return; } - // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled. - if (value && (isEmptyTrack(this.nativeTrack) || this.nativeTrack.muted)) { + // Replace silent empty track or muted track(happens when microphone is disabled from address bar in iOS) with an actual audio track, if enabled or ended track or when silence is detected. + if (value && this.shouldReacquireTrack()) { await this.replaceTrackWith(this.settings); } await super.setEnabled(value); @@ -303,6 +303,42 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { return this.processedTrack || this.nativeTrack; } + private addTrackEventListeners(track: MediaStreamTrack) { + track.addEventListener('mute', this.handleTrackMute); + track.addEventListener('unmute', this.handleTrackUnmute); + } + + private removeTrackEventListeners(track: MediaStreamTrack) { + track.removeEventListener('mute', this.handleTrackMute); + track.removeEventListener('unmute', this.handleTrackUnmute); + } + + private handleTrackMute = () => { + HMSLogger.d(this.TAG, 'muted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: true, + reason, + }), + ); + }; + + /** @internal */ + handleTrackUnmute = async () => { + HMSLogger.d(this.TAG, 'unmuted natively'); + const reason = document.visibilityState === 'hidden' ? 'visibility-change' : 'incoming-call'; + this.eventBus.analytics.publish( + this.sendInterruptionEvent({ + started: false, + reason, + }), + ); + await this.setEnabled(this.enabled); + // whatsapp call doesn't seem to send video unmute natively, so use audio unmute to play video + this.eventBus.localAudioUnmutedNatively.publish(); + }; + private replaceSenderTrack = async () => { if (!this.transceiver || this.transceiver.direction !== 'sendonly') { HMSLogger.d(this.TAG, `transceiver for ${this.trackId} not available or not connected yet`); @@ -311,6 +347,12 @@ export class HMSLocalAudioTrack extends HMSAudioTrack { await this.transceiver.sender.replaceTrack(this.processedTrack || this.nativeTrack); }; + private shouldReacquireTrack = () => { + return ( + isEmptyTrack(this.nativeTrack) || this.isTrackNotPublishing() || this.audioLevelMonitor?.isSilentThisInstant() + ); + }; + private buildNewSettings(settings: Partial) { const { volume, codec, maxBitrate, deviceId, advanced, audioMode } = { ...this.settings, ...settings }; const newSettings = new HMSAudioTrackSettings(volume, codec, maxBitrate, deviceId, advanced, audioMode); diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts index 5e5090ecd8..0089b696b0 100644 --- a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts @@ -85,6 +85,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { this.pluginsManager = new HMSVideoPluginsManager(this, eventBus); this.mediaStreamPluginsManager = new HMSMediaStreamPluginsManager(eventBus, room); this.setFirstTrackId(this.trackId); + this.eventBus.localAudioUnmutedNatively.subscribe(this.handleTrackUnmute); if (isBrowser && source === 'regular' && isMobile()) { document.addEventListener('visibilitychange', this.handleVisibilityChange); } @@ -131,6 +132,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * use this function to set the enabled state of a track. If true the track will be unmuted and muted otherwise. * @param value */ + // eslint-disable-next-line complexity async setEnabled(value: boolean): Promise { if (value === this.enabled) { return; @@ -260,6 +262,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { * @internal */ async cleanup() { + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.handleTrackUnmute); this.removeTrackEventListeners(this.nativeTrack); super.cleanup(); this.transceiver = undefined; @@ -513,12 +516,12 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { private addTrackEventListeners(track: MediaStreamTrack) { track.addEventListener('mute', this.handleTrackMute); - track.addEventListener('unmute', this.handleTrackUnmute); + track.addEventListener('unmute', this.handleTrackUnmuteNatively); } private removeTrackEventListeners(track: MediaStreamTrack) { track.removeEventListener('mute', this.handleTrackMute); - track.removeEventListener('unmute', this.handleTrackUnmute); + track.removeEventListener('unmute', this.handleTrackUnmuteNatively); } private handleTrackMute = () => { @@ -533,7 +536,7 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { }; /** @internal */ - handleTrackUnmute = () => { + handleTrackUnmuteNatively = async () => { HMSLogger.d(this.TAG, 'unmuted natively'); this.eventBus.analytics.publish( this.sendInterruptionEvent({ @@ -541,9 +544,10 @@ export class HMSLocalVideoTrack extends HMSVideoTrack { reason: 'incoming-call', }), ); - super.handleTrackUnmute(); + this.handleTrackUnmute(); this.eventBus.localVideoEnabled.publish({ enabled: this.enabled, track: this }); this.eventBus.localVideoUnmutedNatively.publish(); + await this.setEnabled(this.enabled); }; /** diff --git a/packages/hms-video-store/src/media/tracks/HMSTrack.ts b/packages/hms-video-store/src/media/tracks/HMSTrack.ts index 04f9ff511f..208d021ba4 100644 --- a/packages/hms-video-store/src/media/tracks/HMSTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSTrack.ts @@ -85,6 +85,11 @@ export abstract class HMSTrack { protected setFirstTrackId(trackId: string) { this.firstTrackId = trackId; } + + isTrackNotPublishing = () => { + return this.nativeTrack.readyState === 'ended' || this.nativeTrack.muted; + }; + /** * @internal * It will send event to analytics when interruption start/stop diff --git a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts index 3f6e60b2e7..03e6d7e0b7 100644 --- a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts +++ b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts @@ -93,8 +93,8 @@ export class HMSVideoTrack extends HMSTrack { private reTriggerPlay = ({ videoElement }: { videoElement: HTMLVideoElement }) => { setTimeout(() => { - videoElement.play().catch(() => { - HMSLogger.w('[HMSVideoTrack]', 'failed to play'); + videoElement.play().catch((e: Error) => { + HMSLogger.w('[HMSVideoTrack]', 'failed to play', e.message); }); }, 0); }; diff --git a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts index b962dabd20..e1788ae25e 100644 --- a/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts +++ b/packages/hms-video-store/src/sdk/LocalTrackManager.test.ts @@ -109,6 +109,7 @@ const mockMediaStream = { kind: 'video', getSettings: jest.fn(() => ({ deviceId: 'video-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), getAudioTracks: jest.fn(() => [ @@ -117,6 +118,7 @@ const mockMediaStream = { kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-device-id' })), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), }, ]), addTrack: jest.fn(() => {}), @@ -206,7 +208,13 @@ const mockAudioContext = { return { stream: { getAudioTracks: jest.fn(() => [ - { id: 'audio-id', kind: 'audio', getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })) }, + { + id: 'audio-id', + kind: 'audio', + getSettings: jest.fn(() => ({ deviceId: 'audio-mock-device-id' })), + addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), + }, ]), }, }; @@ -426,6 +434,7 @@ describe('LocalTrackManager', () => { kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), } as unknown as MediaStreamTrack, HMSPeerType.REGULAR, testEventBus, @@ -459,6 +468,7 @@ describe('LocalTrackManager', () => { kind: 'video', getSettings: () => ({ deviceId: 'video-device-id', groupId: 'video-group-id' }), addEventListener: jest.fn(() => {}), + removeEventListener: jest.fn(() => {}), } as unknown as MediaStreamTrack, HMSPeerType.REGULAR, testEventBus, diff --git a/packages/hms-video-store/src/sdk/index.ts b/packages/hms-video-store/src/sdk/index.ts index e0a015a817..3cac0898a7 100644 --- a/packages/hms-video-store/src/sdk/index.ts +++ b/packages/hms-video-store/src/sdk/index.ts @@ -256,6 +256,7 @@ export class HMSSdk implements HMSInterface { this.eventBus.analytics.subscribe(this.sendAnalyticsEvent); this.eventBus.deviceChange.subscribe(this.handleDeviceChange); this.eventBus.localVideoUnmutedNatively.subscribe(this.unpauseRemoteVideoTracks); + this.eventBus.localAudioUnmutedNatively.subscribe(this.unpauseRemoteVideoTracks); this.eventBus.audioPluginFailed.subscribe(this.handleAudioPluginError); } @@ -435,13 +436,6 @@ export class HMSSdk implements HMSInterface { this.analyticsTimer.start(TimedEvent.PREVIEW); this.setUpPreview(config, listener); - // Request permissions and populate devices before waiting for policy - if (config.alwaysRequestPermissions) { - this.localTrackManager.requestPermissions().then(async () => { - await this.initDeviceManagers(); - }); - } - let initSuccessful = false; let networkTestFinished = false; const timerId = setTimeout(() => { @@ -457,7 +451,22 @@ export class HMSSdk implements HMSInterface { this.localPeer.asRole = newRole || this.localPeer.role; } const tracks = await this.localTrackManager.getTracksToPublish(config.settings); - tracks.forEach(track => this.setLocalPeerTrack(track)); + tracks.forEach(track => { + this.setLocalPeerTrack(track); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + HMSLogger.e(this.TAG, error); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } + }); this.localPeer?.audioTrack && this.initPreviewTrackAudioLevelMonitor(); await this.initDeviceManagers(); this.sdkState.isPreviewInProgress = false; @@ -664,6 +673,7 @@ export class HMSSdk implements HMSInterface { this.cleanDeviceManagers(); this.eventBus.analytics.unsubscribe(this.sendAnalyticsEvent); this.eventBus.localVideoUnmutedNatively.unsubscribe(this.unpauseRemoteVideoTracks); + this.eventBus.localAudioUnmutedNatively.unsubscribe(this.unpauseRemoteVideoTracks); this.analyticsTimer.cleanup(); DeviceStorageManager.cleanup(); this.playlistManager.cleanup(); @@ -1332,6 +1342,19 @@ export class HMSSdk implements HMSInterface { private async setAndPublishTracks(tracks: HMSLocalTrack[]) { for (const track of tracks) { await this.transport.publish([track]); + if (track.isTrackNotPublishing()) { + const error = ErrorFactory.TracksErrors.NoDataInTrack( + `${track.type} track has no data. muted: ${track.nativeTrack.muted}, readyState: ${track.nativeTrack.readyState}`, + ); + HMSLogger.e(this.TAG, error); + this.sendAnalyticsEvent( + AnalyticsEventFactory.publish({ + devices: this.deviceManager.getDevices(), + error: error, + }), + ); + this.listener?.onError(error); + } this.setLocalPeerTrack(track); this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, this.localPeer!); } diff --git a/packages/hms-video-store/src/utils/constants.ts b/packages/hms-video-store/src/utils/constants.ts index 5f237a85d3..0888adb791 100644 --- a/packages/hms-video-store/src/utils/constants.ts +++ b/packages/hms-video-store/src/utils/constants.ts @@ -40,6 +40,7 @@ export const HMSEvents = { LOCAL_AUDIO_ENABLED: 'local-audio-enabled', LOCAL_VIDEO_ENABLED: 'local-video-enabled', LOCAL_VIDEO_UNMUTED_NATIVELY: 'local-video-unmuted-natively', + LOCAL_AUDIO_UNMUTED_NATIVELY: 'local-audio-unmuted-natively', STATS_UPDATE: 'stats-update', // emitted by HMSWebrtcInternals RTC_STATS_UPDATE: 'rtc-stats-update', // emitted by RTCStatsMonitor TRACK_DEGRADED: 'track-degraded', diff --git a/packages/hms-video-store/src/utils/media.ts b/packages/hms-video-store/src/utils/media.ts index c9740bcc0a..330b544b14 100644 --- a/packages/hms-video-store/src/utils/media.ts +++ b/packages/hms-video-store/src/utils/media.ts @@ -64,16 +64,27 @@ export const HMSAudioContextHandler: HMSAudioContext = { }, }; -export const getAudioDeviceCategory = (deviceLabel: string) => { +export enum HMSAudioDeviceCategory { + SPEAKERPHONE = 'SPEAKERPHONE', + WIRED = 'WIRED', + BLUETOOTH = 'BLUETOOTH', + EARPIECE = 'EARPIECE', +} + +export const getAudioDeviceCategory = (deviceLabel?: string) => { + if (!deviceLabel) { + HMSLogger.e('[DeviceManager]:', 'No device label provided'); + return HMSAudioDeviceCategory.SPEAKERPHONE; + } const label = deviceLabel.toLowerCase(); if (label.includes('speakerphone')) { - return 'speakerhone'; + return HMSAudioDeviceCategory.SPEAKERPHONE; } else if (label.includes('wired')) { - return 'wired'; + return HMSAudioDeviceCategory.WIRED; } else if (/airpods|buds|wireless|bluetooth/gi.test(label)) { - return 'bluetooth'; + return HMSAudioDeviceCategory.BLUETOOTH; } else if (label.includes('earpiece')) { - return 'earpiece'; + return HMSAudioDeviceCategory.EARPIECE; } - return 'speakerphone'; + return HMSAudioDeviceCategory.SPEAKERPHONE; }; diff --git a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts index fa46e1cd20..c191424a87 100644 --- a/packages/hms-video-store/src/utils/track-audio-level-monitor.ts +++ b/packages/hms-video-store/src/utils/track-audio-level-monitor.ts @@ -133,7 +133,7 @@ export class TrackAudioLevelMonitor { return percent; } - private isSilentThisInstant() { + isSilentThisInstant() { if (!this.analyserNode || !this.dataArray) { HMSLogger.d(this.TAG, 'AudioContext not initialized'); return; diff --git a/packages/hms-virtual-background/package.json b/packages/hms-virtual-background/package.json index 8aadb05854..dd6fc45fc1 100755 --- a/packages/hms-virtual-background/package.json +++ b/packages/hms-virtual-background/package.json @@ -1,5 +1,5 @@ { - "version": "1.13.21", + "version": "1.13.22", "license": "MIT", "name": "@100mslive/hms-virtual-background", "author": "100ms", @@ -62,10 +62,10 @@ "format": "prettier --write src/**/*.ts" }, "peerDependencies": { - "@100mslive/hms-video-store": "0.12.21" + "@100mslive/hms-video-store": "0.12.22" }, "devDependencies": { - "@100mslive/hms-video-store": "0.12.21" + "@100mslive/hms-video-store": "0.12.22" }, "dependencies": { "@mediapipe/selfie_segmentation": "^0.1.1632777926", diff --git a/packages/hms-whiteboard/package.json b/packages/hms-whiteboard/package.json index b311a63ce3..902bd5f662 100644 --- a/packages/hms-whiteboard/package.json +++ b/packages/hms-whiteboard/package.json @@ -2,7 +2,7 @@ "name": "@100mslive/hms-whiteboard", "author": "100ms", "license": "MIT", - "version": "0.0.11", + "version": "0.0.12", "main": "dist/index.cjs.js", "module": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/react-icons/package.json b/packages/react-icons/package.json index 3e2cbbbf78..264ccfe92f 100644 --- a/packages/react-icons/package.json +++ b/packages/react-icons/package.json @@ -4,7 +4,7 @@ "main": "dist/index.cjs.js", "module": "dist/index.js", "typings": "dist/index.d.ts", - "version": "0.10.21", + "version": "0.10.22", "author": "100ms", "license": "MIT", "repository": { diff --git a/packages/react-sdk/package.json b/packages/react-sdk/package.json index 8311a52f08..f2e086dc82 100644 --- a/packages/react-sdk/package.json +++ b/packages/react-sdk/package.json @@ -4,7 +4,7 @@ "main": "dist/index.cjs.js", "module": "dist/index.js", "typings": "dist/index.d.ts", - "version": "0.10.21", + "version": "0.10.22", "author": "100ms", "license": "MIT", "repository": { @@ -48,7 +48,7 @@ "react": ">=16.8 <19.0.0" }, "dependencies": { - "@100mslive/hms-video-store": "0.12.21", + "@100mslive/hms-video-store": "0.12.22", "react-resize-detector": "^7.0.0", "zustand": "^3.6.2" } diff --git a/packages/roomkit-react/package.json b/packages/roomkit-react/package.json index ec97436310..77928515a0 100644 --- a/packages/roomkit-react/package.json +++ b/packages/roomkit-react/package.json @@ -10,7 +10,7 @@ "prebuilt", "roomkit" ], - "version": "0.3.21", + "version": "0.3.22", "author": "100ms", "license": "MIT", "repository": { @@ -75,12 +75,12 @@ "react": ">=17.0.2 <19.0.0" }, "dependencies": { - "@100mslive/hls-player": "0.3.21", + "@100mslive/hls-player": "0.3.22", "@100mslive/hms-noise-cancellation": "0.0.1", - "@100mslive/hms-virtual-background": "1.13.21", - "@100mslive/hms-whiteboard": "0.0.11", - "@100mslive/react-icons": "0.10.21", - "@100mslive/react-sdk": "0.10.21", + "@100mslive/hms-virtual-background": "1.13.22", + "@100mslive/hms-whiteboard": "0.0.12", + "@100mslive/react-icons": "0.10.22", + "@100mslive/react-sdk": "0.10.22", "@100mslive/types-prebuilt": "0.12.12", "@emoji-mart/data": "^1.0.6", "@emoji-mart/react": "^1.0.1", diff --git a/packages/roomkit-react/src/Prebuilt/components/Header/common.jsx b/packages/roomkit-react/src/Prebuilt/components/Header/common.jsx index 3a36c01ad8..f67296ffda 100644 --- a/packages/roomkit-react/src/Prebuilt/components/Header/common.jsx +++ b/packages/roomkit-react/src/Prebuilt/components/Header/common.jsx @@ -2,6 +2,7 @@ import React from 'react'; import { DeviceType, getAudioDeviceCategory, + HMSAudioDeviceCategory, selectIsLocalVideoEnabled, selectLocalVideoTrackID, selectVideoTrackByID, @@ -79,13 +80,13 @@ export const AudioActions = () => { if (!audioFiltered) { return null; } - const deviceCategory = getAudioDeviceCategory(currentSelection.label); + const deviceCategory = getAudioDeviceCategory(currentSelection?.label); let AudioIcon = ; - if (deviceCategory === 'bluetooth') { + if (deviceCategory === HMSAudioDeviceCategory.BLUETOOTH) { AudioIcon = ; - } else if (deviceCategory === 'wired') { + } else if (deviceCategory === HMSAudioDeviceCategory.WIRED) { AudioIcon = ; - } else if (deviceCategory === 'earpiece') { + } else if (deviceCategory === HMSAudioDeviceCategory.EARPIECE) { AudioIcon = ; } return ( @@ -112,7 +113,7 @@ export const AudioActions = () => { await hmsActions.refreshDevices(); }} > - {AudioIcon} + {AudioIcon} ); diff --git a/packages/roomkit-react/src/Prebuilt/components/Notifications/DeviceInUseError.tsx b/packages/roomkit-react/src/Prebuilt/components/Notifications/DeviceInUseError.tsx new file mode 100644 index 0000000000..3249d845ca --- /dev/null +++ b/packages/roomkit-react/src/Prebuilt/components/Notifications/DeviceInUseError.tsx @@ -0,0 +1,79 @@ +import React, { useEffect, useState } from 'react'; +import { HMSNotificationTypes, useHMSNotifications } from '@100mslive/react-sdk'; +import { Button, Dialog, Text } from '../../..'; +// @ts-ignore: No implicit Any +import { DialogContent, DialogRow } from '../../primitives/DialogContent'; +// @ts-ignore: No implicit Any +import { ToastManager } from '../Toast/ToastManager'; + +const Instruction = ({ description }: { description: string }) => ( +
  • + + {description} + +
    • +); + +export function DeviceInUseError() { + const notification = useHMSNotifications(HMSNotificationTypes.ERROR); + const [showDeviceInUseModal, setShowDeviceInUseModal] = useState(false); + const [deviceType, setDeviceType] = useState(''); + + useEffect(() => { + const error = notification?.data; + if (!error || error.code !== 3003) { + return; + } + + const errorMessage = error?.message; + ToastManager.addToast({ + title: `Error: ${errorMessage} - ${error?.description}`, + action: ( + + ), + }); + + const hasAudio = errorMessage.includes('audio'); + const hasVideo = errorMessage.includes('video'); + const hasScreen = errorMessage.includes('screen'); + if (hasAudio && hasVideo) { + setDeviceType('camera and microphone'); + } else if (hasAudio) { + setDeviceType('microphone'); + } else if (hasVideo) { + setDeviceType('camera'); + } else if (hasScreen) { + setDeviceType('screen'); + } + }, [notification]); + + return ( + { + setShowDeviceInUseModal(false); + }} + > + + + + We weren't able to access your {deviceType} since it's either in use by another application or is not + configured properly. Please follow the following instructions to resolve this issue. + + +
      + + Privacy and security > Site settings > ${deviceType} and check if your preferred device is selected as default.`} + /> + + +
    +     +     + ); +} diff --git a/packages/roomkit-react/src/Prebuilt/components/Notifications/Notifications.tsx b/packages/roomkit-react/src/Prebuilt/components/Notifications/Notifications.tsx index e7f6a6247c..c9e9c529fc 100644 --- a/packages/roomkit-react/src/Prebuilt/components/Notifications/Notifications.tsx +++ b/packages/roomkit-react/src/Prebuilt/components/Notifications/Notifications.tsx @@ -21,6 +21,7 @@ import { useRoomLayout, useUpdateRoomLayout } from '../../provider/roomLayoutPro import { ToastManager } from '../Toast/ToastManager'; import { AutoplayBlockedModal } from './AutoplayBlockedModal'; import { ChatNotifications } from './ChatNotifications'; +import { DeviceInUseError } from './DeviceInUseError'; import { HandRaisedNotifications } from './HandRaisedNotifications'; import { InitErrorModal } from './InitErrorModal'; import { PeerNotifications } from './PeerNotifications'; @@ -42,7 +43,18 @@ const pollToastKey: Record = {}; export function Notifications() { const localPeerID = useHMSStore(selectLocalPeerID); - const notification = useHMSNotifications(); + const notification = useHMSNotifications([ + HMSNotificationTypes.NAME_UPDATED, + HMSNotificationTypes.ERROR, + HMSNotificationTypes.ROLE_UPDATED, + HMSNotificationTypes.CHANGE_TRACK_STATE_REQUEST, + HMSNotificationTypes.REMOVED_FROM_ROOM, + HMSNotificationTypes.ROOM_ENDED, + HMSNotificationTypes.DEVICE_CHANGE_UPDATE, + HMSNotificationTypes.POLL_STARTED, + HMSNotificationTypes.POLL_STOPPED, + HMSNotificationTypes.NEW_MESSAGE, + ]); const subscribedNotifications = useSubscribedNotifications() || {}; const roomState = useHMSStore(selectRoomState); const updateRoomLayoutForRole = useUpdateRoomLayout(); @@ -108,6 +120,7 @@ export function Notifications() { if (!subscribedNotifications.ERROR) return; ToastManager.addToast({ title: `Error: ${notification.data?.message} - ${notification.data?.description}`, + duration: 8000, }); break; case HMSNotificationTypes.ROLE_UPDATED: { @@ -204,6 +217,7 @@ export function Notifications() { + ); } diff --git a/packages/roomkit-web/package.json b/packages/roomkit-web/package.json index 104abb13e5..70640532ce 100644 --- a/packages/roomkit-web/package.json +++ b/packages/roomkit-web/package.json @@ -1,6 +1,6 @@ { "name": "@100mslive/roomkit-web", - "version": "0.2.21", + "version": "0.2.22", "description": "A web component implementation of 100ms Prebuilt component", "keywords": [ "web-components", @@ -33,7 +33,7 @@ "build": "rm -rf dist && node ../../scripts/build-webapp" }, "dependencies": { - "@100mslive/roomkit-react": "0.3.21", + "@100mslive/roomkit-react": "0.3.22", "@r2wc/react-to-web-component": "2.0.2" } } diff --git a/yarn.lock b/yarn.lock index 8cb23dcb50..091e434c67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16534,7 +16534,16 @@ string-natural-compare@^3.0.1: resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-3.0.1.tgz#7a42d58474454963759e8e8b7ae63d71c1e7fdf4" integrity sha512-n3sPwynL1nwKi3WJ6AIsClwBMa0zTi54fn2oLU6ndfTSIO05xaznjSf15PcBZU6FNWbmN5Q6cxT4V5hGvB4taw== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -16639,7 +16648,14 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -17992,7 +18008,7 @@ worker-timers@^7.0.40: worker-timers-broker "^6.0.95" worker-timers-worker "^7.0.59" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -18010,6 +18026,15 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"