Release PR

packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts

@@ -164,7 +164,6 @@ export class AudioSinkManager {
     track.setVolume(this.volume);
     HMSLogger.d(this.TAG, 'Audio track added', `${track}`);
     this.init(); // call to create sink element if not already created
-    await this.autoSelectAudioOutput();
     this.audioSink?.append(audioEl);
     this.outputDevice && (await track.setOutputDevice(this.outputDevice));
     audioEl.srcObject = new MediaStream([track.nativeTrack]);
@@ -197,6 +196,11 @@ export class AudioSinkManager {
   };
 
   private handleAudioDeviceChange = (event: HMSDeviceChangeEvent) => {
+    // this means the initial load
+    if (!event.selection) {
+      HMSLogger.d(this.TAG, 'device change called');
+      this.autoSelectAudioOutput();
+    }
     // if there is no selection that means this is an init request. No need to do anything
     if (event.isUserSelection || event.error || !event.selection || event.type === 'video') {
       return;
@@ -286,8 +290,7 @@ export class AudioSinkManager {
    */
   // eslint-disable-next-line complexity
   private autoSelectAudioOutput = async () => {
-    if (!this.audioSink?.children.length) {
-      HMSLogger.d(this.TAG, 'No remote audio added yet');
+    if ('ondevicechange' in navigator.mediaDevices) {
       return;
     }
     let bluetoothDevice: InputDeviceInfo | null = null;

packages/hms-video-store/src/device-manager/DeviceManager.ts

@@ -93,6 +93,10 @@ export class DeviceManager implements HMSDeviceManager {
     !this.initialized && navigator.mediaDevices.addEventListener('devicechange', this.handleDeviceChange);
     this.initialized = true;
     await this.enumerateDevices();
+    // do it only on initial load.
+    if (!force) {
+      await this.updateToActualDefaultDevice();
+    }
     this.logDevices('Init');
     await this.setOutputDevice();
     this.eventBus.deviceChange.publish({
@@ -188,6 +192,23 @@ export class DeviceManager implements HMSDeviceManager {
     }
   };
 
+  /**
+   * For example, if a different device, say OBS is selected as default from chrome settings, when you do getUserMedia with default, that is not the device
+   * you get. So update to the browser settings default device
+   * Update only when initial deviceId is not passed
+   */
+  private updateToActualDefaultDevice = async () => {
+    const localPeer = this.store.getLocalPeer();
+    const videoDeviceId = this.store.getConfig()?.settings?.videoDeviceId;
+    if (!videoDeviceId && localPeer?.videoTrack) {
+      await localPeer.videoTrack.setSettings({ deviceId: this.videoInput[0]?.deviceId }, true);
+    }
+    const audioDeviceId = this.store.getConfig()?.settings?.audioInputDeviceId;
+    if (!audioDeviceId && localPeer?.audioTrack) {
+      await localPeer.audioTrack.setSettings({ deviceId: this.audioInput[0]?.deviceId }, true);
+    }
+  };
+
   private handleDeviceChange = debounce(async () => {
     await this.enumerateDevices();
     this.logDevices('After Device Change');
@@ -366,7 +387,7 @@ export class DeviceManager implements HMSDeviceManager {
       .deviceId(newSelection.deviceId)
       .build();
     try {
-      await (videoTrack as HMSLocalVideoTrack).setSettings(newVideoTrackSettings, true);
+      await videoTrack.setSettings(newVideoTrackSettings, true);
       // On replace track, enabled will be true. Need to be set to previous state
       // videoTrack.setEnabled(enabled); // TODO: remove this once verified on qa.
       this.eventBus.deviceChange.publish({

packages/hms-video-store/src/reactive-store/HMSSDKActions.ts

@@ -88,6 +88,7 @@ import {
   selectTracksMap,
   selectVideoTrackByID,
 } from '../selectors';
+import { getPeerRequestParams } from '../signal/interfaces';
 
 // import { ActionBatcher } from './sdkUtils/ActionBatcher';
 
@@ -602,6 +603,14 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     await this.sdk.lowerRemotePeerHand(peerId);
   }
 
+  async getPeer(peerId: getPeerRequestParams) {
+    const peer = await this.sdk.getPeer(peerId);
+    if (peer) {
+      return SDKToHMS.convertPeer(peer) as HMSPeer;
+    }
+    return undefined;
+  }
+
   getPeerListIterator(options?: HMSPeerListIteratorOptions) {
     const iterator = this.sdk.getPeerListIterator(options);
     return {

packages/hms-video-store/src/sdk/index.ts

@@ -66,6 +66,7 @@ import { SessionStore } from '../session-store';
 import { InteractivityCenter } from '../session-store/interactivity-center';
 import { InitConfig, InitFlags } from '../signal/init/models';
 import {
+  getPeerRequestParams,
   HLSRequestParams,
   HLSTimedMetadataParams,
   HLSVariant,
@@ -728,6 +729,14 @@ export class HMSSdk implements HMSInterface {
     return await this.sendMessageInternal({ message, recipientPeer, type });
   }
 
+  async getPeer(peerID: getPeerRequestParams) {
+    const response = await this.transport.signal.getPeer(peerID);
+    if (response) {
+      return createRemotePeer(response, this.store);
+    }
+    return undefined;
+  }
+
   private async sendMessageInternal({ recipientRoles, recipientPeer, type = 'chat', message }: HMSMessageInput) {
     if (message.replace(/\u200b/g, ' ').trim() === '') {
       HMSLogger.w(this.TAG, 'sendMessage', 'Ignoring empty message send');

packages/hms-video-store/src/signal/jsonrpc/index.ts

@@ -5,7 +5,7 @@ import { HMSConnectionRole, HMSTrickle } from '../../connection/model';
 import { ErrorFactory } from '../../error/ErrorFactory';
 import { HMSAction } from '../../error/HMSAction';
 import { HMSException } from '../../error/HMSException';
-import { SendMessage } from '../../notification-manager';
+import { PeerNotificationInfo, SendMessage } from '../../notification-manager';
 import {
   DEFAULT_SIGNAL_PING_INTERVAL,
   DEFAULT_SIGNAL_PING_TIMEOUT,
@@ -372,8 +372,8 @@ export default class JsonRpcSignal {
     await this.call(HMSSignalMethod.UPDATE_PEER_METADATA, { ...params });
   }
 
-  async getPeer(params: getPeerRequestParams) {
-    await this.call(HMSSignalMethod.GET_PEER, { ...params });
+  async getPeer(params: getPeerRequestParams): Promise<PeerNotificationInfo | undefined> {
+    return await this.call(HMSSignalMethod.GET_PEER, { ...params });
   }
 
   async joinGroup(name: string): Promise<JoinLeaveGroupResponse> {

packages/roomkit-react/src/Prebuilt/plugins/FlyingEmoji.jsx

@@ -1,9 +1,11 @@
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { useMedia } from 'react-use';
 import {
+  selectIsLargeRoom,
   selectLocalPeerID,
   selectPeerNameByID,
   useCustomEvent,
+  useHMSActions,
   useHMSStore,
   useHMSVanillaStore,
 } from '@100mslive/react-sdk';
@@ -43,17 +45,23 @@ const getStartingPoints = isMobile => {
 export function FlyingEmoji() {
   const localPeerId = useHMSStore(selectLocalPeerID);
   const vanillaStore = useHMSVanillaStore();
+  const hmsActions = useHMSActions();
   const [emojis, setEmojis] = useState([]);
   const isMobile = useMedia(cssConfig.media.md);
+  const isLargeRoom = useHMSStore(selectIsLargeRoom);
 
   const startingPoints = useMemo(() => getStartingPoints(isMobile), [isMobile]);
 
   const showFlyingEmoji = useCallback(
-    ({ emojiId, senderId }) => {
+    async ({ emojiId, senderId }) => {
       if (!emojiId || !senderId || document.hidden) {
         return;
       }
-      const senderPeerName = vanillaStore.getState(selectPeerNameByID(senderId));
+      let senderPeerName = vanillaStore.getState(selectPeerNameByID(senderId));
+      if (!senderPeerName && isLargeRoom) {
+        const sender = await hmsActions.getPeer(senderId);
+        senderPeerName = sender?.name;
+      }
       const nameToShow = localPeerId === senderId ? 'You' : senderPeerName;
       const startingPoint = startingPoints[emojiCount % startingPoints.length];
       const id = emojiCount++;
@@ -71,7 +79,7 @@ export function FlyingEmoji() {
         ];
       });
     },
-    [localPeerId, vanillaStore, startingPoints],
+    [vanillaStore, isLargeRoom, localPeerId, startingPoints, hmsActions],
   );
 
   useCustomEvent({
@@ -118,24 +126,28 @@ export function FlyingEmoji() {
             <Box>
               <em-emoji id={emoji.emojiId} size="48px" set="apple"></em-emoji>
             </Box>
-            <Box
-              css={{
-                width: 'fit-content',
-                padding: '$2 $4',
-                background: '$surface_bright',
-                borderRadius: '$1',
-              }}
-            >
-              <Text
+            {emoji.senderName ? (
+              <Box
                 css={{
-                  fontSize: '$space$6',
-                  lineHeight: '$xs',
-                  color: '$on_surface_high',
+                  width: 'fit-content',
+                  padding: '$2 $4',
+                  background: '$surface_bright',
+                  borderRadius: '$1',
                 }}
               >
-                {emoji.senderName}
-              </Text>
-            </Box>
+                <Text
+                  css={{
+                    fontSize: '$space$6',
+                    lineHeight: '$xs',
+                    color: '$on_surface_high',
+                  }}
+                >
+                  {emoji.senderName}
+                </Text>
+              </Box>
+            ) : (
+              ''
+            )}
           </Flex>
         );
       })}

packages/roomkit-react/src/Stats/Stats.tsx

@@ -13,6 +13,7 @@ import {
 import { Tooltip } from '../Tooltip';
 import { formatBytes } from './formatBytes';
 import { Stats } from './StyledStats';
+import { useQoE } from './useQoE';
 
 export interface VideoTileStatsProps {
   videoTrackID?: HMSTrackID;
@@ -33,6 +34,7 @@ export function VideoTileStats({ videoTrackID, audioTrackID, peerID, isLocal = f
   const videoTrackStats = isLocal ? localVideoTrackStats?.[0] : remoteVideoTrackStats;
   const downlinkScore = useHMSStore(selectConnectionQualityByPeerID(peerID))?.downlinkQuality;
   const availableOutgoingBitrate = useHMSStatsStore(selectHMSStats.availablePublishBitrate);
+  const qoe = useQoE({ videoTrackID, audioTrackID, isLocal });
 
   // Viewer role - no stats to show
   if (!(audioTrackStats || videoTrackStats)) {
@@ -87,6 +89,7 @@ export function VideoTileStats({ videoTrackID, audioTrackID, peerID, isLocal = f
             </Fragment>
           ) : (
             <Fragment>
+              <StatsRow show={isNotNullish(qoe)} label="QoE" value={qoe} />
               <StatsRow
                 show={isNotNullishAndNot0(videoTrackStats?.frameWidth)}
                 label="Width"

packages/roomkit-react/src/Stats/useQoE.ts

@@ -0,0 +1,79 @@
+import { useRef } from 'react';
+import { usePrevious } from 'react-use';
+import { HMSTrackID, selectHMSStats, useHMSStatsStore } from '@100mslive/react-sdk';
+
+interface UseQoEProps {
+  videoTrackID?: HMSTrackID;
+  audioTrackID?: HMSTrackID;
+  isLocal?: boolean;
+}
+
+const EXPECTED_RESOLUTION = 1280 * 720;
+
+const clip = (value: number, min_value: number, max_value: number) => {
+  return Math.max(Math.min(value, max_value), min_value);
+};
+
+/**
+ * calculate QoE using 5 params:
+ * p1:freeze_duration_norm
+ * p2:resolution_norm
+ * p3:fps_norm
+ * p4:delay_norm
+ * p5:audio_concealed_norm
+ * the formula is 5*(p1)^3 * (p2)^0.3 * (p3)^0.5 * (p4)^1 * (p5)*2
+ *
+ * https://github.com/100mslive/webrtc-benchmark/blob/daily/sssd.py#L112
+ */
+export const useQoE = ({ videoTrackID, audioTrackID, isLocal = false }: UseQoEProps) => {
+  const audioTrackStats = useHMSStatsStore(selectHMSStats.trackStatsByID(audioTrackID));
+  const videoTrackStats = useHMSStatsStore(selectHMSStats.trackStatsByID(videoTrackID));
+  const prevVideoTrackStats = usePrevious(videoTrackStats);
+  const prevAudioTrackStats = usePrevious(audioTrackStats);
+
+  const prevJitterBufferDelayMs = useRef<number>(0);
+
+  if (isLocal || (videoTrackID && !videoTrackStats) || (audioTrackID && !audioTrackStats)) {
+    return;
+  }
+
+  const resolutionNorm =
+    ((videoTrackStats?.frameWidth || 0) * (videoTrackStats?.frameHeight || 0)) / EXPECTED_RESOLUTION;
+
+  const framesDecodedInLastSec =
+    videoTrackStats?.framesDecoded && prevVideoTrackStats?.framesDecoded
+      ? videoTrackStats.framesDecoded - prevVideoTrackStats.framesDecoded
+      : 0;
+  let freezeDurationNorm =
+    1 - ((videoTrackStats?.totalFreezesDuration || 0) - (prevVideoTrackStats?.totalFreezesDuration || 0));
+  freezeDurationNorm = freezeDurationNorm < 0 ? 0.5 : freezeDurationNorm;
+  freezeDurationNorm = framesDecodedInLastSec === 0 ? 0 : freezeDurationNorm;
+
+  const fpsNorm = framesDecodedInLastSec / 30;
+
+  const prevJBDelay = prevVideoTrackStats?.jitterBufferDelay || 0;
+  const prevJBEmittedCount = prevVideoTrackStats?.jitterBufferEmittedCount || 0;
+  const currentJBDelay = (videoTrackStats?.jitterBufferDelay || 0) - prevJBDelay;
+  const currentJBEmittedCount = (videoTrackStats?.jitterBufferEmittedCount || 0) - prevJBEmittedCount;
+
+  const jitterBufferDelayMs =
+    currentJBEmittedCount > 0 ? (currentJBDelay * 1000) / currentJBEmittedCount : prevJitterBufferDelayMs.current;
+  prevJitterBufferDelayMs.current = jitterBufferDelayMs;
+  const delayNorm = 1 - Math.min(1, jitterBufferDelayMs / 2000);
+
+  const prevConcealedSamples =
+    (prevAudioTrackStats?.concealedSamples || 0) - (prevAudioTrackStats?.silentConcealedSamples || 0);
+  const currentConcealedSamples =
+    (audioTrackStats?.concealedSamples || 0) - (audioTrackStats?.silentConcealedSamples || 0) - prevConcealedSamples;
+
+  const audioConcealedNorm = 1 - currentConcealedSamples / 48000;
+
+  return (
+    5 *
+    clip(freezeDurationNorm, 0, 1) ** 3 *
+    clip(resolutionNorm, 0, 1) ** 0.3 *
+    clip(fpsNorm, 0, 1) ** 0.2 *
+    clip(delayNorm, 0, 1) ** 0.5 *
+    clip(audioConcealedNorm, 0, 1) ** 2
+  ).toFixed(2);
+};