fix: sfu migration issues

- Duplicate tiles on migration - Screenshare not getting removed - add audio/video plugins on migration
100mslive · Aug 1, 2024 · fea3737 · fea3737
1 parent c58345b
commit fea3737
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 41 deletions.
diff --git a/examples/prebuilt-react-integration/src/App.jsx b/examples/prebuilt-react-integration/src/App.jsx
@@ -9,5 +9,16 @@ export default function App() {
     return <Diagnostics />;
   }
 
-  return <HMSPrebuilt roomCode={roomCode} />;
+  return (
+    <HMSPrebuilt
+      roomCode={roomCode}
+      options={{
+        endpoints: {
+          tokenByRoomCode: 'https://auth-nonprod.100ms.live/v2/token',
+          roomLayout: 'https://api-nonprod.100ms.live/v2/layouts/ui',
+          init: 'https://qa-in2-ipv6.100ms.live/init',
+        },
+      }}
+    />
+  );
 }
diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalAudioTrack.ts
@@ -65,14 +65,23 @@ export class HMSLocalAudioTrack extends HMSAudioTrack {
   }
 
   clone(stream?: HMSLocalStream) {
-    return new HMSLocalAudioTrack(
+    const track = new HMSLocalAudioTrack(
       stream || (this.stream as HMSLocalStream).clone(),
       this.nativeTrack.clone(),
       this.source!,
       this.eventBus,
       this.settings,
       this.room,
     );
+
+    if (this.pluginsManager.pluginsMap.size > 0) {
+      this.pluginsManager.pluginsMap.forEach(value => {
+        track
+          .addPlugin(value)
+          .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e));
+      });
+    }
+    return track;
   }
 
   getManuallySelectedDeviceId() {

diff --git a/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSLocalVideoTrack.ts
@@ -90,14 +90,25 @@ export class HMSLocalVideoTrack extends HMSVideoTrack {
   }
 
   clone(stream?: HMSLocalStream) {
-    return new HMSLocalVideoTrack(
+    const track = new HMSLocalVideoTrack(
       stream || (this.stream as HMSLocalStream).clone(),
       this.nativeTrack.clone(),
       this.source!,
       this.eventBus,
       this.settings,
       this.room,
     );
+    if (this.pluginsManager.pluginsMap.size > 0) {
+      this.pluginsManager.pluginsMap.forEach(value => {
+        track
+          .addPlugin(value)
+          .catch((e: Error) => HMSLogger.e(this.TAG, 'Plugin add failed while migrating', value, e));
+      });
+    }
+    if (this.mediaStreamPluginsManager.plugins.size > 0) {
+      track.addStreamPlugins(Array.from(this.mediaStreamPluginsManager.plugins));
+    }
+    return track;
   }
 
   /** @internal */

diff --git a/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts b/packages/hms-video-store/src/notification-manager/managers/TrackManager.ts
@@ -56,12 +56,11 @@ export class TrackManager {
   };
 
   handleTrackRemovedPermanently = (notification: TrackStateNotification) => {
-    HMSLogger.d(this.TAG, `ONTRACKREMOVE`, notification);
+    HMSLogger.d(this.TAG, `ONTRACKREMOVE permanently`, notification);
     const trackIds = Object.keys(notification.tracks);
 
     trackIds.forEach(trackId => {
       const trackStateEntry = this.store.getTrackState(trackId);
-
       if (!trackStateEntry) {
         return;
       }
@@ -205,7 +204,6 @@ export class TrackManager {
       track.type === HMSTrackType.AUDIO
         ? this.eventBus.audioTrackAdded.publish({ track: track as HMSRemoteAudioTrack, peer: hmsPeer as HMSRemotePeer })
         : this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, hmsPeer);
-      this.store.removeTrackState(track.trackId);
       this.tracksToProcess.delete(track.trackId);
     });
   };

diff --git a/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts b/packages/hms-video-store/src/plugins/audio/HMSAudioPluginsManager.ts
@@ -32,7 +32,7 @@ export class HMSAudioPluginsManager {
   private readonly TAG = '[AudioPluginsManager]';
   private readonly hmsTrack: HMSLocalAudioTrack;
   // Map maintains the insertion order
-  private readonly pluginsMap: Map<string, HMSAudioPlugin>;
+  readonly pluginsMap: Map<string, HMSAudioPlugin>;
   private audioContext?: AudioContext;
 
   private sourceNode?: MediaStreamAudioSourceNode;

diff --git a/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSMediaStreamPluginsManager.ts
@@ -8,7 +8,7 @@ import HMSLogger from '../../utils/logger';
 export class HMSMediaStreamPluginsManager {
   private readonly TAG = '[MediaStreamPluginsManager]';
   private analytics: VideoPluginsAnalytics;
-  private plugins: Set<HMSMediaStreamPlugin>;
+  readonly plugins: Set<HMSMediaStreamPlugin>;
   private room?: Room;
 
   constructor(eventBus: EventBus, room?: Room) {

diff --git a/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts b/packages/hms-video-store/src/plugins/video/HMSVideoPluginsManager.ts
@@ -47,7 +47,7 @@ export class HMSVideoPluginsManager {
   private pluginsLoopRunning = false;
   private pluginsLoopState: 'paused' | 'running' = 'paused';
   private readonly hmsTrack: HMSLocalVideoTrack;
-  private readonly pluginsMap: Map<string, HMSVideoPlugin>; // plugin names to their instance mapping
+  readonly pluginsMap: Map<string, HMSVideoPlugin>; // plugin names to their instance mapping
   private inputVideo?: HTMLVideoElement;
   private inputCanvas?: CanvasElement;
   private outputCanvas?: CanvasElement;

diff --git a/packages/hms-video-store/src/sdk/store/Store.ts b/packages/hms-video-store/src/sdk/store/Store.ts
@@ -75,6 +75,15 @@ class Store {
     this.simulcastEnabled = enabled;
   }
 
+  removeRemoteTracks() {
+    this.tracks.forEach(track => {
+      if (track instanceof HMSRemoteAudioTrack || track instanceof HMSRemoteVideoTrack) {
+        this.removeTrack(track);
+        delete this.peerTrackStates[track.trackId];
+      }
+    });
+  }
+
   getEnv() {
     return this.env;
   }

diff --git a/packages/hms-video-store/src/transport/index.ts b/packages/hms-video-store/src/transport/index.ts
@@ -431,36 +431,26 @@ export default class HMSTransport {
   // eslint-disable-next-line complexity
   async handleSFUMigration() {
     HMSLogger.time('sfu migration');
+    this.clearPeerConnections();
     const peers = this.store.getPeerMap();
+    this.store.removeRemoteTracks();
     for (const peerId in peers) {
       const peer = peers[peerId];
       if (peer.isLocal) {
         continue;
       }
-      if (peer.audioTrack) {
-        this.store.removeTrack(peer.audioTrack);
-        peer.audioTrack = undefined;
-      }
-      if (peer.videoTrack) {
-        this.store.removeTrack(peer.videoTrack);
-        peer.videoTrack = undefined;
-      }
-      while (peer.auxiliaryTracks.length > 0) {
-        const track = peer.auxiliaryTracks.shift();
-        if (track) {
-          this.store.removeTrack(track);
-        }
-      }
+      peer.audioTrack = undefined;
+      peer.videoTrack = undefined;
+      peer.auxiliaryTracks = [];
     }
-    this.clearPeerConnections();
-    this.createPeerConnections();
-    await this.negotiateOnFirstPublish();
+
     const localPeer = this.store.getLocalPeer();
     if (!localPeer) {
       return;
     }
-
-    let tracksToPublish = [];
+    this.createPeerConnections();
+    this.trackStates.clear();
+    await this.negotiateOnFirstPublish();
     const streamMap = new Map<string, HMSLocalStream>();
     if (localPeer.audioTrack) {
       const stream = localPeer.audioTrack.stream as HMSLocalStream;
@@ -469,7 +459,8 @@ export default class HMSTransport {
       }
       const newTrack = localPeer.audioTrack.clone(streamMap.get(stream.id));
       this.store.removeTrack(localPeer.audioTrack);
-      tracksToPublish.push(newTrack);
+      localPeer.audioTrack.cleanup();
+      await this.publishTrack(newTrack);
       localPeer.audioTrack = newTrack;
     }
 
@@ -480,7 +471,8 @@ export default class HMSTransport {
       }
       this.store.removeTrack(localPeer.videoTrack);
       const newTrack = localPeer.videoTrack.clone(streamMap.get(stream.id));
-      tracksToPublish.push(newTrack);
+      localPeer.videoTrack.cleanup();
+      await this.publishTrack(newTrack);
       localPeer.videoTrack = newTrack;
     }
 
@@ -497,16 +489,13 @@ export default class HMSTransport {
         if (newTrack.type === 'video' && newTrack.source === 'screen') {
           newTrack.nativeTrack.addEventListener('ended', this.onScreenshareStop);
         }
+        track.cleanup();
+        await this.publishTrack(newTrack);
         auxTracks.push(newTrack);
       }
     }
     localPeer.auxiliaryTracks = auxTracks;
-    tracksToPublish = tracksToPublish.concat(auxTracks);
     streamMap.clear();
-    for (const track of tracksToPublish) {
-      await this.publishTrack(track);
-      // this.listener?.onTrackUpdate(HMSTrackUpdate.TRACK_ADDED, track, localPeer);
-    }
     this.listener?.onSFUMigration?.();
     HMSLogger.timeEnd('sfu migration');
   }
@@ -573,7 +562,7 @@ export default class HMSTransport {
     HMSLogger.d(TAG, `✅ publishTrack: trackId=${track.trackId}`, `${track}`, this.callbacks);
   }
 
-  private async unpublishTrack(track: HMSLocalTrack, sfuMigration = false): Promise<void> {
+  private async unpublishTrack(track: HMSLocalTrack): Promise<void> {
     HMSLogger.d(TAG, `⏳ unpublishTrack: trackId=${track.trackId}`, `${track}`);
     if (track.publishedTrackId && this.trackStates.has(track.publishedTrackId)) {
       this.trackStates.delete(track.publishedTrackId);
@@ -599,11 +588,9 @@ export default class HMSTransport {
     const stream = track.stream as HMSLocalStream;
     stream.removeSender(track);
     await p;
-    if (!sfuMigration) {
-      await track.cleanup();
-      // remove track from store on unpublish
-      this.store.removeTrack(track);
-    }
+    await track.cleanup();
+    // remove track from store on unpublish
+    this.store.removeTrack(track);
     HMSLogger.d(TAG, `✅ unpublishTrack: trackId=${track.trackId}`, this.callbacks);
   }