[Fix]Capture quality adaptive policy criteria (#563)

Sources/StreamVideo/Utils/ThermalStateObserver.swift

@@ -108,6 +108,15 @@ final class ThermalStateObserver: ObservableObject, ThermalStateObserving {
     }
 }
 
+extension ProcessInfo.ThermalState: Comparable {
+    public static func < (
+        lhs: ProcessInfo.ThermalState,
+        rhs: ProcessInfo.ThermalState
+    ) -> Bool {
+        lhs.rawValue < rhs.rawValue
+    }
+}
+
 /// Provides the default value of the `Appearance` class.
 enum ThermalStateObserverKey: InjectionKey {
     static var currentValue: any ThermalStateObserving = ThermalStateObserver.shared

Sources/StreamVideo/WebRTC/v2/PeerConnection/MediaAdapters/VideoCapturePolicy/VideoCapturePolicy.swift

@@ -2,6 +2,8 @@
 // Copyright © 2024 Stream.io Inc. All rights reserved.
 //
 
+import Foundation
+
 /// Defines a video capture policy used by the `LocalVideoAdapter` to adjust video capture quality based on
 /// instructions from the SFU (Selective Forwarding Unit).
 ///
@@ -23,15 +25,35 @@ public class VideoCapturePolicy: @unchecked Sendable {
     ) async throws { /* No operation by default */ }
 }
 
-/// A final class that adapts the video capture quality dynamically.
+/// A final class that adapts the video capture quality dynamically. The policy requires the following criteria
+/// in order to start adapting the capture quality:
+/// - Either the thermal state is `.serious` or higher **or** the current device doesn't have a
+/// neuralEngine (which is required for efficiently resizing frames).
+/// - Requested encodings to activate are different than the currently activated ones.
+/// - There is a running videoCapturingSession
 final class AdaptiveVideoCapturePolicy: VideoCapturePolicy, @unchecked Sendable {
+
+    @Injected(\.thermalStateObserver) private var thermalStateObserver
+
+    private let neuralEngineExistsProvider: () -> Bool
+    private var lastActiveEncodings: Set<String>?
+
+    init(_ neuralEngineExistsProvider: @escaping () -> Bool) {
+        self.neuralEngineExistsProvider = neuralEngineExistsProvider
+        super.init()
+    }
+
     /// Overrides the method to update capture quality using an adaptive policy.
     override func updateCaptureQuality(
         with activeEncodings: Set<String>,
         for activeSession: VideoCaptureSession?
     ) async throws {
         /// Ensure there is an active session to work with.
-        guard let activeSession else { return }
+        guard
+            shouldUpdateCaptureQuality,
+            lastActiveEncodings != activeEncodings,
+            let activeSession
+        else { return }
 
         /// Filter the default video codecs to include only those matching the active encodings.
         let videoCodecs = VideoCodec
@@ -40,6 +62,17 @@ final class AdaptiveVideoCapturePolicy: VideoCapturePolicy, @unchecked Sendable
 
         try await activeSession.capturer
             .updateCaptureQuality(videoCodecs, on: activeSession.device)
+        lastActiveEncodings = activeEncodings
+        log.debug(
+            "Video capture quality adapted to [\(activeEncodings.sorted().joined(separator: ","))].",
+            subsystems: .webRTC
+        )
+    }
+
+    // MARK: - Private helpers
+
+    private var shouldUpdateCaptureQuality: Bool {
+        thermalStateObserver.state > .fair || !neuralEngineExistsProvider()
     }
 }
 
@@ -57,7 +90,7 @@ extension VideoCapturePolicy {
     /// This `adaptive` policy adjusts the device's video capture quality to match the quality requested
     /// by the SFU (Selective Forwarding Unit). By capturing video at the requested quality, it helps
     /// reduce processing overhead and improve performance, especially on older or less powerful devices.
-    static let adaptive: VideoCapturePolicy = AdaptiveVideoCapturePolicy()
+    static let adaptive: VideoCapturePolicy = AdaptiveVideoCapturePolicy { neuralEngineExists }
 }
 
 extension VideoCapturePolicy: InjectionKey {

StreamVideo.xcodeproj/project.pbxproj

@@ -446,6 +446,9 @@
 		40C4DF572C1C61BD0035DBC2 /* URL+Convenience.swift in Sources */ = {isa = PBXBuildFile; fileRef = 401A64B22A9DF86200534ED1 /* URL+Convenience.swift */; };
 		40C689182C64DDC70054528A /* Publisher+TaskSink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C689172C64DDC70054528A /* Publisher+TaskSink.swift */; };
 		40C6891C2C657F280054528A /* Publisher+AsyncStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C6891B2C657F280054528A /* Publisher+AsyncStream.swift */; };
+		40C75BB72CB4044600C167C3 /* MockThermalStateObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C75BB62CB4044600C167C3 /* MockThermalStateObserver.swift */; };
+		40C75BB82CB4045100C167C3 /* MockThermalStateObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C75BB62CB4044600C167C3 /* MockThermalStateObserver.swift */; };
+		40C75BB92CB40D8700C167C3 /* Mockable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4013387B2BF248E9007318BD /* Mockable.swift */; };
 		40C7B82C2B612D6000FB9DB2 /* ParticipantsListViewModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C7B82B2B612D6000FB9DB2 /* ParticipantsListViewModifier.swift */; };
 		40C7B8322B61325500FB9DB2 /* ModalButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C7B8312B61325500FB9DB2 /* ModalButton.swift */; };
 		40C7B8342B613A8200FB9DB2 /* ControlBadgeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40C7B8332B613A8200FB9DB2 /* ControlBadgeView.swift */; };
@@ -1701,6 +1704,7 @@
 		40C689192C64F74F0054528A /* SFUSignalService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SFUSignalService.swift; sourceTree = "<group>"; };
 		40C6891B2C657F280054528A /* Publisher+AsyncStream.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publisher+AsyncStream.swift"; sourceTree = "<group>"; };
 		40C6891D2C6661990054528A /* SFUEventAdapter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SFUEventAdapter.swift; sourceTree = "<group>"; };
+		40C75BB62CB4044600C167C3 /* MockThermalStateObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockThermalStateObserver.swift; sourceTree = "<group>"; };
 		40C7B82B2B612D6000FB9DB2 /* ParticipantsListViewModifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ParticipantsListViewModifier.swift; sourceTree = "<group>"; };
 		40C7B8312B61325500FB9DB2 /* ModalButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModalButton.swift; sourceTree = "<group>"; };
 		40C7B8332B613A8200FB9DB2 /* ControlBadgeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlBadgeView.swift; sourceTree = "<group>"; };
@@ -4620,6 +4624,7 @@
 		8492B87629081CE700006649 /* Mock */ = {
 			isa = PBXGroup;
 			children = (
+				40C75BB62CB4044600C167C3 /* MockThermalStateObserver.swift */,
 				40FE5EBC2C9C82A6006B0881 /* MockRTCVideoCapturerDelegate.swift */,
 				40483CB92C9B1E6000B4FCA8 /* MockWebRTCCoordinatorFactory.swift */,
 				40AF6A3A2C93469000BA2935 /* MockWebSocketClientFactory.swift */,
@@ -6606,6 +6611,7 @@
 				40C9E4552C988CE100802B28 /* WebRTCJoinRequestFactory_Tests.swift in Sources */,
 				84F58B9329EEB53E00010C4C /* EventMiddleware_Mock.swift in Sources */,
 				40FE5EBF2C9C82CD006B0881 /* CVPixelBuffer+Convenience.swift in Sources */,
+				40C75BB72CB4044600C167C3 /* MockThermalStateObserver.swift in Sources */,
 				842747EC29EED59000E063AD /* JSONDecoder_Tests.swift in Sources */,
 				406B3C142C8F870400FC93A1 /* MockActiveCallProvider.swift in Sources */,
 				841FF5052A5D815700809BBB /* VideoCapturerUtils_Tests.swift in Sources */,
@@ -6898,6 +6904,7 @@
 				40245F452BE2746300FCF075 /* CallIngressResponse+Dummy.swift in Sources */,
 				40245F462BE2746300FCF075 /* GeofenceSettings+Dummy.swift in Sources */,
 				40245F472BE2746300FCF075 /* CallRejectedEvent+Dummy.swift in Sources */,
+				40C75BB92CB40D8700C167C3 /* Mockable.swift in Sources */,
 				40FE5EC12C9C82CD006B0881 /* CVPixelBuffer+Convenience.swift in Sources */,
 				40245F482BE2746300FCF075 /* CallParticipantResponse+Dummy.swift in Sources */,
 				40245F492BE2746300FCF075 /* EgressHLSResponse+Dummy.swift in Sources */,
@@ -6913,6 +6920,7 @@
 				40245F672BE27B8400FCF075 /* StatelessSpeakerIconView_Tests.swift in Sources */,
 				40245F692BE27CCB00FCF075 /* StatelessParticipantsListButton_Tests.swift in Sources */,
 				40245F532BE2746300FCF075 /* TranscriptionSettings+Dummy.swift in Sources */,
+				40C75BB82CB4045100C167C3 /* MockThermalStateObserver.swift in Sources */,
 				40245F542BE2746300FCF075 /* BroadcastSettingsResponse+Dummy.swift in Sources */,
 				40245F552BE2746300FCF075 /* UserResponse+Dummy.swift in Sources */,
 				40245F562BE2746300FCF075 /* ScreensharingSettings+Dummy.swift in Sources */,

StreamVideoSwiftUITests/CallView/VideoRenderer_Tests.swift

@@ -8,51 +8,70 @@ import StreamSwiftTestHelpers
 @testable import StreamVideoSwiftUI
 import XCTest
 
+@MainActor
 final class VideoRenderer_Tests: XCTestCase {
 
-    private var mockThermalStateObserver: MockThermalStateObserver! = .init()
-    private var subject: VideoRenderer!
-
-    override func setUp() {
-        super.setUp()
-
-        InjectedValues[\.thermalStateObserver] = mockThermalStateObserver
-        subject = .init(frame: .zero)
-    }
+    private lazy var thermalStateSubject: PassthroughSubject<ProcessInfo.ThermalState, Never>! = .init()
+    private lazy var mockThermalStateObserver: MockThermalStateObserver! = .init()
+    private lazy var maximumFramesPerSecond: Int! = UIScreen.main.maximumFramesPerSecond
+    private lazy var subject: VideoRenderer! = .init(frame: .zero)
 
     override func tearDown() {
+        InjectedValues[\.thermalStateObserver] = ThermalStateObserver { .nominal }
+        thermalStateSubject = nil
         mockThermalStateObserver = nil
+        maximumFramesPerSecond = nil
+        subject = nil
         super.tearDown()
     }
 
     // MARK: - preferredFramesPerSecond
 
-    func testFPSForNominalThermalState() {
-        mockThermalStateObserver.state = .nominal
-        XCTAssertEqual(subject.preferredFramesPerSecond, UIScreen.main.maximumFramesPerSecond)
+    func testFPSForNominalThermalState() async {
+        await assertPreferredFramesPerSecond(
+            thermalState: .nominal,
+            expected: Double(maximumFramesPerSecond)
+        )
     }
 
-    func testFPSForFairThermalState() {
-        mockThermalStateObserver.state = .fair
-        XCTAssertEqual(subject.preferredFramesPerSecond, UIScreen.main.maximumFramesPerSecond)
+    func testFPSForFairThermalState() async {
+        await assertPreferredFramesPerSecond(
+            thermalState: .fair,
+            expected: Double(maximumFramesPerSecond)
+        )
     }
 
-    func testFPSForSeriousThermalState() {
-        mockThermalStateObserver.state = .serious
-        XCTAssertEqual(subject.preferredFramesPerSecond, Int(Double(UIScreen.main.maximumFramesPerSecond) * 0.5))
+    func testFPSForSeriousThermalState() async {
+        await assertPreferredFramesPerSecond(
+            thermalState: .serious,
+            expected: Double(maximumFramesPerSecond) * 0.5
+        )
     }
 
-    func testFPSForCriticalThermalState() {
-        mockThermalStateObserver.state = .critical
-        XCTAssertEqual(subject.preferredFramesPerSecond, Int(Double(UIScreen.main.maximumFramesPerSecond) * 0.4))
+    func testFPSForCriticalThermalState() async {
+        await assertPreferredFramesPerSecond(
+            thermalState: .critical,
+            expected: Double(maximumFramesPerSecond) * 0.4
+        )
     }
-}
 
-// MARK: - Private Helpers
+    // MARK: - Private helpers
+
+    private func assertPreferredFramesPerSecond(
+        thermalState: ProcessInfo.ThermalState,
+        expected: Double,
+        file: StaticString = #file,
+        line: UInt = #line
+    ) async {
+        mockThermalStateObserver.stub(
+            for: \.statePublisher,
+            with: thermalStateSubject.eraseToAnyPublisher()
+        )
+        _ = subject
+        thermalStateSubject.send(thermalState)
 
-private final class MockThermalStateObserver: ThermalStateObserving {
-    var state: ProcessInfo.ThermalState = .nominal { didSet { stateSubject.send(state) } }
-    lazy var stateSubject: CurrentValueSubject<ProcessInfo.ThermalState, Never> = .init(state)
-    var statePublisher: AnyPublisher<ProcessInfo.ThermalState, Never> { stateSubject.eraseToAnyPublisher() }
-    var scale: CGFloat = 1
+        await fulfillment(file: file, line: line) {
+            [subject] in subject?.preferredFramesPerSecond == Int(expected)
+        }
+    }
 }

StreamVideoTests/Controllers/CallController_Tests.swift

@@ -294,12 +294,13 @@ final class CallController_Tests: StreamVideoTestCase, @unchecked Sendable {
             .coordinator
             .changePinState(isEnabled: false, sessionId: user.id)
 
-        await assertNilAsync(
-            await mockWebRTCCoordinatorFactory
+        await fulfillment {
+            await self
+                .mockWebRTCCoordinatorFactory
                 .mockCoordinatorStack
                 .coordinator
-                .stateAdapter.participants[user.id]?.pin
-        )
+                .stateAdapter.participants[self.user.id]?.pin == nil
+        }
     }
 
     // MARK: - startNoiseCancellation

StreamVideoTests/Mock/MockThermalStateObserver.swift

@@ -0,0 +1,47 @@
+//
+// Copyright © 2024 Stream.io Inc. All rights reserved.
+//
+
+import Combine
+@testable import StreamVideo
+
+final class MockThermalStateObserver: ThermalStateObserving, Mockable {
+
+    // MARK: - Mockable
+
+    typealias FunctionKey = MockFunctionKey
+    typealias FunctionInputKey = EmptyPayloadable
+    var stubbedProperty: [String: Any] = [:]
+    var stubbedFunction: [FunctionKey: Any] = [:]
+    @Atomic var stubbedFunctionInput: [FunctionKey: [FunctionInputKey]] = [:]
+    func stub<T>(for keyPath: KeyPath<MockThermalStateObserver, T>, with value: T) {
+        stubbedProperty[propertyKey(for: keyPath)] = value
+    }
+
+    func stub<T>(for function: FunctionKey, with value: T) {}
+
+    enum MockFunctionKey: Hashable, CaseIterable {}
+
+    // MARK: - Properties
+
+    var state: ProcessInfo.ThermalState {
+        get { self[dynamicMember: \.state] }
+        set { _ = newValue }
+    }
+
+    var statePublisher: AnyPublisher<ProcessInfo.ThermalState, Never> {
+        get { self[dynamicMember: \.statePublisher] }
+        set { _ = newValue }
+    }
+
+    var scale: CGFloat {
+        get { self[dynamicMember: \.scale] }
+        set { _ = newValue }
+    }
+
+    init() {
+        InjectedValues[\.thermalStateObserver] = self
+        stub(for: \.state, with: .nominal)
+        stub(for: \.scale, with: 1)
+    }
+}