Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
…into latest-protobuf-end-event
  • Loading branch information
martinmitrevski committed Apr 25, 2024
2 parents 973ba6d + c7fed9e commit 34b4f7a
Show file tree
Hide file tree
Showing 18 changed files with 487 additions and 91 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ private struct DemoMoreControlsViewModifier: ViewModifier {
)
}

DemoTranscriptionButtonView(viewModel: viewModel)

DemoMoreControlListButtonView(
action: { isStatsPresented = true },
label: "Stats"
Expand All @@ -133,6 +135,56 @@ private struct DemoMoreControlsViewModifier: ViewModifier {
}
}

struct DemoTranscriptionButtonView: View {

@ObservedObject var viewModel: CallViewModel
@State private var isTranscriptionAvailable = false
@State private var isTranscribing = false

init(viewModel: CallViewModel) {
self.viewModel = viewModel
isTranscriptionAvailable = (viewModel.call?.state.settings?.transcription.mode ?? .disabled) != .disabled
isTranscribing = viewModel.call?.state.transcribing == true
}

var body: some View {
Group {
if isTranscriptionAvailable {
DemoMoreControlListButtonView(
action: {
Task {
do {
if isTranscribing {
try await viewModel.call?.stopTranscription()
} else {
try await viewModel.call?.startTranscription()
}
} catch {
log.error(error)
}
}
},
label: isTranscribing ? "Disable Transcription" : "Transcription"
) {
Image(
systemName: isTranscribing
? "captions.bubble.fill"
: "captions.bubble"
)
}
.onReceive(viewModel.call?.state.$transcribing) { isTranscribing = $0 }
}
}
.onReceive(viewModel.call?.state.$settings) {
guard let mode = $0?.transcription.mode else {
isTranscriptionAvailable = false
return
}
isTranscriptionAvailable = mode != .disabled
}
}
}

extension View {

@ViewBuilder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
400D91D52B63E27300EBA47D /* 07-dependency-injection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 400D91D42B63E27300EBA47D /* 07-dependency-injection.swift */; };
404CAED82B8E3874007087BC /* 06-apply-video-filters.swift in Sources */ = {isa = PBXBuildFile; fileRef = 404CAED72B8E3874007087BC /* 06-apply-video-filters.swift */; };
4068C1252B67C056006B0BEE /* 03-callkit-integration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4068C1242B67C056006B0BEE /* 03-callkit-integration.swift */; };
408CE0F52BD91B490052EC3A /* 18-transcriptions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */; };
409C39692B67CC5C0090044C /* 04-screensharing.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C39682B67CC5C0090044C /* 04-screensharing.swift */; };
409C396B2B67CD0B0090044C /* 05-picture-in-picture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C396A2B67CD0B0090044C /* 05-picture-in-picture.swift */; };
409C396D2B67CD780090044C /* 08-recording.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C396C2B67CD780090044C /* 08-recording.swift */; };
Expand Down Expand Up @@ -88,6 +89,7 @@
400D91D42B63E27300EBA47D /* 07-dependency-injection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "07-dependency-injection.swift"; sourceTree = "<group>"; };
404CAED72B8E3874007087BC /* 06-apply-video-filters.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "06-apply-video-filters.swift"; sourceTree = "<group>"; };
4068C1242B67C056006B0BEE /* 03-callkit-integration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "03-callkit-integration.swift"; sourceTree = "<group>"; };
408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "18-transcriptions.swift"; sourceTree = "<group>"; };
409C39682B67CC5C0090044C /* 04-screensharing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "04-screensharing.swift"; sourceTree = "<group>"; };
409C396A2B67CD0B0090044C /* 05-picture-in-picture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "05-picture-in-picture.swift"; sourceTree = "<group>"; };
409C396C2B67CD780090044C /* 08-recording.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "08-recording.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -252,6 +254,7 @@
40FFDCA32B640772004DA7A2 /* 15-long-press-to-focus.swift */,
40AB356A2B73965D00E465CC /* 16-snapshot.swift */,
40CB9FA82B7FC7DB006BED93 /* 17-camera-zoom.swift */,
408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */,
);
path = "05-ui-cookbook";
sourceTree = "<group>";
Expand Down Expand Up @@ -409,6 +412,7 @@
400D91D12B63DEA200EBA47D /* 04-camera-and-microphone.swift in Sources */,
40FFDC4F2B63EE50004DA7A2 /* 04-active-call.swift in Sources */,
40FFDCA02B640676004DA7A2 /* 13-pinning-users.swift in Sources */,
408CE0F52BD91B490052EC3A /* 18-transcriptions.swift in Sources */,
409C396D2B67CD780090044C /* 08-recording.swift in Sources */,
40FFDC602B63F2EC004DA7A2 /* 04-local-video.swift in Sources */,
40FFDC632B63F370004DA7A2 /* 02-sound-indicator.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import StreamVideo
import StreamVideoSwiftUI
import SwiftUI
import Combine
import AVFoundation

@MainActor
fileprivate func content() {
container {
struct TranscriptionButtonView: View {

@ObservedObject var viewModel: CallViewModel
@State private var isTranscriptionAvailable = false
@State private var isTranscribing = false

init(viewModel: CallViewModel) {
self.viewModel = viewModel
self.isTranscriptionAvailable = (viewModel.call?.state.settings?.transcription.mode ?? .disabled) != .disabled
self.isTranscribing = viewModel.call?.state.transcribing == true
}

var body: some View {
if let call = viewModel.call {
Group {
if isTranscriptionAvailable {
Button {
Task {
do {
if isTranscribing {
try await viewModel.call?.stopTranscription()
} else {
try await viewModel.call?.startTranscription()
}
} catch {
log.error(error)
}
}
} label: {
Label {
Text(isTranscribing ? "Disable Transcription" : "Transcription")
} icon: {
Image(
systemName: isTranscribing
? "captions.bubble.fill"
: "captions.bubble"
)
}
}
.onReceive(call.state.$transcribing) { isTranscribing = $0 }
}
}
.onReceive(call.state.$settings) {
guard let mode = $0?.transcription.mode else {
isTranscriptionAvailable = false
return
}
isTranscriptionAvailable = mode != .disabled
}
}
}
}
}
}
93 changes: 83 additions & 10 deletions Sources/StreamVideo/Call.swift
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,8 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
)
self.callController.call = self
subscribeToLocalCallSettingsChanges()
executeOnMain { [weak self] in
guard let self else { return }
self
.state
.$settings
.map(\.?.audio.noiseCancellation)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
subscribeToNoiseCancellationSettingsChanges()
subscribeToTranscriptionSettingsChanges()
}

internal convenience init(
Expand Down Expand Up @@ -893,6 +885,36 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
try callController.zoom(by: factor)
}

/// Starts transcribing a conversation, optionally specifying an external storage location.
///
/// - Parameter transcriptionExternalStorage: The external storage location for the
/// transcription (optional).
@discardableResult
public func startTranscription(
transcriptionExternalStorage: String? = nil
) async throws -> StartTranscriptionResponse {
try await coordinatorClient.startTranscription(
type: callType,
id: callId,
startTranscriptionRequest: .init(
transcriptionExternalStorage: transcriptionExternalStorage
)
)
}

/// Stops a conversation from being transcribed and returns whether the stop request was successful
/// or not.
///
/// - Returns: A StopTranscriptionResponse indicating whether the stop request was successful
/// or not.
@discardableResult
public func stopTranscription() async throws -> StopTranscriptionResponse {
try await coordinatorClient.stopTranscription(
type: callType,
id: callId
)
}

// MARK: - Internal

internal func update(reconnectionStatus: ReconnectionStatus) {
Expand Down Expand Up @@ -1017,6 +1039,31 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
.store(in: &cancellables)
}

private func subscribeToNoiseCancellationSettingsChanges() {
executeOnMain { [weak self] in
guard let self else { return }
self
.state
.$settings
.map(\.?.audio.noiseCancellation)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
}

private func subscribeToTranscriptionSettingsChanges() {
executeOnMain {
self
.state
.$settings
.map(\.?.transcription)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
}

private func updateCallSettingsManagers(with callSettings: CallSettings) {
microphone.status = callSettings.audioOn ? .enabled : .disabled
camera.status = callSettings.videoOn ? .enabled : .disabled
Expand Down Expand Up @@ -1071,4 +1118,30 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
log.debug("NoiseCancellationSettings updated. No action!")
}
}

/// Handles updates to transcription settings.
/// - Parameter value: The updated `TranscriptionSettings` value.
private func didUpdate(_ value: TranscriptionSettings?) {
guard let value else {
log.debug("TranscriptionSettings updated. No action!")
return
}

Task { @MainActor in
do {
switch value.mode {
case .disabled where state.transcribing == true:
log.debug("TranscriptionSettings updated with mode:\(value.mode). Will deactivate transcriptions.")
try await stopTranscription()
case .autoOn where state.transcribing == false:
log.debug("TranscriptionSettings updated with mode:\(value.mode). Will activate transcriptions.")
try await startTranscription()
default:
log.debug("TranscriptionSettings updated with mode:\(value.mode). No action required.")
}
} catch {
log.error(error)
}
}
}
}
6 changes: 3 additions & 3 deletions Sources/StreamVideo/CallState.swift
Original file line number Diff line number Diff line change
Expand Up @@ -192,13 +192,13 @@ public class CallState: ObservableObject {
case .typeClosedCaptionEvent:
break
case .typeCallTranscriptionFailedEvent:
break
transcribing = false
case .typeCallTranscriptionReadyEvent:
break
case .typeCallTranscriptionStartedEvent:
break
transcribing = true
case .typeCallTranscriptionStoppedEvent:
break
transcribing = false
}
}

Expand Down
9 changes: 9 additions & 0 deletions Sources/StreamVideo/Utils/Models+Sendable.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//
// Copyright © 2024 Stream.io Inc. All rights reserved.
//

import Foundation

extension StartTranscriptionResponse: @unchecked Sendable {}
extension StopTranscriptionResponse: @unchecked Sendable {}
extension TranscriptionSettings: @unchecked Sendable {}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ open class StreamAudioFilterProcessingModule: NSObject, RTCAudioProcessingModule
self.capturePostProcessingDelegate = capturePostProcessingDelegate
}

deinit {
processingModule.capturePostProcessingDelegate = nil
processingModule.renderPreProcessingDelegate = nil
capturePostProcessingDelegate.audioProcessingRelease()
}

public func apply(_ config: RTCAudioProcessingConfig) {
processingModule.apply(config)
}
Expand Down
12 changes: 12 additions & 0 deletions StreamVideo.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@
407F29FF2AA6011500C3EAF8 /* MemoryLogViewer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4093861E2AA0A21800FF5AF4 /* MemoryLogViewer.swift */; };
407F2A002AA6011B00C3EAF8 /* LogQueue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4093861B2AA0A11500FF5AF4 /* LogQueue.swift */; };
408679F72BD12F1000D027E0 /* AudioFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408679F62BD12F1000D027E0 /* AudioFilter.swift */; };
408CE0F72BD95EB60052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F82BD95F170052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F92BD95F1B0052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F32BD905920052EC3A /* Models+Sendable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F22BD905920052EC3A /* Models+Sendable.swift */; };
408D29A22B6D209700885473 /* SnapshotViewModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29A12B6D209700885473 /* SnapshotViewModifier.swift */; };
408D29A42B6D251600885473 /* UIView+Snapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29A32B6D251600885473 /* UIView+Snapshot.swift */; };
408D29AD2B6D680D00885473 /* SnapshotTrigger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29AB2B6D638100885473 /* SnapshotTrigger.swift */; };
Expand Down Expand Up @@ -1131,6 +1135,8 @@
407AF7192B6163DD00E9E3E7 /* StreamMediaDurationFormatter_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamMediaDurationFormatter_Tests.swift; sourceTree = "<group>"; };
407D5D3C2ACEF0C500B5044E /* VisibilityThresholdModifier_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisibilityThresholdModifier_Tests.swift; sourceTree = "<group>"; };
408679F62BD12F1000D027E0 /* AudioFilter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFilter.swift; sourceTree = "<group>"; };
408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VideoConfig+Dummy.swift"; sourceTree = "<group>"; };
408CE0F22BD905920052EC3A /* Models+Sendable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Models+Sendable.swift"; sourceTree = "<group>"; };
408D29A12B6D209700885473 /* SnapshotViewModifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SnapshotViewModifier.swift; sourceTree = "<group>"; };
408D29A32B6D251600885473 /* UIView+Snapshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIView+Snapshot.swift"; sourceTree = "<group>"; };
408D29AB2B6D638100885473 /* SnapshotTrigger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SnapshotTrigger.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -3334,6 +3340,7 @@
842747F229EED8D900E063AD /* InternetConnection_Mock.swift */,
82E3BA522A0BAF4B001AB93E /* WebSocketClientEnvironment_Mock.swift */,
406303412AD848000091AE77 /* CallParticipant_Mock.swift */,
408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */,
);
path = Mock;
sourceTree = "<group>";
Expand Down Expand Up @@ -3454,6 +3461,7 @@
40CB9FA32B7F8EA4006BED93 /* AVCaptureSession+ActiveCaptureDevice.swift */,
40013DDB2B87AA2300915453 /* SerialActor.swift */,
40A0E9612B88D3DC0089E8D3 /* UIInterfaceOrientation+CGOrientation.swift */,
408CE0F22BD905920052EC3A /* Models+Sendable.swift */,
);
path = Utils;
sourceTree = "<group>";
Expand Down Expand Up @@ -4894,6 +4902,7 @@
8490032529D308A000AD9BB4 /* GetCallResponse.swift in Sources */,
841947982886D9CD0007B36E /* BundleExtensions.swift in Sources */,
842E70D32B91BE1700D2D68B /* CallRecordingReadyEvent.swift in Sources */,
408CE0F32BD905920052EC3A /* Models+Sendable.swift in Sources */,
842E70D42B91BE1700D2D68B /* CallTranscription.swift in Sources */,
84274F462884249A00CF8794 /* ConnectionRecoveryHandler.swift in Sources */,
842B8E2F2A2DFED900863A87 /* StopRecordingResponse.swift in Sources */,
Expand Down Expand Up @@ -5029,6 +5038,7 @@
84F58B7829EE945900010C4C /* RetryStrategy_Tests.swift in Sources */,
84DCA2182A398C53000C3411 /* Call_Tests.swift in Sources */,
40F017472BBEEF5100E89FD1 /* ThumbnailResponse+Dummy.swift in Sources */,
408CE0F72BD95EB60052EC3A /* VideoConfig+Dummy.swift in Sources */,
84F58B7029EE914400010C4C /* BackgroundTaskScheduler_Tests.swift in Sources */,
842747F329EED8D900E063AD /* InternetConnection_Mock.swift in Sources */,
8478A0EC29F2604A0001F860 /* ControllerTestCase.swift in Sources */,
Expand Down Expand Up @@ -5318,6 +5328,7 @@
82E3BA4C2A0BAE40001AB93E /* EquatableEvent.swift in Sources */,
40914C9C2B56AA6600F6A13E /* StreamBufferTransformerTests.swift in Sources */,
82FF40C42A17C74D00B4D95E /* IncomingCallView_Tests.swift in Sources */,
408CE0F82BD95F170052EC3A /* VideoConfig+Dummy.swift in Sources */,
829F7BFA29FABC0E003EBACE /* ViewFactory.swift in Sources */,
8457BF802A5C18A7000AE567 /* ToastView_Tests.swift in Sources */,
82FF40C82A17C75C00B4D95E /* LobbyView_Tests.swift in Sources */,
Expand Down Expand Up @@ -5347,6 +5358,7 @@
82E3BA512A0BAEAB001AB93E /* MockFunc.swift in Sources */,
82E3BA472A0BAE1F001AB93E /* VirtualTime.swift in Sources */,
82E3BA392A0BADB9001AB93E /* CallController_Mock.swift in Sources */,
408CE0F92BD95F1B0052EC3A /* VideoConfig+Dummy.swift in Sources */,
82E3BA432A0BAE0A001AB93E /* EventBatcher_Mock.swift in Sources */,
8493227E29093A420013C029 /* StreamVideo_Mock.swift in Sources */,
82E3BA4F2A0BAE4E001AB93E /* VirtualTimer.swift in Sources */,
Expand Down
Loading

0 comments on commit 34b4f7a

Please sign in to comment.