Skip to content

Commit

Permalink
[Feature]Toggle Transcriptions (#358)
Browse files Browse the repository at this point in the history
  • Loading branch information
ipavlidakis authored Apr 24, 2024
1 parent 87b4205 commit c7fed9e
Show file tree
Hide file tree
Showing 9 changed files with 406 additions and 45 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ private struct DemoMoreControlsViewModifier: ViewModifier {
)
}

DemoTranscriptionButtonView(viewModel: viewModel)

DemoMoreControlListButtonView(
action: { isStatsPresented = true },
label: "Stats"
Expand All @@ -133,6 +135,56 @@ private struct DemoMoreControlsViewModifier: ViewModifier {
}
}

struct DemoTranscriptionButtonView: View {

@ObservedObject var viewModel: CallViewModel
@State private var isTranscriptionAvailable = false
@State private var isTranscribing = false

init(viewModel: CallViewModel) {
self.viewModel = viewModel
isTranscriptionAvailable = (viewModel.call?.state.settings?.transcription.mode ?? .disabled) != .disabled
isTranscribing = viewModel.call?.state.transcribing == true
}

var body: some View {
Group {
if isTranscriptionAvailable {
DemoMoreControlListButtonView(
action: {
Task {
do {
if isTranscribing {
try await viewModel.call?.stopTranscription()
} else {
try await viewModel.call?.startTranscription()
}
} catch {
log.error(error)
}
}
},
label: isTranscribing ? "Disable Transcription" : "Transcription"
) {
Image(
systemName: isTranscribing
? "captions.bubble.fill"
: "captions.bubble"
)
}
.onReceive(viewModel.call?.state.$transcribing) { isTranscribing = $0 }
}
}
.onReceive(viewModel.call?.state.$settings) {
guard let mode = $0?.transcription.mode else {
isTranscriptionAvailable = false
return
}
isTranscriptionAvailable = mode != .disabled
}
}
}

extension View {

@ViewBuilder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
400D91D52B63E27300EBA47D /* 07-dependency-injection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 400D91D42B63E27300EBA47D /* 07-dependency-injection.swift */; };
404CAED82B8E3874007087BC /* 06-apply-video-filters.swift in Sources */ = {isa = PBXBuildFile; fileRef = 404CAED72B8E3874007087BC /* 06-apply-video-filters.swift */; };
4068C1252B67C056006B0BEE /* 03-callkit-integration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4068C1242B67C056006B0BEE /* 03-callkit-integration.swift */; };
408CE0F52BD91B490052EC3A /* 18-transcriptions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */; };
409C39692B67CC5C0090044C /* 04-screensharing.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C39682B67CC5C0090044C /* 04-screensharing.swift */; };
409C396B2B67CD0B0090044C /* 05-picture-in-picture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C396A2B67CD0B0090044C /* 05-picture-in-picture.swift */; };
409C396D2B67CD780090044C /* 08-recording.swift in Sources */ = {isa = PBXBuildFile; fileRef = 409C396C2B67CD780090044C /* 08-recording.swift */; };
Expand Down Expand Up @@ -88,6 +89,7 @@
400D91D42B63E27300EBA47D /* 07-dependency-injection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "07-dependency-injection.swift"; sourceTree = "<group>"; };
404CAED72B8E3874007087BC /* 06-apply-video-filters.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "06-apply-video-filters.swift"; sourceTree = "<group>"; };
4068C1242B67C056006B0BEE /* 03-callkit-integration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "03-callkit-integration.swift"; sourceTree = "<group>"; };
408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "18-transcriptions.swift"; sourceTree = "<group>"; };
409C39682B67CC5C0090044C /* 04-screensharing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "04-screensharing.swift"; sourceTree = "<group>"; };
409C396A2B67CD0B0090044C /* 05-picture-in-picture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "05-picture-in-picture.swift"; sourceTree = "<group>"; };
409C396C2B67CD780090044C /* 08-recording.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "08-recording.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -252,6 +254,7 @@
40FFDCA32B640772004DA7A2 /* 15-long-press-to-focus.swift */,
40AB356A2B73965D00E465CC /* 16-snapshot.swift */,
40CB9FA82B7FC7DB006BED93 /* 17-camera-zoom.swift */,
408CE0F42BD91B490052EC3A /* 18-transcriptions.swift */,
);
path = "05-ui-cookbook";
sourceTree = "<group>";
Expand Down Expand Up @@ -409,6 +412,7 @@
400D91D12B63DEA200EBA47D /* 04-camera-and-microphone.swift in Sources */,
40FFDC4F2B63EE50004DA7A2 /* 04-active-call.swift in Sources */,
40FFDCA02B640676004DA7A2 /* 13-pinning-users.swift in Sources */,
408CE0F52BD91B490052EC3A /* 18-transcriptions.swift in Sources */,
409C396D2B67CD780090044C /* 08-recording.swift in Sources */,
40FFDC602B63F2EC004DA7A2 /* 04-local-video.swift in Sources */,
40FFDC632B63F370004DA7A2 /* 02-sound-indicator.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import StreamVideo
import StreamVideoSwiftUI
import SwiftUI
import Combine
import AVFoundation

@MainActor
fileprivate func content() {
container {
struct TranscriptionButtonView: View {

@ObservedObject var viewModel: CallViewModel
@State private var isTranscriptionAvailable = false
@State private var isTranscribing = false

init(viewModel: CallViewModel) {
self.viewModel = viewModel
self.isTranscriptionAvailable = (viewModel.call?.state.settings?.transcription.mode ?? .disabled) != .disabled
self.isTranscribing = viewModel.call?.state.transcribing == true
}

var body: some View {
if let call = viewModel.call {
Group {
if isTranscriptionAvailable {
Button {
Task {
do {
if isTranscribing {
try await viewModel.call?.stopTranscription()
} else {
try await viewModel.call?.startTranscription()
}
} catch {
log.error(error)
}
}
} label: {
Label {
Text(isTranscribing ? "Disable Transcription" : "Transcription")
} icon: {
Image(
systemName: isTranscribing
? "captions.bubble.fill"
: "captions.bubble"
)
}
}
.onReceive(call.state.$transcribing) { isTranscribing = $0 }
}
}
.onReceive(call.state.$settings) {
guard let mode = $0?.transcription.mode else {
isTranscriptionAvailable = false
return
}
isTranscriptionAvailable = mode != .disabled
}
}
}
}
}
}
93 changes: 83 additions & 10 deletions Sources/StreamVideo/Call.swift
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,8 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
)
self.callController.call = self
subscribeToLocalCallSettingsChanges()
executeOnMain { [weak self] in
guard let self else { return }
self
.state
.$settings
.map(\.?.audio.noiseCancellation)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
subscribeToNoiseCancellationSettingsChanges()
subscribeToTranscriptionSettingsChanges()
}

internal convenience init(
Expand Down Expand Up @@ -893,6 +885,36 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
try callController.zoom(by: factor)
}

/// Starts transcribing a conversation, optionally specifying an external storage location.
///
/// - Parameter transcriptionExternalStorage: The external storage location for the
/// transcription (optional).
@discardableResult
public func startTranscription(
transcriptionExternalStorage: String? = nil
) async throws -> StartTranscriptionResponse {
try await coordinatorClient.startTranscription(
type: callType,
id: callId,
startTranscriptionRequest: .init(
transcriptionExternalStorage: transcriptionExternalStorage
)
)
}

/// Stops a conversation from being transcribed and returns whether the stop request was successful
/// or not.
///
/// - Returns: A StopTranscriptionResponse indicating whether the stop request was successful
/// or not.
@discardableResult
public func stopTranscription() async throws -> StopTranscriptionResponse {
try await coordinatorClient.stopTranscription(
type: callType,
id: callId
)
}

// MARK: - Internal

internal func update(reconnectionStatus: ReconnectionStatus) {
Expand Down Expand Up @@ -1017,6 +1039,31 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
.store(in: &cancellables)
}

private func subscribeToNoiseCancellationSettingsChanges() {
executeOnMain { [weak self] in
guard let self else { return }
self
.state
.$settings
.map(\.?.audio.noiseCancellation)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
}

private func subscribeToTranscriptionSettingsChanges() {
executeOnMain {
self
.state
.$settings
.map(\.?.transcription)
.removeDuplicates()
.sink { [weak self] in self?.didUpdate($0) }
.store(in: &self.cancellables)
}
}

private func updateCallSettingsManagers(with callSettings: CallSettings) {
microphone.status = callSettings.audioOn ? .enabled : .disabled
camera.status = callSettings.videoOn ? .enabled : .disabled
Expand Down Expand Up @@ -1071,4 +1118,30 @@ public class Call: @unchecked Sendable, WSEventsSubscriber {
log.debug("NoiseCancellationSettings updated. No action!")
}
}

/// Handles updates to transcription settings.
/// - Parameter value: The updated `TranscriptionSettings` value.
private func didUpdate(_ value: TranscriptionSettings?) {
guard let value else {
log.debug("TranscriptionSettings updated. No action!")
return
}

Task { @MainActor in
do {
switch value.mode {
case .disabled where state.transcribing == true:
log.debug("TranscriptionSettings updated with mode:\(value.mode). Will deactivate transcriptions.")
try await stopTranscription()
case .autoOn where state.transcribing == false:
log.debug("TranscriptionSettings updated with mode:\(value.mode). Will activate transcriptions.")
try await startTranscription()
default:
log.debug("TranscriptionSettings updated with mode:\(value.mode). No action required.")
}
} catch {
log.error(error)
}
}
}
}
6 changes: 3 additions & 3 deletions Sources/StreamVideo/CallState.swift
Original file line number Diff line number Diff line change
Expand Up @@ -192,13 +192,13 @@ public class CallState: ObservableObject {
case .typeClosedCaptionEvent:
break
case .typeCallTranscriptionFailedEvent:
break
transcribing = false
case .typeCallTranscriptionReadyEvent:
break
case .typeCallTranscriptionStartedEvent:
break
transcribing = true
case .typeCallTranscriptionStoppedEvent:
break
transcribing = false
}
}

Expand Down
9 changes: 9 additions & 0 deletions Sources/StreamVideo/Utils/Models+Sendable.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//
// Copyright © 2024 Stream.io Inc. All rights reserved.
//

import Foundation

extension StartTranscriptionResponse: @unchecked Sendable {}
extension StopTranscriptionResponse: @unchecked Sendable {}
extension TranscriptionSettings: @unchecked Sendable {}
4 changes: 4 additions & 0 deletions StreamVideo.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@
408CE0F72BD95EB60052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F82BD95F170052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F92BD95F1B0052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
408CE0F32BD905920052EC3A /* Models+Sendable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F22BD905920052EC3A /* Models+Sendable.swift */; };
408D29A22B6D209700885473 /* SnapshotViewModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29A12B6D209700885473 /* SnapshotViewModifier.swift */; };
408D29A42B6D251600885473 /* UIView+Snapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29A32B6D251600885473 /* UIView+Snapshot.swift */; };
408D29AD2B6D680D00885473 /* SnapshotTrigger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408D29AB2B6D638100885473 /* SnapshotTrigger.swift */; };
Expand Down Expand Up @@ -1135,6 +1136,7 @@
407D5D3C2ACEF0C500B5044E /* VisibilityThresholdModifier_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisibilityThresholdModifier_Tests.swift; sourceTree = "<group>"; };
408679F62BD12F1000D027E0 /* AudioFilter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFilter.swift; sourceTree = "<group>"; };
408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VideoConfig+Dummy.swift"; sourceTree = "<group>"; };
408CE0F22BD905920052EC3A /* Models+Sendable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Models+Sendable.swift"; sourceTree = "<group>"; };
408D29A12B6D209700885473 /* SnapshotViewModifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SnapshotViewModifier.swift; sourceTree = "<group>"; };
408D29A32B6D251600885473 /* UIView+Snapshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIView+Snapshot.swift"; sourceTree = "<group>"; };
408D29AB2B6D638100885473 /* SnapshotTrigger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SnapshotTrigger.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -3459,6 +3461,7 @@
40CB9FA32B7F8EA4006BED93 /* AVCaptureSession+ActiveCaptureDevice.swift */,
40013DDB2B87AA2300915453 /* SerialActor.swift */,
40A0E9612B88D3DC0089E8D3 /* UIInterfaceOrientation+CGOrientation.swift */,
408CE0F22BD905920052EC3A /* Models+Sendable.swift */,
);
path = Utils;
sourceTree = "<group>";
Expand Down Expand Up @@ -4899,6 +4902,7 @@
8490032529D308A000AD9BB4 /* GetCallResponse.swift in Sources */,
841947982886D9CD0007B36E /* BundleExtensions.swift in Sources */,
842E70D32B91BE1700D2D68B /* CallRecordingReadyEvent.swift in Sources */,
408CE0F32BD905920052EC3A /* Models+Sendable.swift in Sources */,
842E70D42B91BE1700D2D68B /* CallTranscription.swift in Sources */,
84274F462884249A00CF8794 /* ConnectionRecoveryHandler.swift in Sources */,
842B8E2F2A2DFED900863A87 /* StopRecordingResponse.swift in Sources */,
Expand Down
Loading

0 comments on commit c7fed9e

Please sign in to comment.