From 6bf69130d6eb4edf036b53c6b2fe070d2beaf04b Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Mon, 6 Nov 2023 13:38:37 -0500 Subject: [PATCH 01/42] improve CallConfig --- extensions/warp-blink-wrtc/src/signaling.rs | 1 - warp/src/blink/call_config.rs | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/signaling.rs index 323c2db65..0b08d06d6 100644 --- a/extensions/warp-blink-wrtc/src/signaling.rs +++ b/extensions/warp-blink-wrtc/src/signaling.rs @@ -28,7 +28,6 @@ pub enum CallSignal { Join { call_id: Uuid }, #[display(fmt = "Leave")] Leave { call_id: Uuid }, - #[display(fmt = "Muted")] Muted, #[display(fmt = "Unmuted")] diff --git a/warp/src/blink/call_config.rs b/warp/src/blink/call_config.rs index dcc9a1938..5b91e1e51 100644 --- a/warp/src/blink/call_config.rs +++ b/warp/src/blink/call_config.rs @@ -1,10 +1,18 @@ +use std::collections::HashSet; + use crate::crypto::DID; #[derive(Default, Debug, Clone)] pub struct CallConfig { - pub recording: bool, + pub self_recording: bool, pub self_muted: bool, pub self_deafened: bool, - pub participants_muted: Vec, - pub participants_deafened: Vec, + pub participants_joined: HashSet, +} + +#[derive(Default, Debug, Clone)] +pub struct ParticipantState { + pub muted: bool, + pub deafened: bool, + pub recording: bool, } From 5feb3f6eeb50b4c07bdef2ba9258933917933075 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Mon, 6 Nov 2023 14:56:52 -0500 Subject: [PATCH 02/42] add file to handle sending stuff over gossipsub --- .../src/blink_impl/gossipsub_sender.rs | 128 ++++++++++++++++++ .../warp-blink-wrtc/src/blink_impl/mod.rs | 2 + extensions/warp-blink-wrtc/src/signaling.rs | 12 -- extensions/warp-blink-wrtc/src/store.rs | 4 +- 4 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs new file mode 100644 index 000000000..2aeaa1fad --- /dev/null +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -0,0 +1,128 @@ +use std::{fmt::Display, sync::Arc}; + +use rust_ipfs::Ipfs; +use serde::Serialize; +use tokio::sync::{ + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, +}; +use warp::crypto::{cipher::Cipher, DID}; + +use crate::store::ecdh_encrypt; + +pub enum GossipSubSignal { + Aes { + group_key: Vec, + signal: Vec, + topic: String, + }, + Ecdh { + dest: DID, + signal: Vec, + topic: String, + }, +} + +pub struct GossibSubSender { + // used for signing messages + ch: UnboundedSender, + notify: Arc, +} + +pub fn init(own_id: DID, ipfs: Ipfs) -> GossibSubSender { + let (tx, rx) = mpsc::unbounded_channel(); + let notify = Arc::new(Notify::new()); + let notify2 = notify.clone(); + tokio::spawn(async move { + run(own_id, ipfs, rx, notify2).await; + }); + GossibSubSender { ch: tx, notify } +} + +impl Drop for GossibSubSender { + fn drop(&mut self) { + self.notify.notify_waiters(); + } +} + +impl GossibSubSender { + pub fn send_signal_aes( + &self, + group_key: Vec, + signal: T, + topic: String, + ) -> anyhow::Result<()> { + let signal = serde_cbor::to_vec(&signal)?; + self.ch.send(GossipSubSignal::Aes { + group_key, + signal, + topic, + }); + + Ok(()) + } + + pub fn send_signal_ecdh( + &self, + dest: DID, + signal: T, + topic: String, + ) -> anyhow::Result<()> { + let signal = serde_cbor::to_vec(&signal)?; + self.ch.send(GossipSubSignal::Ecdh { + dest, + signal, + topic, + }); + + Ok(()) + } +} + +async fn run( + own_id: DID, + ipfs: Ipfs, + mut ch: UnboundedReceiver, + notify: Arc, +) { + loop { + tokio::select! { + opt = ch.recv() => match opt { + Some(cmd) => match cmd { + GossipSubSignal::Aes { group_key, signal, topic } => { + let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt aes message"); + continue; + } + }; + if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { + log::error!("failed to publish message"); + } + }, + GossipSubSignal::Ecdh { dest, signal, topic } => { + let encrypted = match ecdh_encrypt(&own_id, &dest, signal) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt ecdh message"); + continue; + } + }; + if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { + log::error!("failed to publish message"); + } + } + } + None => { + log::debug!("GossipSubTask channel closed"); + return; + } + }, + _ = notify.notified() => { + log::debug!("GossipSubTask terminated"); + return; + } + } + } +} diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 46ae58460..7e2977930 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -8,6 +8,8 @@ mod webrtc_handler; use webrtc_handler::run as handle_webrtc; use webrtc_handler::WebRtcHandlerParams; +mod gossipsub_sender; + use anyhow::{bail, Context}; use async_trait::async_trait; use cpal::traits::{DeviceTrait, HostTrait}; diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/signaling.rs index 0b08d06d6..cdc271f82 100644 --- a/extensions/warp-blink-wrtc/src/signaling.rs +++ b/extensions/warp-blink-wrtc/src/signaling.rs @@ -43,18 +43,6 @@ pub enum InitiationSignal { /// invite a peer to join a call #[display(fmt = "Offer")] Offer { call_info: CallInfo }, - /// used to dismiss an incoming call dialog - /// is needed when someone offers a call and - /// everyone who joined the call leaves. if this - /// happens and someone hasn't rejected the call, - /// they may have a call dialog displayed. they need - /// to track how many people joined and left the call to - /// know when to dismiss the dialog. - #[display(fmt = "Join")] - Join { call_id: Uuid }, - /// used to dismiss an incoming call dialog - #[display(fmt = "Leave")] - Leave { call_id: Uuid }, } pub mod ipfs_routes { diff --git a/extensions/warp-blink-wrtc/src/store.rs b/extensions/warp-blink-wrtc/src/store.rs index e05cc46b1..a7d0ba6b1 100644 --- a/extensions/warp-blink-wrtc/src/store.rs +++ b/extensions/warp-blink-wrtc/src/store.rs @@ -78,7 +78,7 @@ pub fn decode_gossipsub_msg_aes( Ok(data) } -fn ecdh_encrypt>(own_did: &DID, recipient: &DID, data: K) -> Result> { +pub fn ecdh_encrypt>(own_did: &DID, recipient: &DID, data: K) -> Result> { let prikey = Ed25519KeyPair::from_secret_key(&own_did.private_key_bytes()).get_x25519(); let did_pubkey = recipient.public_key_bytes(); @@ -89,7 +89,7 @@ fn ecdh_encrypt>(own_did: &DID, recipient: &DID, data: K) -> Resu Ok(data) } -fn ecdh_decrypt>(own_did: &DID, sender: &DID, data: K) -> Result> { +pub fn ecdh_decrypt>(own_did: &DID, sender: &DID, data: K) -> Result> { let prikey = Ed25519KeyPair::from_secret_key(&own_did.private_key_bytes()).get_x25519(); let did_pubkey = sender.public_key_bytes(); From 7e6fcccb8a8bde563a705c3906b681433216a22d Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Mon, 6 Nov 2023 17:02:43 -0500 Subject: [PATCH 03/42] add gossipsub listener --- .../src/blink_impl/gossipsub_listener.rs | 305 ++++++++++++++++++ .../src/blink_impl/gossipsub_sender.rs | 101 ++++-- .../warp-blink-wrtc/src/blink_impl/mod.rs | 1 + extensions/warp-blink-wrtc/src/signaling.rs | 23 +- 4 files changed, 402 insertions(+), 28 deletions(-) create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs new file mode 100644 index 000000000..103115fc7 --- /dev/null +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -0,0 +1,305 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures::StreamExt; +use rust_ipfs::Ipfs; +use tokio::sync::{ + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, +}; +use uuid::Uuid; +use warp::crypto::DID; + +use crate::{ + signaling::{ + ipfs_routes::{call_initiation_route, call_signal_route, peer_signal_route}, + CallSignal, GossipSubSignal, InitiationSignal, PeerSignal, + }, + store::PeerIdExt, +}; + +use super::gossipsub_sender::GossipSubSender; + +enum GossipSubCmd { + // unsubscribe from the call and close any webrtc connections + UnsubCall { call_id: Uuid }, + DisconnectWebrtc { call_id: Uuid }, + // receive call wide broadcasts + SubCall { call_id: Uuid, group_key: Vec }, + // webrtc signaling for a peer + ConnectWebRtc { call_id: Uuid, peer: DID }, + // allow peers to offer calls + ReceiveCalls { own_id: DID }, +} +pub struct GossipSubListener { + ch: UnboundedSender, + // when GossipSubSender gets cloned, NotifyWrapper doesn't get cloned. + // when NotifyWrapper finally gets dropped, then it's ok to call notify_waiters + notify: Arc, +} + +struct NotifyWrapper { + notify: Arc, +} + +impl Drop for NotifyWrapper { + fn drop(&mut self) { + self.notify.notify_waiters(); + } +} + +impl GossipSubListener { + pub fn init( + ipfs: Ipfs, + event_ch: UnboundedReceiver, + rsp_ch: UnboundedSender, + gossipsub_sender: GossipSubSender, + ) -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + let notify = Arc::new(Notify::new()); + let notify2 = notify.clone(); + tokio::spawn(async move { + run(ipfs, rx, rsp_ch, gossipsub_sender, notify2).await; + }); + Self { + ch: tx, + notify: Arc::new(NotifyWrapper { notify }), + } + } + + pub fn unsubscribe_call(&self, call_id: Uuid) { + let _ = self.ch.send(GossipSubCmd::UnsubCall { call_id }); + } + + pub fn unsubscribe_webrtc(&self, call_id: Uuid) { + let _ = self.ch.send(GossipSubCmd::DisconnectWebrtc { call_id }); + } +} + +async fn run( + ipfs: Ipfs, + mut ch: UnboundedReceiver, + tx: UnboundedSender, + gossipsub_sender: GossipSubSender, + notify: Arc, +) { + // for tracking webrtc subscriptions + let mut current_call: Option = None; + let mut connected_peers: HashMap> = HashMap::new(); + let mut subscribed_calls: HashMap> = HashMap::new(); + + let call_signal_notify = Arc::new(Notify::new()); + let call_offer_notify = Arc::new(Notify::new()); + loop { + tokio::select! { + opt = ch.recv() => match opt { + Some(cmd) => match cmd { + GossipSubCmd::UnsubCall { call_id } => { + if let Some(call) = subscribed_calls.remove(&call_id) { + call.notify_waiters(); + } + if matches!(current_call.as_ref(), Some(&call_id)) { + let _ = current_call.take(); + for (_peer_id, notify) in connected_peers.drain() { + notify.notify_waiters(); + } + } + } + GossipSubCmd::DisconnectWebrtc { call_id } => { + if matches!(current_call.as_ref(), Some(&call_id)) { + for (_peer_id, notify) in connected_peers.drain() { + notify.notify_waiters(); + } + } + } + GossipSubCmd::SubCall { call_id, group_key } => { + let notify = Arc::new(Notify::new()); + if let Some(prev) = subscribed_calls.insert(call_id, notify.clone()) { + prev.notify_waiters(); + } + + let mut call_signal_stream = match ipfs + .pubsub_subscribe(call_signal_route(&call_id)) + .await + { + Ok(s) => s, + Err(e) => { + log::error!("failed to subscribe to call signal stream: {e}"); + continue; + } + }; + + let ch = tx.clone(); + let gossipsub_sender = gossipsub_sender.clone(); + tokio::spawn(async move { + loop { + tokio::select!{ + _ = notify.notified() => { + log::debug!("call signal stream terminated by notify"); + break; + } + opt = call_signal_stream.next() => match opt { + Some(msg) => { + let sender = match msg.source.and_then(|s| s.to_did().ok()) { + Some(id) => id, + None => { + log::error!("msg received without source"); + continue + } + }; + match gossipsub_sender.decode_signal_aes::(group_key.clone(), msg.data.clone()).await { + Ok(msg) => { + let _ = ch.send(GossipSubSignal::Call{ + sender, + call_id: call_id.clone(), + signal: msg + }); + }, + Err(e) => { + log::error!("failed to decode call signal: {e}"); + } + }; + } + None => { + log::debug!("call signal stream terminated!"); + break; + } + } + }; + } + }); + }, + GossipSubCmd::ConnectWebRtc { call_id, peer } => { + if !matches!(current_call.as_ref(), Some(&call_id)) { + if current_call.is_some() { + for (_peer_id, notify) in connected_peers.drain() { + notify.notify_waiters(); + } + } + current_call.replace(call_id); + } + let notify = Arc::new(Notify::new()); + if let Some(prev) = connected_peers.insert(peer.clone(), notify.clone()) { + prev.notify_waiters(); + } + + let mut peer_signal_stream = match ipfs + .pubsub_subscribe(peer_signal_route(&peer, &call_id)) + .await + { + Ok(s) => s, + Err(e) => { + log::error!("failed to subscribe to peer signal stream: {e}"); + continue; + } + }; + let ch = tx.clone(); + let gossipsub_sender = gossipsub_sender.clone(); + tokio::spawn(async move { + loop { + tokio::select!{ + _ = notify.notified() => { + log::debug!("peer signal stream terminated by notify"); + break; + } + opt = peer_signal_stream.next() => match opt { + Some(msg) => { + let sender = match msg.source.and_then(|s| s.to_did().ok()) { + Some(id) => id, + None => { + log::error!("msg received without source"); + continue + } + }; + match gossipsub_sender.decode_signal_ecdh::(sender.clone(), msg.data.clone()).await { + Ok(msg) => { + let _ = ch.send(GossipSubSignal::Peer { + sender, + call_id: call_id.clone(), + signal: msg + }); + }, + Err(e) => { + log::error!("failed to decode peer signal: {e}"); + } + }; + } + None => { + log::debug!("peer signal stream terminated!"); + break; + } + } + }; + } + }); + }, + GossipSubCmd::ReceiveCalls { own_id } => { + call_offer_notify.notify_waiters(); + let mut call_offer_stream = match ipfs + .pubsub_subscribe(call_initiation_route(&own_id)) + .await + { + Ok(s) => s, + Err(e) => { + log::error!("failed to subscribe to call offer stream: {e}"); + continue; + } + }; + let ch = tx.clone(); + let notify = call_offer_notify.clone(); + let gossipsub_sender = gossipsub_sender.clone(); + tokio::spawn(async move { + loop { + tokio::select!{ + _ = notify.notified() => { + log::debug!("call offer stream terminated by notify"); + break; + } + opt = call_offer_stream.next() => match opt { + Some(msg) => { + let sender = match msg.source.and_then(|s| s.to_did().ok()) { + Some(id) => id, + None => { + log::error!("msg received without source"); + continue + } + }; + match gossipsub_sender.decode_signal_ecdh::(sender.clone(), msg.data.clone()).await { + Ok(msg) => { + let _ = ch.send(GossipSubSignal::Initiation{ + sender, + signal: msg + }); + }, + Err(e) => { + log::error!("failed to decode call offer: {e}"); + } + }; + } + None => { + log::debug!("call offer stream terminated!"); + break; + } + } + }; + } + }); + }, + } + None => { + log::debug!("GossipSubListener channel closed"); + break; + } + }, + _ = notify.notified() => { + log::debug!("GossipSubListener terminated"); + break; + } + } + + for (_peer_id, notify) in connected_peers.drain() { + notify.notify_waiters(); + } + call_signal_notify.notify_waiters(); + call_offer_notify.notify_waiters(); + } +} diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 2aeaa1fad..94732aa7d 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -1,51 +1,67 @@ use std::{fmt::Display, sync::Arc}; +use futures::channel::oneshot; use rust_ipfs::Ipfs; -use serde::Serialize; +use serde::{de::DeserializeOwned, Serialize}; use tokio::sync::{ mpsc::{self, UnboundedReceiver, UnboundedSender}, Notify, }; use warp::crypto::{cipher::Cipher, DID}; -use crate::store::ecdh_encrypt; +use crate::store::{ecdh_decrypt, ecdh_encrypt}; -pub enum GossipSubSignal { - Aes { +enum GossipSubCmd { + SendAes { group_key: Vec, signal: Vec, topic: String, }, - Ecdh { + SendEcdh { dest: DID, signal: Vec, topic: String, }, + DecodeEcdh { + src: DID, + data: Vec, + rsp: oneshot::Sender>>, + }, } -pub struct GossibSubSender { +#[derive(Clone)] +pub struct GossipSubSender { // used for signing messages - ch: UnboundedSender, - notify: Arc, + ch: UnboundedSender, + // when GossipSubSender gets cloned, NotifyWrapper doesn't get cloned. + // when NotifyWrapper finally gets dropped, then it's ok to call notify_waiters + notify: Arc, } -pub fn init(own_id: DID, ipfs: Ipfs) -> GossibSubSender { - let (tx, rx) = mpsc::unbounded_channel(); - let notify = Arc::new(Notify::new()); - let notify2 = notify.clone(); - tokio::spawn(async move { - run(own_id, ipfs, rx, notify2).await; - }); - GossibSubSender { ch: tx, notify } +struct NotifyWrapper { + notify: Arc, } -impl Drop for GossibSubSender { +impl Drop for NotifyWrapper { fn drop(&mut self) { self.notify.notify_waiters(); } } -impl GossibSubSender { +impl GossipSubSender { + pub fn init(own_id: DID, ipfs: Ipfs) -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + let notify = Arc::new(Notify::new()); + let notify2 = notify.clone(); + tokio::spawn(async move { + run(own_id, ipfs, rx, notify2).await; + }); + Self { + ch: tx, + notify: Arc::new(NotifyWrapper { notify }), + } + } + pub fn send_signal_aes( &self, group_key: Vec, @@ -53,7 +69,7 @@ impl GossibSubSender { topic: String, ) -> anyhow::Result<()> { let signal = serde_cbor::to_vec(&signal)?; - self.ch.send(GossipSubSignal::Aes { + self.ch.send(GossipSubCmd::SendAes { group_key, signal, topic, @@ -69,7 +85,7 @@ impl GossibSubSender { topic: String, ) -> anyhow::Result<()> { let signal = serde_cbor::to_vec(&signal)?; - self.ch.send(GossipSubSignal::Ecdh { + self.ch.send(GossipSubCmd::SendEcdh { dest, signal, topic, @@ -77,19 +93,46 @@ impl GossibSubSender { Ok(()) } + + // this one doesn't require access to own_id. it can be decrypted using just the group key. + pub async fn decode_signal_aes( + &self, + group_key: Vec, + message: Vec, + ) -> anyhow::Result { + let decrypted = Cipher::direct_decrypt(&message, &group_key)?; + let data: T = serde_cbor::from_slice(&decrypted)?; + Ok(data) + } + + pub async fn decode_signal_ecdh( + &self, + src: DID, + message: Vec, + ) -> anyhow::Result { + let (tx, rx) = oneshot::channel(); + self.ch.send(GossipSubCmd::DecodeEcdh { + src, + data: message, + rsp: tx, + }); + let bytes = rx.await??; + let data: T = serde_cbor::from_slice(&bytes)?; + Ok(data) + } } async fn run( own_id: DID, ipfs: Ipfs, - mut ch: UnboundedReceiver, + mut ch: UnboundedReceiver, notify: Arc, ) { loop { tokio::select! { opt = ch.recv() => match opt { Some(cmd) => match cmd { - GossipSubSignal::Aes { group_key, signal, topic } => { + GossipSubCmd::SendAes { group_key, signal, topic } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { Ok(r) => r, Err(e) => { @@ -101,7 +144,7 @@ async fn run( log::error!("failed to publish message"); } }, - GossipSubSignal::Ecdh { dest, signal, topic } => { + GossipSubCmd::SendEcdh { dest, signal, topic } => { let encrypted = match ecdh_encrypt(&own_id, &dest, signal) { Ok(r) => r, Err(e) => { @@ -113,14 +156,22 @@ async fn run( log::error!("failed to publish message"); } } + GossipSubCmd::DecodeEcdh { src, data, rsp } => { + let r = || { + let bytes = ecdh_decrypt(&own_id, &src, &data)?; + Ok(bytes) + }; + + rsp.send(r()); + } } None => { - log::debug!("GossipSubTask channel closed"); + log::debug!("GossibSubSender channel closed"); return; } }, _ = notify.notified() => { - log::debug!("GossipSubTask terminated"); + log::debug!("GossibSubSender terminated"); return; } } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 7e2977930..b6581e2c7 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -8,6 +8,7 @@ mod webrtc_handler; use webrtc_handler::run as handle_webrtc; use webrtc_handler::WebRtcHandlerParams; +mod gossipsub_listener; mod gossipsub_sender; use anyhow::{bail, Context}; diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/signaling.rs index cdc271f82..549b46188 100644 --- a/extensions/warp-blink-wrtc/src/signaling.rs +++ b/extensions/warp-blink-wrtc/src/signaling.rs @@ -2,12 +2,29 @@ use derive_more::Display; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use warp::blink::CallInfo; +use warp::{blink::CallInfo, crypto::DID}; use webrtc::{ ice_transport::ice_candidate::RTCIceCandidate, peer_connection::sdp::session_description::RTCSessionDescription, }; +pub enum GossipSubSignal { + Peer { + sender: DID, + call_id: Uuid, + signal: PeerSignal, + }, + Call { + sender: DID, + call_id: Uuid, + signal: CallSignal, + }, + Initiation { + sender: DID, + signal: InitiationSignal, + }, +} + #[derive(Serialize, Deserialize, Display)] pub enum PeerSignal { #[display(fmt = "Ice")] @@ -25,9 +42,9 @@ pub enum PeerSignal { #[derive(Serialize, Deserialize, Display)] pub enum CallSignal { #[display(fmt = "Join")] - Join { call_id: Uuid }, + Join, #[display(fmt = "Leave")] - Leave { call_id: Uuid }, + Leave, #[display(fmt = "Muted")] Muted, #[display(fmt = "Unmuted")] From 0b8373772bc437756e71f5e88b8eb93f91a6470a Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Mon, 6 Nov 2023 20:47:20 -0500 Subject: [PATCH 04/42] wire up public functions for gossipsub_listener --- .../src/blink_impl/gossipsub_listener.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 103115fc7..3761605ad 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -21,10 +21,10 @@ use super::gossipsub_sender::GossipSubSender; enum GossipSubCmd { // unsubscribe from the call and close any webrtc connections - UnsubCall { call_id: Uuid }, + UnsubscribeCall { call_id: Uuid }, DisconnectWebrtc { call_id: Uuid }, // receive call wide broadcasts - SubCall { call_id: Uuid, group_key: Vec }, + SubscribeCall { call_id: Uuid, group_key: Vec }, // webrtc signaling for a peer ConnectWebRtc { call_id: Uuid, peer: DID }, // allow peers to offer calls @@ -67,12 +67,26 @@ impl GossipSubListener { } pub fn unsubscribe_call(&self, call_id: Uuid) { - let _ = self.ch.send(GossipSubCmd::UnsubCall { call_id }); + let _ = self.ch.send(GossipSubCmd::UnsubscribeCall { call_id }); } pub fn unsubscribe_webrtc(&self, call_id: Uuid) { let _ = self.ch.send(GossipSubCmd::DisconnectWebrtc { call_id }); } + + pub fn subscribe_call(&self, call_id: Uuid, group_key: Vec) { + let _ = self + .ch + .send(GossipSubCmd::SubscribeCall { call_id, group_key }); + } + + pub fn connect_webrtc(&self, call_id: Uuid, peer: DID) { + let _ = self.ch.send(GossipSubCmd::ConnectWebRtc { call_id, peer }); + } + + pub fn receive_calls(&self, own_id: DID) { + let _ = self.ch.send(GossipSubCmd::ReceiveCalls { own_id }); + } } async fn run( @@ -93,7 +107,7 @@ async fn run( tokio::select! { opt = ch.recv() => match opt { Some(cmd) => match cmd { - GossipSubCmd::UnsubCall { call_id } => { + GossipSubCmd::UnsubscribeCall { call_id } => { if let Some(call) = subscribed_calls.remove(&call_id) { call.notify_waiters(); } @@ -111,7 +125,7 @@ async fn run( } } } - GossipSubCmd::SubCall { call_id, group_key } => { + GossipSubCmd::SubscribeCall { call_id, group_key } => { let notify = Arc::new(Notify::new()); if let Some(prev) = subscribed_calls.insert(call_id, notify.clone()) { prev.notify_waiters(); From ffc4a9dd4b705791916c82ff1bcd524dc12aa8ff Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Tue, 7 Nov 2023 13:21:38 -0500 Subject: [PATCH 05/42] saving progress --- .../src/blink_impl/data/mod.rs | 3 + .../src/blink_impl/data/notify_wrapper.rs | 12 + .../src/blink_impl/event_handler.rs | 281 ++++++++++++ .../src/blink_impl/gossipsub_listener.rs | 74 +-- .../src/blink_impl/gossipsub_sender.rs | 67 ++- .../warp-blink-wrtc/src/blink_impl/mod.rs | 423 +++++------------- .../src/blink_impl/webrtc_handler.rs | 20 +- .../warp-blink-wrtc/src/host_media/mod.rs | 105 ++++- extensions/warp-blink-wrtc/src/signaling.rs | 7 +- .../warp-blink-wrtc/src/simple_webrtc/mod.rs | 6 +- warp/src/blink/call_config.rs | 4 +- warp/src/blink/mod.rs | 4 + 12 files changed, 620 insertions(+), 386 deletions(-) create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/data/notify_wrapper.rs create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index 4e007ed10..700c816a2 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -4,6 +4,9 @@ use warp::{ crypto::DID, }; +mod notify_wrapper; +pub use notify_wrapper::*; + #[derive(Clone)] pub struct ActiveCall { pub call: CallInfo, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/notify_wrapper.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/notify_wrapper.rs new file mode 100644 index 000000000..1c39ff0d8 --- /dev/null +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/notify_wrapper.rs @@ -0,0 +1,12 @@ +use std::sync::Arc; +use tokio::sync::Notify; + +pub struct NotifyWrapper { + pub notify: Arc, +} + +impl Drop for NotifyWrapper { + fn drop(&mut self) { + self.notify.notify_waiters(); + } +} diff --git a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs new file mode 100644 index 000000000..5ad625b34 --- /dev/null +++ b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs @@ -0,0 +1,281 @@ +use futures::channel::oneshot; +use futures::StreamExt; +use serde::{de::DeserializeOwned, Serialize}; +use std::{collections::HashMap, fmt::Display, sync::Arc, time::Duration}; +use tokio::{ + sync::{ + broadcast, + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, + }, + time::Instant, +}; +use uuid::Uuid; +use warp::{ + blink::{BlinkEventKind, CallConfig, CallInfo, ParticipantState}, + crypto::{cipher::Cipher, DID}, + sync::RwLock, +}; +use webrtc::{ + rtp_transceiver::rtp_codec::RTCRtpCodecCapability, + track::track_local::track_local_static_rtp::TrackLocalStaticRTP, +}; + +use crate::{ + blink_impl::data::{CallState, PeerState}, + host_media::{ + self, + audio::{AudioCodec, AudioHardwareConfig}, + }, + signaling::{ipfs_routes, GossipSubSignal, PeerSignal}, + simple_webrtc::{self, events::WebRtcEventStream, MediaSourceId}, +}; + +use super::{data::NotifyWrapper, gossipsub_sender::GossipSubSender}; + +enum EventHandlerCmd { + SetActiveCall { + call_info: CallInfo, + }, + AddMediaSource { + source_id: MediaSourceId, + codec: RTCRtpCodecCapability, + rsp: oneshot::Sender>>, + }, + RemoveMediaSource { + source_id: MediaSourceId, + }, + LeaveCall, +} + +#[derive(Clone)] +pub struct EventHandler { + ch: UnboundedSender, + notify: Arc, +} + +impl EventHandler { + pub fn new( + webrtc_controller: simple_webrtc::Controller, + webrtc_event_stream: WebRtcEventStream, + gossipsub_sender: GossipSubSender, + signal_rx: UnboundedReceiver, + event_ch: broadcast::Sender, + ) -> Self { + let (tx, cmd_rx) = mpsc::unbounded_channel(); + let notify = Arc::new(Notify::new()); + let notify2 = notify.clone(); + tokio::spawn(async move { + run( + webrtc_controller, + webrtc_event_stream, + gossipsub_sender, + cmd_rx, + signal_rx, + event_ch, + notify2, + ) + .await; + }); + Self { + ch: tx, + notify: Arc::new(NotifyWrapper { notify }), + } + } + + pub fn set_active_call(&self, call_info: CallInfo) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::SetActiveCall { call_info })?; + Ok(()) + } + + pub async fn add_media_source( + &self, + source_id: MediaSourceId, + codec: RTCRtpCodecCapability, + ) -> anyhow::Result> { + let (tx, rx) = oneshot::channel(); + self.ch.send(EventHandlerCmd::AddMediaSource { + source_id, + codec, + rsp: tx, + })?; + rx.await? + } + + pub fn remove_media_source(&self, source_id: MediaSourceId) -> anyhow::Result<()> { + self.ch + .send(EventHandlerCmd::RemoveMediaSource { source_id })?; + Ok(()) + } + + pub fn leave_call(&self) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::LeaveCall)?; + Ok(()) + } +} + +async fn run( + mut webrtc_controller: simple_webrtc::Controller, + mut webrtc_event_stream: WebRtcEventStream, + gossipsub_sender: GossipSubSender, + mut cmd_rx: UnboundedReceiver, + mut signal_rx: UnboundedReceiver, + event_ch: broadcast::Sender, + notify: Arc, +) { + let mut call_configs: HashMap = HashMap::new(); + let mut call_infos: HashMap = HashMap::new(); + let mut active_call: Option = None; + let mut active_call_state = CallState::Uninitialized; + + loop { + tokio::select! { + _ = notify.notified() => { + log::debug!("quitting blink event handler"); + break; + }, + opt = cmd_rx.recv() => { + let cmd = match opt { + Some(r) => r, + None => { + log::debug!("blink handler cmd_rx channel is closed. quitting"); + break; + } + }; + match cmd { + EventHandlerCmd::SetActiveCall { call_info } => { + if active_call.replace(call_info.call_id()).is_some() { + webrtc_controller.deinit(); + host_media::reset().await; + } + call_infos.insert(call_info.call_id(), call_info); + }, + EventHandlerCmd::AddMediaSource { source_id, codec, rsp } => { + let r = webrtc_controller.add_media_source(source_id, codec).await; + let _ = rsp.send(r); + }, + EventHandlerCmd::RemoveMediaSource { source_id } => { + let _ = webrtc_controller.remove_media_source(source_id).await; + }, + EventHandlerCmd::LeaveCall => { + if active_call.take().is_some() { + webrtc_controller.deinit(); + host_media::reset().await; + } + }, + } + }, + opt = signal_rx.recv() => { + let cmd = match opt { + Some(r) => r, + None => { + log::debug!("blink handler signal_rx channel is closed. quitting"); + break; + } + }; + match cmd { + GossipSubSignal::Peer { sender, call_id, signal } => match signal { + _ if !matches!(active_call.as_ref(), Some(&call_id)) => { + log::debug!("received webrtc signal for non-active call"); + continue; + } + _ if active_call_state != CallState::Started => { + log::debug!("received signal for uninitialized call: {:?}", std::mem::discriminant(&signal)); + continue; + } + crate::signaling::PeerSignal::Ice(ice) => { + if let Err(e) = webrtc_controller.recv_ice(&sender, ice).await { + log::error!("failed to recv_ice {}", e); + } + }, + crate::signaling::PeerSignal::Sdp(sdp) => { + log::debug!("received signal: SDP"); + if let Err(e) = webrtc_controller.recv_sdp(&sender, sdp).await { + log::error!("failed to recv_sdp: {}", e); + } + }, + crate::signaling::PeerSignal::Dial(sdp) => { + log::debug!("received signal: Dial"); + // emits the SDP Event, which is sent to the peer via the SDP signal + if let Err(e) = webrtc_controller.accept_call(&sender, sdp).await { + log::error!("failed to accept_call: {}", e); + } + }, + }, + GossipSubSignal::Call { sender, call_id, signal } => match signal { + crate::signaling::CallSignal::Join => todo!(), + crate::signaling::CallSignal::Leave => todo!(), + crate::signaling::CallSignal::Muted => todo!(), + crate::signaling::CallSignal::Unmuted => todo!(), + crate::signaling::CallSignal::Deafened => todo!(), + crate::signaling::CallSignal::Undeafened => todo!(), + }, + GossipSubSignal::Initiation { sender, signal } => match signal { + crate::signaling::InitiationSignal::Offer { call_info } => todo!(), + }, + } + } + opt = webrtc_event_stream.next() => { + let event = match opt { + Some(r) => r, + None => { + log::debug!("webrtc_event_stream closed!"); + // todo: get new webrtc controller or something + continue; + } + }; + + match event { + simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { + let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); + let signal = PeerSignal::Ice(*candidate); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + }, + simple_webrtc::events::EmittedEvents::Connected { peer } => { + let ac = active_call.unwrap_or_default(); + if call_infos.get(&ac).map(|x| x.contains_participant(&peer)).unwrap_or_default() { + if let Some(config) = call_configs.get_mut(&ac) { + config.participants_joined.insert(peer, ParticipantState::default()); + } + } else { + log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); + webrtc_controller.hang_up(&peer).await; + } + }, + simple_webrtc::events::EmittedEvents::Disconnected { peer } + | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { + if let Some(config) = call_configs.get_mut(&active_call.unwrap_or_default()) { + config.participants_joined.remove(&peer); + } + if let Err(e) = host_media::remove_sink_track(peer.clone()).await { + log::error!("failed to send media_track command: {e}"); + } + webrtc_controller.hang_up(&peer).await; + }, + simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => todo!(), + simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { + let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); + let signal = PeerSignal::Sdp(*sdp); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + }, + simple_webrtc::events::EmittedEvents::CallInitiated { dest, sdp } => { + let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); + let signal = PeerSignal::Dial(*sdp); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + }, + simple_webrtc::events::EmittedEvents::TrackAdded { peer, track } => { + if let Err(e) = host_media::create_audio_sink_track(peer.clone(), event_ch.clone(), track, AudioCodec::default()).await { + log::error!("failed to send media_track command: {e}"); + } + }, + } + } + } + } +} diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 3761605ad..c24ca9150 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -1,13 +1,16 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use futures::StreamExt; use rust_ipfs::Ipfs; -use tokio::sync::{ - mpsc::{self, UnboundedReceiver, UnboundedSender}, - Notify, +use tokio::{ + sync::{ + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, + }, + time::Instant, }; use uuid::Uuid; -use warp::crypto::DID; +use warp::{crypto::DID, sync::RwLock}; use crate::{ signaling::{ @@ -17,7 +20,7 @@ use crate::{ store::PeerIdExt, }; -use super::gossipsub_sender::GossipSubSender; +use super::{data::NotifyWrapper, gossipsub_sender::GossipSubSender}; enum GossipSubCmd { // unsubscribe from the call and close any webrtc connections @@ -30,6 +33,7 @@ enum GossipSubCmd { // allow peers to offer calls ReceiveCalls { own_id: DID }, } +#[derive(Clone)] pub struct GossipSubListener { ch: UnboundedSender, // when GossipSubSender gets cloned, NotifyWrapper doesn't get cloned. @@ -37,19 +41,9 @@ pub struct GossipSubListener { notify: Arc, } -struct NotifyWrapper { - notify: Arc, -} - -impl Drop for NotifyWrapper { - fn drop(&mut self) { - self.notify.notify_waiters(); - } -} - impl GossipSubListener { - pub fn init( - ipfs: Ipfs, + pub fn new( + ipfs: Arc>>, event_ch: UnboundedReceiver, rsp_ch: UnboundedSender, gossipsub_sender: GossipSubSender, @@ -90,17 +84,36 @@ impl GossipSubListener { } async fn run( - ipfs: Ipfs, + ipfs: Arc>>, mut ch: UnboundedReceiver, tx: UnboundedSender, gossipsub_sender: GossipSubSender, notify: Arc, ) { + let notify2 = notify.clone(); + let mut timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(100), + Duration::from_millis(100), + ); + let ipfs = loop { + tokio::select! { + _ = notify2.notified() => { + log::debug!("GossibSubListener channel closed"); + return; + }, + _ = timer.tick() => { + if ipfs.read().is_some() { + break ipfs.read().clone().unwrap(); + } + } + } + }; + // for tracking webrtc subscriptions let mut current_call: Option = None; - let mut connected_peers: HashMap> = HashMap::new(); let mut subscribed_calls: HashMap> = HashMap::new(); + let webrtc_notify = Arc::new(Notify::new()); let call_signal_notify = Arc::new(Notify::new()); let call_offer_notify = Arc::new(Notify::new()); loop { @@ -113,16 +126,12 @@ async fn run( } if matches!(current_call.as_ref(), Some(&call_id)) { let _ = current_call.take(); - for (_peer_id, notify) in connected_peers.drain() { - notify.notify_waiters(); - } + webrtc_notify.notify_waiters(); } } GossipSubCmd::DisconnectWebrtc { call_id } => { if matches!(current_call.as_ref(), Some(&call_id)) { - for (_peer_id, notify) in connected_peers.drain() { - notify.notify_waiters(); - } + webrtc_notify.notify_waiters(); } } GossipSubCmd::SubscribeCall { call_id, group_key } => { @@ -185,16 +194,10 @@ async fn run( GossipSubCmd::ConnectWebRtc { call_id, peer } => { if !matches!(current_call.as_ref(), Some(&call_id)) { if current_call.is_some() { - for (_peer_id, notify) in connected_peers.drain() { - notify.notify_waiters(); - } + webrtc_notify.notify_waiters(); } current_call.replace(call_id); } - let notify = Arc::new(Notify::new()); - if let Some(prev) = connected_peers.insert(peer.clone(), notify.clone()) { - prev.notify_waiters(); - } let mut peer_signal_stream = match ipfs .pubsub_subscribe(peer_signal_route(&peer, &call_id)) @@ -207,6 +210,7 @@ async fn run( } }; let ch = tx.clone(); + let notify = webrtc_notify.clone(); let gossipsub_sender = gossipsub_sender.clone(); tokio::spawn(async move { loop { @@ -310,9 +314,7 @@ async fn run( } } - for (_peer_id, notify) in connected_peers.drain() { - notify.notify_waiters(); - } + webrtc_notify.notify_waiters(); call_signal_notify.notify_waiters(); call_offer_notify.notify_waiters(); } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 94732aa7d..b20a5b673 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -1,16 +1,24 @@ -use std::{fmt::Display, sync::Arc}; +use std::{fmt::Display, sync::Arc, time::Duration}; use futures::channel::oneshot; use rust_ipfs::Ipfs; use serde::{de::DeserializeOwned, Serialize}; -use tokio::sync::{ - mpsc::{self, UnboundedReceiver, UnboundedSender}, - Notify, +use tokio::{ + sync::{ + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, + }, + time::Instant, +}; +use warp::{ + crypto::{cipher::Cipher, DID}, + sync::RwLock, }; -use warp::crypto::{cipher::Cipher, DID}; use crate::store::{ecdh_decrypt, ecdh_encrypt}; +use super::data::NotifyWrapper; + enum GossipSubCmd { SendAes { group_key: Vec, @@ -38,18 +46,8 @@ pub struct GossipSubSender { notify: Arc, } -struct NotifyWrapper { - notify: Arc, -} - -impl Drop for NotifyWrapper { - fn drop(&mut self) { - self.notify.notify_waiters(); - } -} - impl GossipSubSender { - pub fn init(own_id: DID, ipfs: Ipfs) -> Self { + pub fn new(own_id: Arc>>, ipfs: Arc>>) -> Self { let (tx, rx) = mpsc::unbounded_channel(); let notify = Arc::new(Notify::new()); let notify2 = notify.clone(); @@ -123,11 +121,44 @@ impl GossipSubSender { } async fn run( - own_id: DID, - ipfs: Ipfs, + own_id: Arc>>, + ipfs: Arc>>, mut ch: UnboundedReceiver, notify: Arc, ) { + let notify2 = notify.clone(); + let mut timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(100), + Duration::from_millis(100), + ); + let own_id = loop { + tokio::select! { + _ = notify2.notified() => { + log::debug!("GossibSubSender channel closed"); + return; + }, + _ = timer.tick() => { + if own_id.read().is_some() { + break own_id.write().take().unwrap(); + } + } + } + }; + + let ipfs = loop { + tokio::select! { + _ = notify2.notified() => { + log::debug!("GossibSubSender channel closed"); + return; + }, + _ = timer.tick() => { + if ipfs.read().is_some() { + break ipfs.read().clone().unwrap(); + } + } + } + }; + loop { tokio::select! { opt = ch.recv() => match opt { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index b6581e2c7..fce9ddafe 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -5,22 +5,18 @@ mod data; use data::*; mod webrtc_handler; -use webrtc_handler::run as handle_webrtc; -use webrtc_handler::WebRtcHandlerParams; +mod event_handler; mod gossipsub_listener; mod gossipsub_sender; -use anyhow::{bail, Context}; +use anyhow::bail; use async_trait::async_trait; use cpal::traits::{DeviceTrait, HostTrait}; use rust_ipfs::{Ipfs, Keypair}; -use std::{any::Any, collections::HashMap, str::FromStr, sync::Arc, time::Duration}; +use std::{any::Any, str::FromStr, sync::Arc, time::Duration}; use tokio::{ - sync::{ - broadcast::{self}, - RwLock, - }, + sync::broadcast::{self}, task::JoinHandle, }; use uuid::Uuid; @@ -39,54 +35,41 @@ use crate::{ self, audio::{ automute::{AutoMuteCmd, AUDIO_CMD_CH}, - AudioCodec, AudioHardwareConfig, AudioSampleRate, + AudioCodec, }, mp4_logger::Mp4LoggerConfig, }, signaling::{ipfs_routes, CallSignal, InitiationSignal}, - simple_webrtc::{self, events::WebRtcEventStream}, - store::{send_signal_aes, send_signal_ecdh}, + simple_webrtc::{self}, }; +use self::gossipsub_listener::GossipSubListener; +use self::gossipsub_sender::GossipSubSender; + // implements Blink #[derive(Clone)] pub struct BlinkImpl { - ipfs: Arc>>, - pending_calls: Arc>>, - active_call: Arc>>, - webrtc_controller: Arc>, - // the DID generated from Multipass, never cloned. contains the private key - own_id: Arc>>, + // the DID generated from Multipass. has been cloned. doesn't contain the private key anymore. + own_id: Arc>>, ui_event_ch: broadcast::Sender, - audio_source_config: Arc>, - audio_sink_config: Arc>, // subscribes to IPFS topic to receive incoming calls offer_handler: Arc>>, - // handles 3 streams: one for webrtc events and two IPFS topics - // pertains to the active_call, which is stored in STATIC_DATA - webrtc_handler: Arc>>>, - // prevents the UI from running multiple tests simultaneously - audio_device_config: Arc>, + gossipsub_listener: GossipSubListener, + gossipsub_sender: GossipSubSender, + event_handler: event_handler::EventHandler, + + drop_handler: Arc, } -impl Drop for BlinkImpl { +struct DropHandler {} +impl Drop for DropHandler { fn drop(&mut self) { - let webrtc_handler = std::mem::take(&mut *self.webrtc_handler.write()); - if let Some(handle) = webrtc_handler { - handle.abort(); - } - self.offer_handler.write().abort(); - let webrtc_controller = self.webrtc_controller.clone(); tokio::spawn(async move { - if let Err(e) = webrtc_controller.write().await.deinit().await { - log::error!("error in webrtc_controller deinit: {e}"); - } host_media::audio::automute::stop(); host_media::reset().await; - //rtp_logger::deinit().await; - log::debug!("deinit finished"); + log::debug!("blink drop handler finished"); }); } } @@ -95,69 +78,53 @@ impl BlinkImpl { pub async fn new(account: Box) -> anyhow::Result> { log::trace!("initializing WebRTC"); - // check SupportedStreamConfigs. if those channels aren't supported, use the default. - let mut source_config = AudioHardwareConfig { - sample_rate: AudioSampleRate::High, - channels: 1, - }; - - let mut sink_config = AudioHardwareConfig { - sample_rate: AudioSampleRate::High, - channels: 1, - }; - let mut selected_speaker = None; let mut selected_microphone = None; let cpal_host = cpal::default_host(); if let Some(input_device) = cpal_host.default_input_device() { selected_microphone = input_device.name().ok(); - match Self::get_min_source_channels(&input_device) { - Ok(channels) => { - source_config.channels = channels; - host_media::change_audio_input(input_device, source_config.clone()).await?; - } - Err(e) => log::error!("{e}"), - } + host_media::change_audio_input(input_device).await?; } else { log::warn!("blink started with no input device"); } if let Some(output_device) = cpal_host.default_output_device() { selected_speaker = output_device.name().ok(); - match Self::get_min_sink_channels(&output_device) { - Ok(channels) => { - sink_config.channels = channels; - host_media::change_audio_output(output_device, sink_config.clone()).await?; - } - Err(e) => log::error!("{e}"), - } + host_media::change_audio_output(output_device).await?; } else { log::warn!("blink started with no output device"); } + let ipfs = Arc::new(warp::sync::RwLock::new(None)); + let own_id_private = Arc::new(warp::sync::RwLock::new(None)); + let gossipsub_sender = GossipSubSender::new(own_id_private, ipfs.clone()); + let gossipsub_listener = + GossipSubListener::new(ipfs.clone(), todo!(), todo!(), gossipsub_sender.clone()); + + let webrtc_controller = simple_webrtc::Controller::new()?; + let webrtc_event_stream = webrtc_controller.get_event_stream(); + let event_handler = event_handler::EventHandler::new( + webrtc_controller, + webrtc_event_stream, + gossipsub_sender.clone(), + todo!(), + ); + let (ui_event_ch, _rx) = broadcast::channel(1024); let blink_impl = Self { - ipfs: Arc::new(RwLock::new(None)), - pending_calls: Arc::new(RwLock::new(HashMap::new())), - active_call: Arc::new(RwLock::new(None)), - webrtc_controller: Arc::new(RwLock::new(simple_webrtc::Controller::new()?)), - own_id: Arc::new(RwLock::new(None)), + own_id: Arc::new(warp::sync::RwLock::new(None)), ui_event_ch, - audio_source_config: Arc::new(RwLock::new(source_config)), - audio_sink_config: Arc::new(RwLock::new(sink_config)), offer_handler: Arc::new(warp::sync::RwLock::new(tokio::spawn(async {}))), - webrtc_handler: Arc::new(warp::sync::RwLock::new(None)), - audio_device_config: Arc::new(RwLock::new(host_media::audio::DeviceConfig::new( - selected_speaker, - selected_microphone, - ))), + gossipsub_sender, + gossipsub_listener, + event_handler, + drop_handler: Arc::new(DropHandler {}), }; - let ipfs = blink_impl.ipfs.clone(); let own_id = blink_impl.own_id.clone(); let offer_handler = blink_impl.offer_handler.clone(); - let pending_calls = blink_impl.pending_calls.clone(); let ui_event_ch = blink_impl.ui_event_ch.clone(); + let gossipsub_listener = blink_impl.gossipsub_listener.clone(); tokio::spawn(async move { let f = async move { @@ -179,35 +146,15 @@ impl BlinkImpl { None => bail!("Unable to use IPFS Handle"), }; - let call_offer_stream = match _ipfs - .pubsub_subscribe(ipfs_routes::call_initiation_route(&identity.did_key())) - .await - { - Ok(s) => s, - Err(e) => { - log::error!("failed to subscribe to call_broadcast_route: {e}"); - return Err(e); - } - }; - let _own_id = get_keypair_did(_ipfs.keypair()?)?; - own_id.write().await.replace(_own_id); - - let _offer_handler = tokio::spawn(async move { - handle_call_initiation( - own_id.clone(), - pending_calls, - call_offer_stream, - ui_event_ch, - ) - .await; - }); - - let mut x = offer_handler.write(); - *x = _offer_handler; - - // set ipfs last to quickly detect that webrtc hasn't been initialized. - ipfs.write().await.replace(_ipfs); + let public_did = _own_id.clone(); + // this one better not be cloned + own_id_private.write().replace(_own_id); + // this one is for blink and can be cloned. might not even be needed. + own_id.write().replace(public_did.clone()); + ipfs.write().replace(_ipfs); + + gossipsub_listener.receive_calls(public_did); log::trace!("finished initializing WebRTC"); Ok(()) }; @@ -224,12 +171,15 @@ impl BlinkImpl { async fn init_call(&mut self, call: CallInfo) -> Result<(), Error> { //rtp_logger::init(call.call_id(), std::path::PathBuf::from("")).await?; - let lock = self.own_id.read().await; - let own_id = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - let ipfs = self.get_ipfs().await?; - self.active_call.write().await.replace(call.clone().into()); - let audio_source_config = self.audio_source_config.read().await; + let own_id = self + .own_id + .read() + .clone() + .ok_or(Error::BlinkNotInitialized)?; + + self.event_handler.set_active_call(call.clone()); + let webrtc_codec = AudioCodec::default(); // ensure there is an audio source track let rtc_rtp_codec: RTCRtpCodecCapability = RTCRtpCodecCapability { @@ -239,9 +189,7 @@ impl BlinkImpl { ..Default::default() }; let track = self - .webrtc_controller - .write() - .await + .event_handler .add_media_source(host_media::AUDIO_SOURCE_ID.into(), rtc_rtp_codec) .await?; if let Err(e) = host_media::create_audio_source_track( @@ -249,119 +197,23 @@ impl BlinkImpl { self.ui_event_ch.clone(), track, webrtc_codec, - audio_source_config.clone(), ) .await { let _ = self - .webrtc_controller - .write() - .await - .remove_media_source(host_media::AUDIO_SOURCE_ID.into()) - .await; + .event_handler + .remove_media_source(host_media::AUDIO_SOURCE_ID.into()); return Err(e); } - // next, create event streams and pass them to a task - let call_signaling_stream = ipfs - .pubsub_subscribe(ipfs_routes::call_signal_route(&call.call_id())) - .await - .context("failed to subscribe to call_broadcast_route")?; - - let peer_signaling_stream = ipfs - .pubsub_subscribe(ipfs_routes::peer_signal_route(own_id, &call.call_id())) - .await - .context("failed to subscribe to call_signaling_route")?; - - let webrtc_event_stream = WebRtcEventStream(Box::pin( - self.webrtc_controller - .read() - .await - .get_event_stream() - .context("failed to get webrtc event stream")?, - )); - - let webrtc_handler = std::mem::take(&mut *self.webrtc_handler.write()); - if let Some(handle) = webrtc_handler { - // just to be safe - handle.abort(); - } + self.gossipsub_listener + .subscribe_call(call.call_id(), call.group_key()); + self.gossipsub_listener + .connect_webrtc(call.call_id(), own_id); - let own_id = self.own_id.clone(); - let active_call = self.active_call.clone(); - let webrtc_controller = self.webrtc_controller.clone(); - let audio_sink_config = self.audio_sink_config.clone(); - let ui_event_ch = self.ui_event_ch.clone(); - let event_ch2 = ui_event_ch.clone(); - - let webrtc_handle = tokio::task::spawn(async move { - handle_webrtc( - WebRtcHandlerParams { - own_id, - event_ch: event_ch2, - ipfs, - active_call, - webrtc_controller, - audio_sink_config, - ch: ui_event_ch, - call_signaling_stream, - peer_signaling_stream, - }, - webrtc_event_stream, - ) - .await; - }); - - self.webrtc_handler.write().replace(webrtc_handle); - Ok(()) - } - - async fn update_audio_source_config( - &mut self, - input_device: &cpal::Device, - ) -> anyhow::Result<()> { - let min_channels = Self::get_min_source_channels(input_device)?; - self.audio_source_config.write().await.channels = min_channels; Ok(()) } - fn get_min_source_channels(input_device: &cpal::Device) -> anyhow::Result { - let min_channels = - input_device - .supported_input_configs()? - .fold(None, |acc: Option, x| match acc { - None => Some(x.channels()), - Some(y) => Some(std::cmp::min(x.channels(), y)), - }); - let channels = min_channels.ok_or(anyhow::anyhow!( - "unsupported audio input device - no input configuration available" - ))?; - Ok(channels) - } - - async fn update_audio_sink_config( - &mut self, - output_device: &cpal::Device, - ) -> anyhow::Result<()> { - let min_channels = Self::get_min_sink_channels(output_device)?; - self.audio_sink_config.write().await.channels = min_channels; - Ok(()) - } - - fn get_min_sink_channels(output_device: &cpal::Device) -> anyhow::Result { - let min_channels = - output_device - .supported_output_configs()? - .fold(None, |acc: Option, x| match acc { - None => Some(x.channels()), - Some(y) => Some(std::cmp::min(x.channels(), y)), - }); - let channels = min_channels.ok_or(anyhow::anyhow!( - "unsupported audio output device. no output configuration available" - ))?; - Ok(channels) - } - async fn select_microphone(&mut self, device_name: &str) -> Result<(), Error> { let host = cpal::default_host(); let device: cpal::Device = if device_name.to_ascii_lowercase().eq("default") { @@ -375,9 +227,7 @@ impl BlinkImpl { r.ok_or(Error::AudioDeviceNotFound)? }; - self.update_audio_source_config(&device).await?; - host_media::change_audio_input(device, self.audio_source_config.read().await.clone()) - .await?; + host_media::change_audio_input(device).await?; Ok(()) } @@ -394,20 +244,12 @@ impl BlinkImpl { r.ok_or(Error::AudioDeviceNotFound)? }; - self.update_audio_sink_config(&device).await?; - host_media::change_audio_output(device, self.audio_sink_config.read().await.clone()) - .await?; + host_media::change_audio_output(device).await?; Ok(()) } } impl BlinkImpl { - async fn get_ipfs(&self) -> Result { - let lock = self.ipfs.read().await; - let ipfs = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - Ok(ipfs.clone()) - } - async fn ensure_call_not_in_progress(&self) -> Result<(), Error> { if let Some(ac) = self.active_call.read().await.as_ref() { if ac.call_state != CallState::Closed { @@ -472,33 +314,28 @@ impl Blink for BlinkImpl { mut participants: Vec, ) -> Result { self.ensure_call_not_in_progress().await?; - let ipfs = self.get_ipfs().await?; + let own_id = self + .own_id + .read() + .clone() + .ok_or(Error::BlinkNotInitialized)?; - // need to drop lock to self.own_id before calling self.init_call - { - let lock = self.own_id.read().await; - let own_id = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - if !participants.contains(own_id) { - participants.push(DID::from_str(&own_id.fingerprint())?); - }; - } + if !participants.contains(&own_id) { + participants.push(DID::from_str(&own_id.fingerprint())?); + }; let call_info = CallInfo::new(conversation_id, participants.clone()); self.init_call(call_info.clone()).await?; - let lock = self.own_id.read().await; - let own_id = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - + // todo: periodically re-send offer signal + participants.retain(|x| x != &own_id); for dest in participants { - if dest == *own_id { - continue; - } let topic = ipfs_routes::call_initiation_route(&dest); let signal = InitiationSignal::Offer { call_info: call_info.clone(), }; - if let Err(e) = send_signal_ecdh(&ipfs, own_id, &dest, signal, topic).await { + if let Err(e) = self.gossipsub_sender.send_signal_ecdh(dest, signal, topic) { log::error!("failed to send signal: {e}"); } } @@ -517,22 +354,30 @@ impl Blink for BlinkImpl { }; self.init_call(call.clone()).await?; + + // todo: periodically re-send join signals let call_id = call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Join { call_id }; - - let ipfs = self.get_ipfs().await?; - send_signal_aes(&ipfs, &call.group_key(), signal, topic) - .await - .map_err(|e| Error::FailedToSendSignal(e.to_string())) + let signal = CallSignal::Join; + if let Err(e) = self + .gossipsub_sender + .send_signal_aes(call.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + Err(Error::OtherWithContext("could not answer call".into())) + } else { + Ok(()) + } } /// use the Leave signal as a courtesy, to let the group know not to expect you to join. async fn reject_call(&mut self, call_id: Uuid) -> Result<(), Error> { - let ipfs = self.get_ipfs().await?; if let Some(pc) = self.pending_calls.write().await.remove(&call_id) { let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Leave { call_id }; - if let Err(e) = send_signal_aes(&ipfs, &pc.call.group_key(), signal, topic).await { + let signal = CallSignal::Leave; + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(pc.call.group_key(), signal, topic) + { log::error!("failed to send signal: {e}"); } Ok(()) @@ -544,9 +389,12 @@ impl Blink for BlinkImpl { } /// end/leave the current call async fn leave_call(&mut self) -> Result<(), Error> { - let lock = self.own_id.read().await; - let own_id = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - let ipfs = self.get_ipfs().await?; + let own_id = self + .own_id + .read() + .clone() + .ok_or(Error::BlinkNotInitialized)?; + if let Some(ac) = self.active_call.write().await.as_mut() { match ac.call_state.clone() { CallState::Started => { @@ -568,33 +416,15 @@ impl Blink for BlinkImpl { let call_id = ac.call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Leave { call_id }; - if let Err(e) = send_signal_aes(&ipfs, &ac.call.group_key(), signal, topic).await { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to leave call"); - } - - // send extra quit signal - for participant in ac - .call - .participants() - .iter() - .filter(|x| !ac.connected_participants.contains_key(x)) + let signal = CallSignal::Leave; + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(ac.call.group_key(), signal, topic) { - if participant == own_id { - continue; - } - let topic = ipfs_routes::call_initiation_route(participant); - let signal = InitiationSignal::Leave { - call_id: ac.call.call_id(), - }; - if let Err(e) = send_signal_ecdh(&ipfs, own_id, participant, signal, topic).await { - log::error!("failed to send signal: {e}"); - } + log::error!("failed to send signal: {e}"); } - let r = self.webrtc_controller.write().await.deinit().await; + let r = self.event_handler.leave_call(); host_media::reset().await; //rtp_logger::deinit().await; let _ = r?; @@ -609,19 +439,13 @@ impl Blink for BlinkImpl { // ------ Select input/output devices ------ async fn get_audio_device_config(&self) -> Box { - Box::new(self.audio_device_config.read().await.clone()) + Box::new(host_media::get_audio_device_config().await) } async fn set_audio_device_config( &mut self, config: Box, ) -> Result<(), Error> { - let audio_device_config = host_media::audio::DeviceConfig::new( - config.speaker_device_name(), - config.microphone_device_name(), - ); - *self.audio_device_config.write().await = audio_device_config; - if let Some(device_name) = config.speaker_device_name() { self.select_speaker(&device_name).await?; } @@ -647,15 +471,15 @@ impl Blink for BlinkImpl { } host_media::mute_self().await?; - let lock = self.ipfs.read().await; - let ipfs = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - if let Some(ac) = self.active_call.write().await.as_mut() { ac.call_config.self_muted = true; let call_id = ac.call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Muted; - if let Err(e) = send_signal_aes(ipfs, &ac.call.group_key(), signal, topic).await { + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(ac.call.group_key(), signal, topic) + { log::error!("failed to send signal: {e}"); } else { log::debug!("sent signal to mute self"); @@ -670,15 +494,15 @@ impl Blink for BlinkImpl { } host_media::unmute_self().await?; - let lock = self.ipfs.read().await; - let ipfs = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - if let Some(ac) = self.active_call.write().await.as_mut() { ac.call_config.self_muted = false; let call_id = ac.call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Unmuted; - if let Err(e) = send_signal_aes(ipfs, &ac.call.group_key(), signal, topic).await { + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(ac.call.group_key(), signal, topic) + { log::error!("failed to send signal: {e}"); } else { log::debug!("sent signal to unmute self"); @@ -692,15 +516,16 @@ impl Blink for BlinkImpl { return Err(Error::CallNotInProgress); } host_media::deafen().await?; - let lock = self.ipfs.read().await; - let ipfs = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; if let Some(ac) = self.active_call.write().await.as_mut() { ac.call_config.self_deafened = true; let call_id = ac.call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Deafened; - if let Err(e) = send_signal_aes(ipfs, &ac.call.group_key(), signal, topic).await { + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(ac.call.group_key(), signal, topic) + { log::error!("failed to send signal: {e}"); } else { log::debug!("sent signal to deafen self"); @@ -714,15 +539,15 @@ impl Blink for BlinkImpl { return Err(Error::CallNotInProgress); } host_media::undeafen().await?; - let lock = self.ipfs.read().await; - let ipfs = lock.as_ref().ok_or(Error::BlinkNotInitialized)?; - if let Some(ac) = self.active_call.write().await.as_mut() { ac.call_config.self_deafened = false; let call_id = ac.call.call_id(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Undeafened; - if let Err(e) = send_signal_aes(ipfs, &ac.call.group_key(), signal, topic).await { + if let Err(e) = + self.gossipsub_sender + .send_signal_aes(ac.call.group_key(), signal, topic) + { log::error!("failed to send signal: {e}"); } else { log::debug!("sent signal to undeafen self"); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs index 8af337f7d..61dee56ac 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs @@ -6,7 +6,7 @@ use futures::StreamExt; use rust_ipfs::{Ipfs, SubscriptionStream}; use std::sync::Arc; use tokio::sync::{ - broadcast::{self, Sender}, + broadcast::{self}, RwLock, }; @@ -26,7 +26,6 @@ pub struct WebRtcHandlerParams { pub active_call: Arc>>, pub webrtc_controller: Arc>, pub audio_sink_config: Arc>, - pub ch: Sender, pub call_signaling_stream: SubscriptionStream, pub peer_signaling_stream: SubscriptionStream, } @@ -39,7 +38,6 @@ pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEve active_call, webrtc_controller, audio_sink_config: audio_sink_codec, - ch, call_signaling_stream, peer_signaling_stream, } = params; @@ -92,7 +90,7 @@ pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEve log::error!("failed to dial peer: {e}"); continue; } - if let Err(e) = ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: sender }) { log::error!("failed to send ParticipantJoined Event: {e}"); } @@ -112,27 +110,27 @@ pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEve continue; } webrtc_controller.write().await.hang_up(&sender).await; - if let Err(e) = ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { log::error!("failed to send ParticipantLeft event: {e}"); } }, CallSignal::Muted => { - if let Err(e) = ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { log::error!("failed to send ParticipantMuted event: {e}"); } }, CallSignal::Unmuted => { - if let Err(e) = ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { log::error!("failed to send ParticipantUnmuted event: {e}"); } } CallSignal::Deafened => { - if let Err(e) = ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { log::error!("failed to send ParticipantDeafened event: {e}"); } }, CallSignal::Undeafened => { - if let Err(e) = ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { + if let Err(e) = event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { log::error!("failed to send ParticipantUndeafened event: {e}"); } }, @@ -261,7 +259,7 @@ pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEve EmittedEvents::Connected { peer } => { active_call.connected_participants.insert(peer.clone(), PeerState::Connected); let event = BlinkEventKind::ParticipantJoined { call_id, peer_id: peer}; - let _ = ch.send(event); + let _ = event_ch.send(event); } EmittedEvents::ConnectionClosed { peer } => { // sometimes this event triggers without Disconnected being triggered. @@ -283,7 +281,7 @@ pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEve //rtp_logger::deinit().await; host_media::reset().await; let event = BlinkEventKind::CallTerminated { call_id }; - let _ = ch.send(event); + let _ = event_ch.send(event); // terminate the task on purpose. return; } diff --git a/extensions/warp-blink-wrtc/src/host_media/mod.rs b/extensions/warp-blink-wrtc/src/host_media/mod.rs index afc2c7715..e2eb1b521 100644 --- a/extensions/warp-blink-wrtc/src/host_media/mod.rs +++ b/extensions/warp-blink-wrtc/src/host_media/mod.rs @@ -7,7 +7,7 @@ use cpal::traits::{DeviceTrait, HostTrait}; use once_cell::sync::Lazy; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{broadcast, RwLock}; -use warp::blink::BlinkEventKind; +use warp::blink::{AudioDeviceConfig, BlinkEventKind}; use warp::crypto::DID; use warp::error::Error; use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP; @@ -16,13 +16,16 @@ use webrtc::track::track_remote::TrackRemote; pub(crate) mod audio; use audio::{create_sink_track, create_source_track, AudioCodec, AudioHardwareConfig}; -use self::mp4_logger::Mp4LoggerConfig; +use self::audio::DeviceConfig; +use self::{audio::AudioSampleRate, mp4_logger::Mp4LoggerConfig}; pub(crate) mod mp4_logger; struct Data { audio_input_device: Option, audio_output_device: Option, + audio_source_config: AudioHardwareConfig, + audio_sink_config: AudioHardwareConfig, audio_source_track: Option>, audio_sink_tracks: HashMap>, recording: bool, @@ -36,6 +39,14 @@ static mut DATA: Lazy = Lazy::new(|| { Data { audio_input_device: cpal_host.default_input_device(), audio_output_device: cpal_host.default_output_device(), + audio_source_config: AudioHardwareConfig { + sample_rate: AudioSampleRate::High, + channels: 1, + }, + audio_sink_config: AudioHardwareConfig { + sample_rate: AudioSampleRate::High, + channels: 1, + }, audio_source_track: None, audio_sink_tracks: HashMap::new(), recording: false, @@ -85,7 +96,6 @@ pub async fn create_audio_source_track( event_ch: broadcast::Sender, track: Arc, webrtc_codec: AudioCodec, - source_config: AudioHardwareConfig, ) -> Result<(), Error> { let _lock = LOCK.write().await; let input_device = match unsafe { DATA.audio_input_device.as_ref() } { @@ -94,7 +104,7 @@ pub async fn create_audio_source_track( }; let muted = unsafe { DATA.muted }; - + let source_config = unsafe { DATA.audio_source_config.clone() }; let source_track = create_source_track( own_id, event_ch, @@ -140,7 +150,6 @@ pub async fn create_audio_sink_track( track: Arc, // the format to decode to. Opus supports encoding and decoding to arbitrary sample rates and number of channels. webrtc_codec: AudioCodec, - sink_config: AudioHardwareConfig, ) -> anyhow::Result<()> { let _lock = LOCK.write().await; let output_device = match unsafe { DATA.audio_output_device.as_ref() } { @@ -150,6 +159,7 @@ pub async fn create_audio_sink_track( } }; let deafened = unsafe { DATA.deafened }; + let sink_config = unsafe { DATA.audio_sink_config.clone() }; let sink_track = create_sink_track( peer_id.clone(), event_ch, @@ -183,12 +193,12 @@ pub async fn create_audio_sink_track( Ok(()) } -pub async fn change_audio_input( - device: cpal::Device, - source_config: AudioHardwareConfig, -) -> anyhow::Result<()> { +pub async fn change_audio_input(device: cpal::Device) -> anyhow::Result<()> { let _lock = LOCK.write().await; + let mut source_config = unsafe { DATA.audio_source_config.clone() }; + source_config.channels = get_min_source_channels(&device)?; + // change_input_device destroys the audio stream. if that function fails. there should be // no audio_input. unsafe { @@ -196,19 +206,32 @@ pub async fn change_audio_input( } if let Some(source) = unsafe { DATA.audio_source_track.as_mut() } { - source.change_input_device(&device, source_config)?; + source.change_input_device(&device, source_config.clone())?; } unsafe { DATA.audio_input_device.replace(device); + DATA.audio_source_config = source_config; } Ok(()) } -pub async fn change_audio_output( - device: cpal::Device, - sink_config: AudioHardwareConfig, -) -> anyhow::Result<()> { +pub async fn set_audio_source_config(source_config: AudioHardwareConfig) { let _lock = LOCK.write().await; + unsafe { + DATA.audio_source_config = source_config; + } +} + +pub async fn get_audio_source_config() -> AudioHardwareConfig { + let _lock = LOCK.write().await; + unsafe { DATA.audio_source_config.clone() } +} + +pub async fn change_audio_output(device: cpal::Device) -> anyhow::Result<()> { + let _lock = LOCK.write().await; + + let mut sink_config = unsafe { DATA.audio_sink_config.clone() }; + sink_config.channels = get_min_sink_channels(&device)?; // todo: if this fails, return an error or keep going? for (_k, v) in unsafe { DATA.audio_sink_tracks.iter_mut() } { @@ -219,10 +242,37 @@ pub async fn change_audio_output( unsafe { DATA.audio_output_device.replace(device); + DATA.audio_sink_config = sink_config; } Ok(()) } +pub async fn set_audio_sink_config(sink_config: AudioHardwareConfig) { + let _lock = LOCK.write().await; + unsafe { + DATA.audio_sink_config = sink_config; + } +} + +pub async fn get_audio_sink_config() -> AudioHardwareConfig { + let _lock = LOCK.write().await; + unsafe { DATA.audio_sink_config.clone() } +} + +pub async fn get_audio_device_config() -> DeviceConfig { + let _lock = LOCK.write().await; + unsafe { + DeviceConfig::new( + DATA.audio_input_device + .as_ref() + .map(|x| x.name().unwrap_or_default()), + DATA.audio_output_device + .as_ref() + .map(|x| x.name().unwrap_or_default()), + ) + } +} + pub async fn remove_sink_track(peer_id: DID) -> anyhow::Result<()> { let _lock = LOCK.write().await; unsafe { @@ -343,3 +393,30 @@ pub async fn set_peer_audio_gain(peer_id: DID, multiplier: f32) -> anyhow::Resul Ok(()) } + +fn get_min_source_channels(input_device: &cpal::Device) -> anyhow::Result { + let min_channels = input_device + .supported_input_configs()? + .fold(None, |acc: Option, x| match acc { + None => Some(x.channels()), + Some(y) => Some(std::cmp::min(x.channels(), y)), + }); + let channels = min_channels.ok_or(anyhow::anyhow!( + "unsupported audio input device - no input configuration available" + ))?; + Ok(channels) +} + +fn get_min_sink_channels(output_device: &cpal::Device) -> anyhow::Result { + let min_channels = + output_device + .supported_output_configs()? + .fold(None, |acc: Option, x| match acc { + None => Some(x.channels()), + Some(y) => Some(std::cmp::min(x.channels(), y)), + }); + let channels = min_channels.ok_or(anyhow::anyhow!( + "unsupported audio output device. no output configuration available" + ))?; + Ok(channels) +} diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/signaling.rs index 549b46188..085851cc3 100644 --- a/extensions/warp-blink-wrtc/src/signaling.rs +++ b/extensions/warp-blink-wrtc/src/signaling.rs @@ -8,6 +8,7 @@ use webrtc::{ peer_connection::sdp::session_description::RTCSessionDescription, }; +#[derive(Clone)] pub enum GossipSubSignal { Peer { sender: DID, @@ -25,7 +26,7 @@ pub enum GossipSubSignal { }, } -#[derive(Serialize, Deserialize, Display)] +#[derive(Serialize, Deserialize, Display, Clone)] pub enum PeerSignal { #[display(fmt = "Ice")] Ice(RTCIceCandidate), @@ -39,7 +40,7 @@ pub enum PeerSignal { // this is used for webrtc signaling. // it is somewhat redundant but for now i'll leave it in. -#[derive(Serialize, Deserialize, Display)] +#[derive(Serialize, Deserialize, Display, Clone)] pub enum CallSignal { #[display(fmt = "Join")] Join, @@ -55,7 +56,7 @@ pub enum CallSignal { Undeafened, } -#[derive(Serialize, Deserialize, Display)] +#[derive(Serialize, Deserialize, Display, Clone)] pub enum InitiationSignal { /// invite a peer to join a call #[display(fmt = "Offer")] diff --git a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs index e0aa4d762..b1ea8259b 100644 --- a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs +++ b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs @@ -51,7 +51,7 @@ pub mod events; pub use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability; use webrtc::rtp_transceiver::rtp_sender::RTCRtpSender; -use self::events::EmittedEvents; +use self::events::{EmittedEvents, WebRtcEventStream}; /// simple-webrtc /// This library augments the [webrtc-rs](https://github.com/webrtc-rs/webrtc) library, hopefully @@ -141,7 +141,7 @@ impl Controller { bail!("peers is not empty after deinit") } } - pub fn get_event_stream(&self) -> anyhow::Result> { + pub fn get_event_stream(&self) -> WebRtcEventStream { let mut rx = self.event_ch.subscribe(); let stream = async_stream::stream! { loop { @@ -152,7 +152,7 @@ impl Controller { }; } }; - Ok(Box::pin(stream)) + WebRtcEventStream(Box::pin(stream)) } /// creates a RTCPeerConnection, sets the local SDP object, emits a CallInitiatedEvent, diff --git a/warp/src/blink/call_config.rs b/warp/src/blink/call_config.rs index 5b91e1e51..11dfc723a 100644 --- a/warp/src/blink/call_config.rs +++ b/warp/src/blink/call_config.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::HashMap; use crate::crypto::DID; @@ -7,7 +7,7 @@ pub struct CallConfig { pub self_recording: bool, pub self_muted: bool, pub self_deafened: bool, - pub participants_joined: HashSet, + pub participants_joined: HashMap, } #[derive(Default, Debug, Clone)] diff --git a/warp/src/blink/mod.rs b/warp/src/blink/mod.rs index 5d0927c0b..f8a1348b0 100644 --- a/warp/src/blink/mod.rs +++ b/warp/src/blink/mod.rs @@ -185,6 +185,10 @@ impl CallInfo { self.participants.clone() } + pub fn contains_participant(&self, id: &DID) -> bool { + self.participants.contains(id) + } + pub fn group_key(&self) -> Vec { self.group_key.clone() } From c70e77e71b289808da8cf225322756864b0e0d5c Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 13:41:24 -0500 Subject: [PATCH 06/42] saving progress --- .../src/blink_impl/call_initiation.rs | 2 - .../src/blink_impl/data/mod.rs | 149 +++-- .../src/blink_impl/event_handler.rs | 527 ++++++++++++++++-- .../src/blink_impl/gossipsub_listener.rs | 17 +- .../src/blink_impl/gossipsub_sender.rs | 19 +- .../warp-blink-wrtc/src/blink_impl/mod.rs | 325 ++--------- warp/src/blink/call_config.rs | 18 - warp/src/blink/call_state.rs | 66 +++ warp/src/blink/mod.rs | 8 +- warp/src/error.rs | 2 + 10 files changed, 748 insertions(+), 385 deletions(-) delete mode 100644 warp/src/blink/call_config.rs create mode 100644 warp/src/blink/call_state.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs b/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs index 36110dc06..1cd0e7693 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs @@ -13,8 +13,6 @@ use crate::{ store::{decode_gossipsub_msg_ecdh, PeerIdExt}, }; -use super::data::PendingCall; - pub async fn run( own_id: Arc>>, pending_calls: Arc>>, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index 700c816a2..ce599d0f8 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -1,6 +1,7 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +use uuid::Uuid; use warp::{ - blink::{CallConfig, CallInfo}, + blink::{CallInfo, CallState}, crypto::DID, }; @@ -8,43 +9,129 @@ mod notify_wrapper; pub use notify_wrapper::*; #[derive(Clone)] -pub struct ActiveCall { - pub call: CallInfo, - pub connected_participants: HashMap, - pub call_state: CallState, - pub call_config: CallConfig, +pub struct CallData { + pub info: CallInfo, + pub state: CallState, } -#[derive(Clone, Eq, PartialEq)] -pub enum PeerState { - Disconnected, - Initializing, - Connected, - Closed, +impl CallData { + pub fn new(info: CallInfo, state: CallState) -> Self { + Self { info, state } + } + + pub fn get_info(&self) -> CallInfo { + self.info.clone() + } + + pub fn get_state(&self) -> CallState { + self.state.clone() + } } -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum CallState { - // the call was offered but no one joined and there is no peer connection - Uninitialized, - // at least one peer has connected - Started, - Closing, - Closed, + +pub struct CallDataMap { + pub own_id: DID, + pub map: HashMap, } -// used when a call is accepted -impl From for ActiveCall { - fn from(value: CallInfo) -> Self { +impl CallDataMap { + pub fn new(own_id: DID) -> Self { Self { - call: value, - connected_participants: HashMap::new(), - call_state: CallState::Uninitialized, - call_config: CallConfig::default(), + own_id, + map: HashMap::default(), + } + } + pub fn add_call(&mut self, info: CallInfo, sender: &DID) { + let call_id = info.call_id(); + if self.map.contains_key(&call_id) { + log::warn!("tried to add a call for which a key already exists"); + return; + } + + let mut state = CallState::new(self.own_id.clone()); + state.add_participant(sender); + self.map.insert(call_id, CallData::new(info, state)); + } + + pub fn get_pending_calls(&self) -> Vec { + self.map.values().map(|x| x.get_info()).collect() + } +} + +impl CallDataMap { + pub fn add_participant(&mut self, call_id: Uuid, peer_id: &DID) { + if let Some(data) = self.map.get_mut(&call_id) { + if data.info.contains_participant(peer_id) { + data.state.add_participant(peer_id); + } + } + } + + pub fn call_empty(&self, call_id: Uuid) -> bool { + self.map + .get(&call_id) + .map(|data| data.state.participants_joined.is_empty()) + .unwrap_or(true) + } + + pub fn contains_participant(&self, call_id: Uuid, peer_id: &DID) -> bool { + self.map + .get(&call_id) + .map(|data| data.info.contains_participant(peer_id)) + .unwrap_or_default() + } + + pub fn get_call_info(&self, id: Uuid) -> Option { + self.map.get(&id).map(|x| x.get_info()) + } + + pub fn get_call_state(&self, id: Uuid) -> Option { + self.map.get(&id).map(|x| x.get_state()) + } + + pub fn insert(&mut self, id: Uuid, data: CallData) { + self.map.insert(id, data); + } + + pub fn get_call_config(&self, id: Uuid) -> Option { + self.map.get(&id).map(|x| x.get_state()) + } + + pub fn leave_call(&mut self, call_id: Uuid) { + if let Some(data) = self.map.get_mut(&call_id) { + data.state.reset_self(); + } + } + + pub fn participant_in_call(&self, call_id: Uuid, peer_id: &DID) -> bool { + self.map + .get(&call_id) + .map(|x| x.info.contains_participant(peer_id)) + .unwrap_or_default() + } + + pub fn remove_call(&mut self, call_id: Uuid) { + self.map.remove(&call_id); + } + + pub fn remove_participant(&mut self, call_id: Uuid, peer_id: &DID) { + if let Some(data) = self.map.get_mut(&call_id) { + if data.info.contains_participant(peer_id) { + data.state.remove_participant(peer_id); + } } } } -pub struct PendingCall { - pub call: CallInfo, - pub connected_participants: HashSet, +impl CallDataMap { + pub fn set_muted(&mut self, call_id: Uuid, participant: &DID, value: bool) { + if let Some(data) = self.map.get_mut(&call_id) { + data.state.set_muted(participant, value); + } + } + + pub fn set_deafened(&mut self, call_id: Uuid, participant: &DID, value: bool) { + if let Some(data) = self.map.get_mut(&call_id) { + data.state.set_deafened(participant, value); + } + } } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs index 5ad625b34..9543c0aba 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs @@ -12,8 +12,9 @@ use tokio::{ }; use uuid::Uuid; use warp::{ - blink::{BlinkEventKind, CallConfig, CallInfo, ParticipantState}, + blink::{BlinkEventKind, CallInfo, CallState, ParticipantState}, crypto::{cipher::Cipher, DID}, + error::Error, sync::RwLock, }; use webrtc::{ @@ -22,20 +23,30 @@ use webrtc::{ }; use crate::{ - blink_impl::data::{CallState, PeerState}, + blink_impl::data::CallData, host_media::{ self, audio::{AudioCodec, AudioHardwareConfig}, + mp4_logger::Mp4LoggerConfig, }, - signaling::{ipfs_routes, GossipSubSignal, PeerSignal}, + signaling::{ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal}, simple_webrtc::{self, events::WebRtcEventStream, MediaSourceId}, }; -use super::{data::NotifyWrapper, gossipsub_sender::GossipSubSender}; +use super::{ + data::{CallDataMap, NotifyWrapper}, + gossipsub_listener::GossipSubListener, + gossipsub_sender::GossipSubSender, +}; enum EventHandlerCmd { - SetActiveCall { + OfferCall { call_info: CallInfo, + rsp: oneshot::Sender>, + }, + AnswerCall { + call_id: Uuid, + rsp: oneshot::Sender>, }, AddMediaSource { source_id: MediaSourceId, @@ -45,7 +56,33 @@ enum EventHandlerCmd { RemoveMediaSource { source_id: MediaSourceId, }, - LeaveCall, + GetCallInfo { + call_id: Uuid, + rsp: oneshot::Sender>, + }, + LeaveCall { + call_id: Option, + }, + MuteSelf, + UnmuteSelf, + SilenceCall, + UnsilenceCall, + GetPendingCalls { + rsp: oneshot::Sender>, + }, + GetActiveCallInfo { + rsp: oneshot::Sender>, + }, + GetActiveCallState { + rsp: oneshot::Sender>, + }, + RecordCall { + output_dir: String, + rsp: oneshot::Sender>, + }, + StopRecording { + rsp: oneshot::Sender>, + }, } #[derive(Clone)] @@ -59,6 +96,7 @@ impl EventHandler { webrtc_controller: simple_webrtc::Controller, webrtc_event_stream: WebRtcEventStream, gossipsub_sender: GossipSubSender, + gossipsub_listener: GossipSubListener, signal_rx: UnboundedReceiver, event_ch: broadcast::Sender, ) -> Self { @@ -70,6 +108,7 @@ impl EventHandler { webrtc_controller, webrtc_event_stream, gossipsub_sender, + gossipsub_listener, cmd_rx, signal_rx, event_ch, @@ -83,11 +122,22 @@ impl EventHandler { } } - pub fn set_active_call(&self, call_info: CallInfo) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::SetActiveCall { call_info })?; + pub async fn offer_call(&self, call_info: CallInfo) -> anyhow::Result<()> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::OfferCall { call_info, rsp: tx })?; + rx.await??; Ok(()) } + pub async fn answer_call(&self, call_id: Uuid) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::AnswerCall { call_id, rsp: tx }); + rx.await + .map_err(|x| Error::FailedToSendSignal(x.to_string()))? + } + pub async fn add_media_source( &self, source_id: MediaSourceId, @@ -108,25 +158,116 @@ impl EventHandler { Ok(()) } - pub fn leave_call(&self) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::LeaveCall)?; + pub async fn get_call_info(&self, call_id: Uuid) -> Option { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::GetCallInfo { call_id, rsp: tx }) + .ok()?; + rx.await.ok()? + } + + pub fn leave_call(&self, call_id: Option) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::LeaveCall { call_id })?; + Ok(()) + } + + pub fn mute_self(&self) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::MuteSelf)?; Ok(()) } + + pub fn unmute_self(&self) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::UnmuteSelf)?; + Ok(()) + } + pub fn silence_call(&self) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::SilenceCall)?; + Ok(()) + } + pub fn unsilence_call(&self) -> anyhow::Result<()> { + self.ch.send(EventHandlerCmd::UnsilenceCall)?; + Ok(()) + } + + pub async fn get_pending_calls(&self) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::GetPendingCalls { rsp: tx }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; + rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) + } + + pub async fn get_active_call_info(&self) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::GetActiveCallInfo { rsp: tx }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; + rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) + } + + pub async fn get_active_call_state(&self) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::GetActiveCallState { rsp: tx }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; + rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) + } + pub async fn record_call(&self, output_dir: String) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::RecordCall { + output_dir, + rsp: tx, + }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; + rx.await + .map_err(|x| Error::OtherWithContext(x.to_string()))? + } + + pub async fn stop_recording(&self) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.ch + .send(EventHandlerCmd::StopRecording { rsp: tx }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; + rx.await + .map_err(|x| Error::OtherWithContext(x.to_string()))? + } } async fn run( mut webrtc_controller: simple_webrtc::Controller, mut webrtc_event_stream: WebRtcEventStream, gossipsub_sender: GossipSubSender, + gossipsub_listener: GossipSubListener, mut cmd_rx: UnboundedReceiver, mut signal_rx: UnboundedReceiver, - event_ch: broadcast::Sender, + ui_event_ch: broadcast::Sender, notify: Arc, ) { - let mut call_configs: HashMap = HashMap::new(); - let mut call_infos: HashMap = HashMap::new(); + let own_id = { + let notify2 = notify.clone(); + let fut = gossipsub_sender.get_own_id(); + tokio::select! { + _ = notify2.notified() => { + log::debug!("quitting blink event handler"); + return; + } + r = fut => { + match r { + Ok(r) => r, + Err(e) => { + log::debug!("failed to get own id. quitting blink event handler"); + return; + } + } + } + } + }; + // prevent accidental moves + let own_id = &own_id; + + let mut call_data_map = CallDataMap::new(own_id.clone()); let mut active_call: Option = None; - let mut active_call_state = CallState::Uninitialized; loop { tokio::select! { @@ -143,26 +284,270 @@ async fn run( } }; match cmd { - EventHandlerCmd::SetActiveCall { call_info } => { + EventHandlerCmd::OfferCall { call_info, rsp } => { + let prev_active = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&prev_active) { + data.state.reset_self(); + } if active_call.replace(call_info.call_id()).is_some() { webrtc_controller.deinit(); host_media::reset().await; } - call_infos.insert(call_info.call_id(), call_info); + call_data_map.add_call(call_info.clone(), &own_id); + + // automatically add an audio track + let webrtc_codec = AudioCodec::default(); + let rtc_rtp_codec: RTCRtpCodecCapability = RTCRtpCodecCapability { + mime_type: webrtc_codec.mime_type(), + clock_rate: webrtc_codec.sample_rate(), + channels: 1, + ..Default::default() + }; + match webrtc_controller.add_media_source(host_media::AUDIO_SOURCE_ID.into(), rtc_rtp_codec).await { + Ok(track) => { + match host_media::create_audio_source_track( + own_id.clone(), + ui_event_ch.clone(), + track, + webrtc_codec).await + { + Ok(_) => { + gossipsub_listener + .subscribe_call(call_info.call_id(), call_info.group_key()); + gossipsub_listener + .connect_webrtc(call_info.call_id(), own_id.clone()); + + // todo: resend periodically. perhaps somewhere else + let mut participants = call_info.participants(); + participants.retain(|x| x != own_id); + for dest in participants { + let topic = ipfs_routes::call_initiation_route(&dest); + let signal = InitiationSignal::Offer { + call_info: call_info.clone(), + }; + + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + } + rsp.send(Ok(())); + } + Err(e) => { + webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; + rsp.send(Err(e)); + } + } + } + Err(e) => { + rsp.send(Err(Error::OtherWithContext(e.to_string()))); + } + } }, + EventHandlerCmd::AnswerCall { call_id, rsp } => { + let call_info = match call_data_map.get_call_info(call_id) { + Some(r) => r, + None => { + rsp.send(Err(Error::CallNotFound)); + continue; + } + }; + + let prev_active = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&prev_active) { + data.state.reset_self(); + } + if active_call.replace(call_id).is_some() { + webrtc_controller.deinit(); + host_media::reset().await; + } + + // automatically add an audio track + let webrtc_codec = AudioCodec::default(); + let rtc_rtp_codec: RTCRtpCodecCapability = RTCRtpCodecCapability { + mime_type: webrtc_codec.mime_type(), + clock_rate: webrtc_codec.sample_rate(), + channels: 1, + ..Default::default() + }; + match webrtc_controller.add_media_source(host_media::AUDIO_SOURCE_ID.into(), rtc_rtp_codec).await { + Ok(track) => { + let r = host_media::create_audio_source_track( + own_id.clone(), + ui_event_ch.clone(), + track, + webrtc_codec).await; + match r { + Ok(_) => { + gossipsub_listener.subscribe_call(call_id, call_info.group_key()); + gossipsub_listener.connect_webrtc(call_id, own_id.clone()); + let topic = ipfs_routes::call_signal_route(&call_id); + + // todo? periodically re-send join signals. perhaps somewhere else + let signal = CallSignal::Join; + if let Err(e) = + gossipsub_sender + .send_signal_aes(call_info.group_key(), signal, topic) + { + rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); + } else { + rsp.send(Ok(())); + } + } + Err(e) => { + webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; + rsp.send(Err(e)); + } + } + } + Err(e) => { + rsp.send(Err(e.into())); + } + } + } EventHandlerCmd::AddMediaSource { source_id, codec, rsp } => { let r = webrtc_controller.add_media_source(source_id, codec).await; let _ = rsp.send(r); }, + EventHandlerCmd::GetCallInfo { call_id, rsp } => { + let _ = rsp.send(call_data_map.get_call_info(call_id)); + } EventHandlerCmd::RemoveMediaSource { source_id } => { let _ = webrtc_controller.remove_media_source(source_id).await; }, - EventHandlerCmd::LeaveCall => { - if active_call.take().is_some() { + EventHandlerCmd::LeaveCall { call_id } => { + let call_id = call_id.unwrap_or(active_call.unwrap_or_default()); + match call_data_map.get_call_info(call_id) { + Some(info) => { + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Leave; + if let Err(e) = gossipsub_sender + .send_signal_aes(info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } + } + None => { + log::error!("failed to leave call - not found"); + } + } + if matches!(active_call.as_ref(), Some(&call_id)) { + call_data_map.leave_call(call_id); + let _ = active_call.take(); webrtc_controller.deinit(); host_media::reset().await; + if let Err(e) = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id }) { + log::error!("failed to send CallTerminated Event: {e}"); + } } }, + EventHandlerCmd::MuteSelf => { + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + data.state.set_self_muted(true); + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Muted; + if let Err(e) = + gossipsub_sender + .send_signal_aes(data.info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } else { + log::debug!("sent signal to mute self"); + } + } + } + EventHandlerCmd::UnmuteSelf => { + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + data.state.set_self_muted(false); + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Unmuted; + if let Err(e) = + gossipsub_sender + .send_signal_aes(data.info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } else { + log::debug!("sent signal to unmute self"); + } + } + } + EventHandlerCmd::SilenceCall => { + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Err(e) = host_media::deafen().await { + log::error!("{e}"); + } + data.state.set_deafened(own_id, true); + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Deafened; + if let Err(e) = + gossipsub_sender + .send_signal_aes(data.info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } + } + } + EventHandlerCmd::UnsilenceCall => { + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Err(e) = host_media::undeafen().await { + log::error!("{e}"); + } + data.state.set_deafened(own_id, false); + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Undeafened; + if let Err(e) = + gossipsub_sender + .send_signal_aes(data.info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } + } + } + EventHandlerCmd::GetPendingCalls { rsp } => { + let _ = rsp.send(call_data_map.get_pending_calls()); + } + EventHandlerCmd::GetActiveCallState { rsp } => { + if active_call.is_none() { + rsp.send(None); + } else { + rsp.send(call_data_map.get_call_state(active_call.unwrap_or_default())); + } + } + EventHandlerCmd::GetActiveCallInfo { rsp } => { + if active_call.is_none() { + rsp.send(None); + } else { + rsp.send(call_data_map.get_call_info(active_call.unwrap_or_default())); + } + } + EventHandlerCmd::RecordCall { output_dir, rsp } => { + match active_call.and_then(|call_id| call_data_map.get_call_info(call_id)) { + Some(call) => { + let r = host_media::init_recording(Mp4LoggerConfig { + call_id: call.call_id(), + participants: call.participants(), + audio_codec: AudioCodec::default(), + log_path: output_dir.into(), + }) + .await; + let _ = rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); + } + None => { + rsp.send(Err(Error::CallNotInProgress)); + } + } + } + EventHandlerCmd::StopRecording { rsp } => { + if active_call.is_none() { + rsp.send(Err(Error::CallNotInProgress)); + } else { + let r = host_media::pause_recording().await; + rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); + } + } } }, opt = signal_rx.recv() => { @@ -179,8 +564,8 @@ async fn run( log::debug!("received webrtc signal for non-active call"); continue; } - _ if active_call_state != CallState::Started => { - log::debug!("received signal for uninitialized call: {:?}", std::mem::discriminant(&signal)); + _ if !call_data_map.participant_in_call(call_id, &sender) => { + log::debug!("received signal from someone who isn't part of the call"); continue; } crate::signaling::PeerSignal::Ice(ice) => { @@ -203,15 +588,87 @@ async fn run( }, }, GossipSubSignal::Call { sender, call_id, signal } => match signal { - crate::signaling::CallSignal::Join => todo!(), - crate::signaling::CallSignal::Leave => todo!(), - crate::signaling::CallSignal::Muted => todo!(), - crate::signaling::CallSignal::Unmuted => todo!(), - crate::signaling::CallSignal::Deafened => todo!(), - crate::signaling::CallSignal::Undeafened => todo!(), + _ if !call_data_map.participant_in_call(call_id, &sender) => { + log::debug!("received signal from someone who isn't part of the call"); + continue; + } + crate::signaling::CallSignal::Join => { + call_data_map.add_participant(call_id, &sender); + + if matches!(active_call.as_ref(), Some(&call_id)) { + if let Err(e) = webrtc_controller.dial(&sender).await { + log::error!("failed to dial peer: {e}"); + continue; + } + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: sender }) { + log::error!("failed to send ParticipantJoined Event: {e}"); + } + } + }, + crate::signaling::CallSignal::Leave => { + call_data_map.remove_participant(call_id, &sender); + let is_call_empty = call_data_map.call_empty(call_id); + + if matches!(active_call.as_ref(), Some(&call_id)) { + webrtc_controller.hang_up(&sender).await; + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { + log::error!("failed to send ParticipantLeft event: {e}"); + } + } else if is_call_empty { + call_data_map.remove_call(call_id); + if let Err(e) = ui_event_ch.send(BlinkEventKind::CallCancelled { call_id }) { + log::error!("failed to send CallCancelled event: {e}"); + } + } + }, + crate::signaling::CallSignal::Muted => { + call_data_map.set_muted(call_id, &sender, true); + + if matches!(active_call.as_ref(), Some(&call_id)) { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { + log::error!("failed to send ParticipantMuted event: {e}"); + } + } + }, + crate::signaling::CallSignal::Unmuted => { + call_data_map.set_muted(call_id, &sender, false); + + if matches!(active_call.as_ref(), Some(&call_id)) { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { + log::error!("failed to send ParticipantUnmuted event: {e}"); + } + } + }, + crate::signaling::CallSignal::Deafened => { + call_data_map.set_deafened(call_id, &sender, true); + + if matches!(active_call.as_ref(), Some(&call_id)) { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { + log::error!("failed to send ParticipantDeafened event: {e}"); + } + } + }, + crate::signaling::CallSignal::Undeafened => { + call_data_map.set_deafened(call_id, &sender, false); + + if matches!(active_call.as_ref(), Some(&call_id)) { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { + log::error!("failed to send ParticipantUndeafened event: {e}"); + } + } + }, }, GossipSubSignal::Initiation { sender, signal } => match signal { - crate::signaling::InitiationSignal::Offer { call_info } => todo!(), + crate::signaling::InitiationSignal::Offer { call_info } => { + let call_id = call_info.call_id(); + let conversation_id = call_info.conversation_id(); + let participants = call_info.participants(); + call_data_map.add_call(call_info, &sender); + + if let Err(e) = ui_event_ch.send(BlinkEventKind::IncomingCall { call_id, conversation_id, sender, participants }) { + log::error!("failed to send IncomingCall event: {e}"); + } + }, }, } } @@ -235,20 +692,18 @@ async fn run( }, simple_webrtc::events::EmittedEvents::Connected { peer } => { let ac = active_call.unwrap_or_default(); - if call_infos.get(&ac).map(|x| x.contains_participant(&peer)).unwrap_or_default() { - if let Some(config) = call_configs.get_mut(&ac) { - config.participants_joined.insert(peer, ParticipantState::default()); - } + if call_data_map.contains_participant(ac, &peer) { + call_data_map.add_participant(ac, &peer); } else { - log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); + log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); webrtc_controller.hang_up(&peer).await; } }, simple_webrtc::events::EmittedEvents::Disconnected { peer } | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { - if let Some(config) = call_configs.get_mut(&active_call.unwrap_or_default()) { - config.participants_joined.remove(&peer); - } + let ac = active_call.unwrap_or_default(); + call_data_map.remove_participant(ac, &peer); + if let Err(e) = host_media::remove_sink_track(peer.clone()).await { log::error!("failed to send media_track command: {e}"); } @@ -270,7 +725,7 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::TrackAdded { peer, track } => { - if let Err(e) = host_media::create_audio_sink_track(peer.clone(), event_ch.clone(), track, AudioCodec::default()).await { + if let Err(e) = host_media::create_audio_sink_track(peer.clone(), ui_event_ch.clone(), track, AudioCodec::default()).await { log::error!("failed to send media_track command: {e}"); } }, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index c24ca9150..27216d1de 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -44,15 +44,14 @@ pub struct GossipSubListener { impl GossipSubListener { pub fn new( ipfs: Arc>>, - event_ch: UnboundedReceiver, - rsp_ch: UnboundedSender, + signal_tx: UnboundedSender, gossipsub_sender: GossipSubSender, ) -> Self { let (tx, rx) = mpsc::unbounded_channel(); let notify = Arc::new(Notify::new()); let notify2 = notify.clone(); tokio::spawn(async move { - run(ipfs, rx, rsp_ch, gossipsub_sender, notify2).await; + run(ipfs, rx, signal_tx, gossipsub_sender, notify2).await; }); Self { ch: tx, @@ -85,8 +84,8 @@ impl GossipSubListener { async fn run( ipfs: Arc>>, - mut ch: UnboundedReceiver, - tx: UnboundedSender, + mut cmd_rx: UnboundedReceiver, + signal_tx: UnboundedSender, gossipsub_sender: GossipSubSender, notify: Arc, ) { @@ -118,7 +117,7 @@ async fn run( let call_offer_notify = Arc::new(Notify::new()); loop { tokio::select! { - opt = ch.recv() => match opt { + opt = cmd_rx.recv() => match opt { Some(cmd) => match cmd { GossipSubCmd::UnsubscribeCall { call_id } => { if let Some(call) = subscribed_calls.remove(&call_id) { @@ -151,7 +150,7 @@ async fn run( } }; - let ch = tx.clone(); + let ch = signal_tx.clone(); let gossipsub_sender = gossipsub_sender.clone(); tokio::spawn(async move { loop { @@ -209,7 +208,7 @@ async fn run( continue; } }; - let ch = tx.clone(); + let ch = signal_tx.clone(); let notify = webrtc_notify.clone(); let gossipsub_sender = gossipsub_sender.clone(); tokio::spawn(async move { @@ -262,7 +261,7 @@ async fn run( continue; } }; - let ch = tx.clone(); + let ch = signal_tx.clone(); let notify = call_offer_notify.clone(); let gossipsub_sender = gossipsub_sender.clone(); tokio::spawn(async move { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index b20a5b673..dae37cd29 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -35,6 +35,9 @@ enum GossipSubCmd { data: Vec, rsp: oneshot::Sender>>, }, + GetOwnId { + rsp: oneshot::Sender, + }, } #[derive(Clone)] @@ -60,6 +63,13 @@ impl GossipSubSender { } } + pub async fn get_own_id(&self) -> anyhow::Result { + let (tx, rx) = oneshot::channel(); + self.ch.send(GossipSubCmd::GetOwnId { rsp: tx })?; + let id = rx.await?; + Ok(id) + } + pub fn send_signal_aes( &self, group_key: Vec, @@ -71,7 +81,7 @@ impl GossipSubSender { group_key, signal, topic, - }); + })?; Ok(()) } @@ -87,7 +97,7 @@ impl GossipSubSender { dest, signal, topic, - }); + })?; Ok(()) } @@ -113,7 +123,7 @@ impl GossipSubSender { src, data: message, rsp: tx, - }); + })?; let bytes = rx.await??; let data: T = serde_cbor::from_slice(&bytes)?; Ok(data) @@ -163,6 +173,9 @@ async fn run( tokio::select! { opt = ch.recv() => match opt { Some(cmd) => match cmd { + GossipSubCmd::GetOwnId { rsp } => { + rsp.send(own_id.clone()); + } GossipSubCmd::SendAes { group_key, signal, topic } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { Ok(r) => r, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index fce9ddafe..8b6f05805 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -16,12 +16,15 @@ use cpal::traits::{DeviceTrait, HostTrait}; use rust_ipfs::{Ipfs, Keypair}; use std::{any::Any, str::FromStr, sync::Arc, time::Duration}; use tokio::{ - sync::broadcast::{self}, + sync::{ + broadcast::{self}, + mpsc, + }, task::JoinHandle, }; use uuid::Uuid; use warp::{ - blink::{AudioDeviceConfig, Blink, BlinkEventKind, BlinkEventStream, CallConfig, CallInfo}, + blink::{AudioDeviceConfig, Blink, BlinkEventKind, BlinkEventStream, CallInfo, CallState}, crypto::{did_key::Generate, zeroize::Zeroizing, DIDKey, Ed25519KeyPair, Fingerprint, DID}, error::Error, module::Module, @@ -95,11 +98,15 @@ impl BlinkImpl { log::warn!("blink started with no output device"); } + // todo: ensure rx doesn't get dropped + let (ui_event_ch, _rx) = broadcast::channel(1024); + let (signal_tx, signal_rx) = mpsc::unbounded_channel(); + let ipfs = Arc::new(warp::sync::RwLock::new(None)); let own_id_private = Arc::new(warp::sync::RwLock::new(None)); - let gossipsub_sender = GossipSubSender::new(own_id_private, ipfs.clone()); + let gossipsub_sender = GossipSubSender::new(own_id_private.clone(), ipfs.clone()); let gossipsub_listener = - GossipSubListener::new(ipfs.clone(), todo!(), todo!(), gossipsub_sender.clone()); + GossipSubListener::new(ipfs.clone(), signal_tx, gossipsub_sender.clone()); let webrtc_controller = simple_webrtc::Controller::new()?; let webrtc_event_stream = webrtc_controller.get_event_stream(); @@ -107,10 +114,11 @@ impl BlinkImpl { webrtc_controller, webrtc_event_stream, gossipsub_sender.clone(), - todo!(), + gossipsub_listener.clone(), + signal_rx, + ui_event_ch.clone(), ); - let (ui_event_ch, _rx) = broadcast::channel(1024); let blink_impl = Self { own_id: Arc::new(warp::sync::RwLock::new(None)), ui_event_ch, @@ -169,51 +177,6 @@ impl BlinkImpl { Ok(Box::new(blink_impl)) } - async fn init_call(&mut self, call: CallInfo) -> Result<(), Error> { - //rtp_logger::init(call.call_id(), std::path::PathBuf::from("")).await?; - - let own_id = self - .own_id - .read() - .clone() - .ok_or(Error::BlinkNotInitialized)?; - - self.event_handler.set_active_call(call.clone()); - - let webrtc_codec = AudioCodec::default(); - // ensure there is an audio source track - let rtc_rtp_codec: RTCRtpCodecCapability = RTCRtpCodecCapability { - mime_type: webrtc_codec.mime_type(), - clock_rate: webrtc_codec.sample_rate(), - channels: 1, - ..Default::default() - }; - let track = self - .event_handler - .add_media_source(host_media::AUDIO_SOURCE_ID.into(), rtc_rtp_codec) - .await?; - if let Err(e) = host_media::create_audio_source_track( - own_id.clone(), - self.ui_event_ch.clone(), - track, - webrtc_codec, - ) - .await - { - let _ = self - .event_handler - .remove_media_source(host_media::AUDIO_SOURCE_ID.into()); - return Err(e); - } - - self.gossipsub_listener - .subscribe_call(call.call_id(), call.group_key()); - self.gossipsub_listener - .connect_webrtc(call.call_id(), own_id); - - Ok(()) - } - async fn select_microphone(&mut self, device_name: &str) -> Result<(), Error> { let host = cpal::default_host(); let device: cpal::Device = if device_name.to_ascii_lowercase().eq("default") { @@ -249,18 +212,6 @@ impl BlinkImpl { } } -impl BlinkImpl { - async fn ensure_call_not_in_progress(&self) -> Result<(), Error> { - if let Some(ac) = self.active_call.read().await.as_ref() { - if ac.call_state != CallState::Closed { - return Err(Error::OtherWithContext("previous call not finished".into())); - } - } - - Ok(()) - } -} - impl Extension for BlinkImpl { fn id(&self) -> String { "warp-blink-wrtc".to_string() @@ -313,7 +264,6 @@ impl Blink for BlinkImpl { conversation_id: Option, mut participants: Vec, ) -> Result { - self.ensure_call_not_in_progress().await?; let own_id = self .own_id .read() @@ -324,116 +274,25 @@ impl Blink for BlinkImpl { participants.push(DID::from_str(&own_id.fingerprint())?); }; - let call_info = CallInfo::new(conversation_id, participants.clone()); - self.init_call(call_info.clone()).await?; + let call_info = CallInfo::new(conversation_id, participants); + self.event_handler.offer_call(call_info).await?; - // todo: periodically re-send offer signal - participants.retain(|x| x != &own_id); - for dest in participants { - let topic = ipfs_routes::call_initiation_route(&dest); - let signal = InitiationSignal::Offer { - call_info: call_info.clone(), - }; - - if let Err(e) = self.gossipsub_sender.send_signal_ecdh(dest, signal, topic) { - log::error!("failed to send signal: {e}"); - } - } Ok(call_info.call_id()) } /// accept/join a call. Automatically send and receive audio async fn answer_call(&mut self, call_id: Uuid) -> Result<(), Error> { - self.ensure_call_not_in_progress().await?; - let call = match self.pending_calls.write().await.remove(&call_id) { - Some(r) => r.call, - None => { - return Err(Error::OtherWithContext( - "could not answer call: not found".into(), - )) - } - }; - - self.init_call(call.clone()).await?; - + self.event_handler.answer_call(call_id).await // todo: periodically re-send join signals - let call_id = call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Join; - if let Err(e) = self - .gossipsub_sender - .send_signal_aes(call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - Err(Error::OtherWithContext("could not answer call".into())) - } else { - Ok(()) - } } /// use the Leave signal as a courtesy, to let the group know not to expect you to join. async fn reject_call(&mut self, call_id: Uuid) -> Result<(), Error> { - if let Some(pc) = self.pending_calls.write().await.remove(&call_id) { - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Leave; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(pc.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } - Ok(()) - } else { - Err(Error::OtherWithContext( - "could not reject call: not found".into(), - )) - } + self.event_handler.leave_call(Some(call_id)); + Ok(()) } /// end/leave the current call async fn leave_call(&mut self) -> Result<(), Error> { - let own_id = self - .own_id - .read() - .clone() - .ok_or(Error::BlinkNotInitialized)?; - - if let Some(ac) = self.active_call.write().await.as_mut() { - match ac.call_state.clone() { - CallState::Started => { - ac.call_state = CallState::Closing; - } - CallState::Closed => { - log::info!("call already closed"); - return Ok(()); - } - CallState::Uninitialized => { - log::info!("cancelling call"); - ac.call_state = CallState::Closed; - } - CallState::Closing => { - log::warn!("leave_call when call_state is: {:?}", ac.call_state); - return Ok(()); - } - }; - - let call_id = ac.call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Leave; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(ac.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } - - let r = self.event_handler.leave_call(); - host_media::reset().await; - //rtp_logger::deinit().await; - let _ = r?; - Ok(()) - } else { - Err(Error::OtherWithContext( - "tried to leave nonexistent call".into(), - )) - } + self.event_handler.leave_call(None); + Ok(()) } // ------ Select input/output devices ------ @@ -466,104 +325,24 @@ impl Blink for BlinkImpl { // ------ Media controls ------ async fn mute_self(&mut self) -> Result<(), Error> { - if self.active_call.read().await.is_none() { - return Err(Error::CallNotInProgress); - } - host_media::mute_self().await?; - - if let Some(ac) = self.active_call.write().await.as_mut() { - ac.call_config.self_muted = true; - let call_id = ac.call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Muted; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(ac.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to mute self"); - } - } - + self.event_handler.mute_self()?; Ok(()) } async fn unmute_self(&mut self) -> Result<(), Error> { - if self.active_call.read().await.is_none() { - return Err(Error::CallNotInProgress); - } - host_media::unmute_self().await?; - - if let Some(ac) = self.active_call.write().await.as_mut() { - ac.call_config.self_muted = false; - let call_id = ac.call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Unmuted; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(ac.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to unmute self"); - } - } - + self.event_handler.unmute_self()?; Ok(()) } async fn silence_call(&mut self) -> Result<(), Error> { - if self.active_call.read().await.is_none() { - return Err(Error::CallNotInProgress); - } - host_media::deafen().await?; - - if let Some(ac) = self.active_call.write().await.as_mut() { - ac.call_config.self_deafened = true; - let call_id = ac.call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Deafened; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(ac.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to deafen self"); - } - } - + self.event_handler.silence_call()?; Ok(()) } async fn unsilence_call(&mut self) -> Result<(), Error> { - if self.active_call.read().await.is_none() { - return Err(Error::CallNotInProgress); - } - host_media::undeafen().await?; - if let Some(ac) = self.active_call.write().await.as_mut() { - ac.call_config.self_deafened = false; - let call_id = ac.call.call_id(); - let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Undeafened; - if let Err(e) = - self.gossipsub_sender - .send_signal_aes(ac.call.group_key(), signal, topic) - { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to undeafen self"); - } - } - + self.event_handler.unsilence_call()?; Ok(()) } - async fn get_call_config(&self) -> Result, Error> { - Ok(self - .active_call - .read() - .await - .as_ref() - .map(|x| x.call_config.clone())) + async fn get_call_state(&self) -> Result, Error> { + self.event_handler.get_active_call_state().await } async fn enable_camera(&mut self) -> Result<(), Error> { @@ -573,30 +352,10 @@ impl Blink for BlinkImpl { Err(Error::Unimplemented) } async fn record_call(&mut self, output_dir: &str) -> Result<(), Error> { - match self.active_call.read().await.as_ref() { - None => return Err(Error::CallNotInProgress), - Some(ActiveCall { call, .. }) => { - host_media::init_recording(Mp4LoggerConfig { - call_id: call.call_id(), - participants: call.participants(), - audio_codec: AudioCodec::default(), - log_path: output_dir.into(), - }) - .await?; - } - } - - Ok(()) + self.event_handler.record_call(output_dir.into()).await } async fn stop_recording(&mut self) -> Result<(), Error> { - match self.active_call.read().await.as_ref() { - None => return Err(Error::CallNotInProgress), - Some(_) => { - host_media::pause_recording().await?; - } - } - - Ok(()) + self.event_handler.stop_recording().await } fn enable_automute(&mut self) -> Result<(), Error> { @@ -618,20 +377,22 @@ impl Blink for BlinkImpl { // ------ Utility Functions ------ async fn pending_calls(&self) -> Vec { - Vec::from_iter( - self.pending_calls - .read() - .await - .values() - .map(|x| x.call.clone()), - ) + match self.event_handler.get_pending_calls().await { + Ok(r) => r, + Err(e) => { + log::error!("{e}"); + vec![] + } + } } async fn current_call(&self) -> Option { - self.active_call - .read() - .await - .as_ref() - .map(|x| x.call.clone()) + match self.event_handler.get_active_call_info().await { + Ok(r) => r, + Err(e) => { + log::error!("{e}"); + None + } + } } } diff --git a/warp/src/blink/call_config.rs b/warp/src/blink/call_config.rs deleted file mode 100644 index 11dfc723a..000000000 --- a/warp/src/blink/call_config.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::collections::HashMap; - -use crate::crypto::DID; - -#[derive(Default, Debug, Clone)] -pub struct CallConfig { - pub self_recording: bool, - pub self_muted: bool, - pub self_deafened: bool, - pub participants_joined: HashMap, -} - -#[derive(Default, Debug, Clone)] -pub struct ParticipantState { - pub muted: bool, - pub deafened: bool, - pub recording: bool, -} diff --git a/warp/src/blink/call_state.rs b/warp/src/blink/call_state.rs new file mode 100644 index 000000000..f0ef6abbe --- /dev/null +++ b/warp/src/blink/call_state.rs @@ -0,0 +1,66 @@ +use std::collections::HashMap; + +use crate::crypto::DID; + +#[derive(Debug, Clone)] +pub struct CallState { + pub own_id: DID, + pub participants_joined: HashMap, +} + +#[derive(Default, Debug, Clone)] +pub struct ParticipantState { + pub muted: bool, + pub deafened: bool, + pub recording: bool, +} + +impl CallState { + pub fn new(own_id: DID) -> Self { + Self { + own_id, + participants_joined: HashMap::default(), + } + } + pub fn add_participant(&mut self, id: &DID) { + if self.participants_joined.contains_key(id) { + return; + } + self.participants_joined + .insert(id.clone(), ParticipantState::default()); + } + + pub fn is_call_empty(&self) -> bool { + self.participants_joined.is_empty() + } + + pub fn remove_participant(&mut self, id: &DID) { + self.participants_joined.remove(id); + } + + pub fn set_muted(&mut self, id: &DID, muted: bool) { + if let Some(participant) = self.participants_joined.get_mut(id) { + participant.muted = muted; + } + } + + pub fn set_deafened(&mut self, id: &DID, deafened: bool) { + if let Some(participant) = self.participants_joined.get_mut(id) { + participant.deafened = deafened; + } + } + + pub fn set_self_muted(&mut self, muted: bool) { + let own_id = self.own_id.clone(); + self.set_muted(&own_id, muted); + } + + pub fn set_self_deafened(&mut self, deafened: bool) { + let own_id = self.own_id.clone(); + self.set_deafened(&own_id, deafened); + } + + pub fn reset_self(&mut self) { + self.participants_joined.remove(&self.own_id); + } +} diff --git a/warp/src/blink/mod.rs b/warp/src/blink/mod.rs index f8a1348b0..55656eb8c 100644 --- a/warp/src/blink/mod.rs +++ b/warp/src/blink/mod.rs @@ -18,8 +18,8 @@ use mime_types::*; use uuid::Uuid; mod audio_config; pub use audio_config::*; -mod call_config; -pub use call_config::*; +mod call_state; +pub use call_state::*; use crate::{ crypto::DID, @@ -86,7 +86,7 @@ pub trait Blink: Sync + Send + SingleHandle + DynClone { async fn record_call(&mut self, output_dir: &str) -> Result<(), Error>; async fn stop_recording(&mut self) -> Result<(), Error>; - async fn get_call_config(&self) -> Result, Error>; + async fn get_call_state(&self) -> Result, Error>; fn enable_automute(&mut self) -> Result<(), Error>; fn disable_automute(&mut self) -> Result<(), Error>; @@ -152,7 +152,7 @@ pub enum BlinkEventKind { AudioStreamError, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct CallInfo { call_id: Uuid, conversation_id: Option, diff --git a/warp/src/error.rs b/warp/src/error.rs index ad59180ad..3a45a7560 100644 --- a/warp/src/error.rs +++ b/warp/src/error.rs @@ -224,6 +224,8 @@ pub enum Error { AudioHostError(String), #[error("BlinkNotInitialized")] BlinkNotInitialized, + #[error("CallNotFound")] + CallNotFound, #[error("CallNotInProgress")] CallNotInProgress, #[error("FailedToSendSignal: {_0}")] From 0c3716bc50f1cbbb4fa52a274aad713d119748c6 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 13:54:00 -0500 Subject: [PATCH 07/42] fix some clippy --- .../src/blink_impl/call_initiation.rs | 99 ------ .../src/blink_impl/event_handler.rs | 65 ++-- .../src/blink_impl/gossipsub_listener.rs | 6 +- .../src/blink_impl/gossipsub_sender.rs | 12 +- .../warp-blink-wrtc/src/blink_impl/mod.rs | 24 +- .../src/blink_impl/webrtc_handler.rs | 326 ------------------ 6 files changed, 48 insertions(+), 484 deletions(-) delete mode 100644 extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs delete mode 100644 extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs b/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs deleted file mode 100644 index 1cd0e7693..000000000 --- a/extensions/warp-blink-wrtc/src/blink_impl/call_initiation.rs +++ /dev/null @@ -1,99 +0,0 @@ -use futures::StreamExt; -use rust_ipfs::SubscriptionStream; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; -use tokio::sync::{broadcast::Sender, RwLock}; -use uuid::Uuid; -use warp::{blink::BlinkEventKind, crypto::DID, error::Error}; - -use crate::{ - signaling::InitiationSignal, - store::{decode_gossipsub_msg_ecdh, PeerIdExt}, -}; - -pub async fn run( - own_id: Arc>>, - pending_calls: Arc>>, - mut stream: SubscriptionStream, - ch: Sender, -) { - while let Some(msg) = stream.next().await { - let sender = match msg.source.and_then(|s| s.to_did().ok()) { - Some(id) => id, - None => { - log::error!("msg received without source"); - continue; - } - }; - - let signal: InitiationSignal = { - let lock = own_id.read().await; - let own_id = match lock.as_ref().ok_or(Error::BlinkNotInitialized) { - Ok(r) => r, - Err(e) => { - log::error!("{e}"); - continue; - } - }; - - match decode_gossipsub_msg_ecdh(own_id, &sender, &msg) { - Ok(s) => s, - Err(e) => { - log::error!("failed to decode msg from call initiation stream: {e}"); - continue; - } - } - }; - - match signal { - InitiationSignal::Offer { call_info } => { - if !call_info.participants().contains(&sender) { - log::warn!("someone offered a call for which they weren't a participant"); - continue; - } - let call_id = call_info.call_id(); - let evt = BlinkEventKind::IncomingCall { - call_id, - conversation_id: call_info.conversation_id(), - sender: sender.clone(), - participants: call_info.participants(), - }; - - let pc = PendingCall { - call: call_info, - connected_participants: HashSet::from_iter(vec![sender].drain(..)), - }; - pending_calls.write().await.insert(call_id, pc); - if let Err(e) = ch.send(evt) { - log::error!("failed to send IncomingCall event: {e}"); - } - } - InitiationSignal::Join { call_id } => { - if let Some(pc) = pending_calls.write().await.get_mut(&call_id) { - if !pc.call.participants().contains(&sender) { - log::warn!("someone who wasn't a participant tried to cancel the call"); - continue; - } - pc.connected_participants.insert(sender); - } - } - InitiationSignal::Leave { call_id } => { - if let Some(pc) = pending_calls.write().await.get_mut(&call_id) { - if !pc.call.participants().contains(&sender) { - log::warn!("someone who wasn't a participant tried to cancel the call"); - continue; - } - pc.connected_participants.remove(&sender); - if pc.connected_participants.is_empty() { - let evt = BlinkEventKind::CallCancelled { call_id }; - if let Err(e) = ch.send(evt) { - log::error!("failed to send CallCancelled event: {e}"); - } - } - } - } - } - } -} diff --git a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs index 9543c0aba..495e824d4 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs @@ -133,7 +133,8 @@ impl EventHandler { pub async fn answer_call(&self, call_id: Uuid) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::AnswerCall { call_id, rsp: tx }); + .send(EventHandlerCmd::AnswerCall { call_id, rsp: tx }) + .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await .map_err(|x| Error::FailedToSendSignal(x.to_string()))? } @@ -256,7 +257,7 @@ async fn run( match r { Ok(r) => r, Err(e) => { - log::debug!("failed to get own id. quitting blink event handler"); + log::debug!("failed to get own id. quitting blink event handler: {e}"); return; } } @@ -290,7 +291,7 @@ async fn run( data.state.reset_self(); } if active_call.replace(call_info.call_id()).is_some() { - webrtc_controller.deinit(); + let _ = webrtc_controller.deinit().await; host_media::reset().await; } call_data_map.add_call(call_info.clone(), &own_id); @@ -330,16 +331,16 @@ async fn run( log::error!("failed to send signal: {e}"); } } - rsp.send(Ok(())); + let _ = rsp.send(Ok(())); } Err(e) => { - webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; - rsp.send(Err(e)); + let _ = webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; + let _ = rsp.send(Err(e)); } } } Err(e) => { - rsp.send(Err(Error::OtherWithContext(e.to_string()))); + let _ = rsp.send(Err(Error::OtherWithContext(e.to_string()))); } } }, @@ -347,7 +348,7 @@ async fn run( let call_info = match call_data_map.get_call_info(call_id) { Some(r) => r, None => { - rsp.send(Err(Error::CallNotFound)); + let _ = rsp.send(Err(Error::CallNotFound)); continue; } }; @@ -357,7 +358,7 @@ async fn run( data.state.reset_self(); } if active_call.replace(call_id).is_some() { - webrtc_controller.deinit(); + let _ = webrtc_controller.deinit().await; host_media::reset().await; } @@ -388,19 +389,19 @@ async fn run( gossipsub_sender .send_signal_aes(call_info.group_key(), signal, topic) { - rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); + let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { - rsp.send(Ok(())); + let _ = rsp.send(Ok(())); } } Err(e) => { - webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; - rsp.send(Err(e)); + let _ = webrtc_controller.remove_media_source(host_media::AUDIO_SOURCE_ID.into()).await; + let _ = rsp.send(Err(e)); } } } Err(e) => { - rsp.send(Err(e.into())); + let _ = rsp.send(Err(e.into())); } } } @@ -430,10 +431,10 @@ async fn run( log::error!("failed to leave call - not found"); } } - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { call_data_map.leave_call(call_id); let _ = active_call.take(); - webrtc_controller.deinit(); + let _ = webrtc_controller.deinit().await; host_media::reset().await; if let Err(e) = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id }) { log::error!("failed to send CallTerminated Event: {e}"); @@ -511,16 +512,16 @@ async fn run( } EventHandlerCmd::GetActiveCallState { rsp } => { if active_call.is_none() { - rsp.send(None); + let _ = rsp.send(None); } else { - rsp.send(call_data_map.get_call_state(active_call.unwrap_or_default())); + let _ = rsp.send(call_data_map.get_call_state(active_call.unwrap_or_default())); } } EventHandlerCmd::GetActiveCallInfo { rsp } => { if active_call.is_none() { - rsp.send(None); + let _ = rsp.send(None); } else { - rsp.send(call_data_map.get_call_info(active_call.unwrap_or_default())); + let _ = rsp.send(call_data_map.get_call_info(active_call.unwrap_or_default())); } } EventHandlerCmd::RecordCall { output_dir, rsp } => { @@ -536,16 +537,16 @@ async fn run( let _ = rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); } None => { - rsp.send(Err(Error::CallNotInProgress)); + let _ = rsp.send(Err(Error::CallNotInProgress)); } } } EventHandlerCmd::StopRecording { rsp } => { if active_call.is_none() { - rsp.send(Err(Error::CallNotInProgress)); + let _ = rsp.send(Err(Error::CallNotInProgress)); } else { let r = host_media::pause_recording().await; - rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); + let _ = rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); } } } @@ -560,7 +561,7 @@ async fn run( }; match cmd { GossipSubSignal::Peer { sender, call_id, signal } => match signal { - _ if !matches!(active_call.as_ref(), Some(&call_id)) => { + _ if !active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() => { log::debug!("received webrtc signal for non-active call"); continue; } @@ -595,7 +596,7 @@ async fn run( crate::signaling::CallSignal::Join => { call_data_map.add_participant(call_id, &sender); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if let Err(e) = webrtc_controller.dial(&sender).await { log::error!("failed to dial peer: {e}"); continue; @@ -609,7 +610,7 @@ async fn run( call_data_map.remove_participant(call_id, &sender); let is_call_empty = call_data_map.call_empty(call_id); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { webrtc_controller.hang_up(&sender).await; if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { log::error!("failed to send ParticipantLeft event: {e}"); @@ -624,7 +625,7 @@ async fn run( crate::signaling::CallSignal::Muted => { call_data_map.set_muted(call_id, &sender, true); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { log::error!("failed to send ParticipantMuted event: {e}"); } @@ -633,7 +634,7 @@ async fn run( crate::signaling::CallSignal::Unmuted => { call_data_map.set_muted(call_id, &sender, false); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { log::error!("failed to send ParticipantUnmuted event: {e}"); } @@ -642,7 +643,7 @@ async fn run( crate::signaling::CallSignal::Deafened => { call_data_map.set_deafened(call_id, &sender, true); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { log::error!("failed to send ParticipantDeafened event: {e}"); } @@ -651,7 +652,7 @@ async fn run( crate::signaling::CallSignal::Undeafened => { call_data_map.set_deafened(call_id, &sender, false); - if matches!(active_call.as_ref(), Some(&call_id)) { + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { log::error!("failed to send ParticipantUndeafened event: {e}"); } @@ -709,7 +710,9 @@ async fn run( } webrtc_controller.hang_up(&peer).await; }, - simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => todo!(), + simple_webrtc::events::EmittedEvents::ConnectionClosed { peer: _ } => { + // todo + }, simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Sdp(*sdp); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 27216d1de..1e1f5a9f6 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -123,13 +123,13 @@ async fn run( if let Some(call) = subscribed_calls.remove(&call_id) { call.notify_waiters(); } - if matches!(current_call.as_ref(), Some(&call_id)) { + if current_call.as_ref().map(|x| x == &call_id).unwrap_or_default(){ let _ = current_call.take(); webrtc_notify.notify_waiters(); } } GossipSubCmd::DisconnectWebrtc { call_id } => { - if matches!(current_call.as_ref(), Some(&call_id)) { + if current_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { webrtc_notify.notify_waiters(); } } @@ -191,7 +191,7 @@ async fn run( }); }, GossipSubCmd::ConnectWebRtc { call_id, peer } => { - if !matches!(current_call.as_ref(), Some(&call_id)) { + if !current_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if current_call.is_some() { webrtc_notify.notify_waiters(); } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index dae37cd29..a1c883404 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -174,30 +174,30 @@ async fn run( opt = ch.recv() => match opt { Some(cmd) => match cmd { GossipSubCmd::GetOwnId { rsp } => { - rsp.send(own_id.clone()); + let _ = rsp.send(own_id.clone()); } GossipSubCmd::SendAes { group_key, signal, topic } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { Ok(r) => r, Err(e) => { - log::error!("failed to encrypt aes message"); + log::error!("failed to encrypt aes message: {e}"); continue; } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { - log::error!("failed to publish message"); + log::error!("failed to publish message: {e}"); } }, GossipSubCmd::SendEcdh { dest, signal, topic } => { let encrypted = match ecdh_encrypt(&own_id, &dest, signal) { Ok(r) => r, Err(e) => { - log::error!("failed to encrypt ecdh message"); + log::error!("failed to encrypt ecdh message: {e}"); continue; } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { - log::error!("failed to publish message"); + log::error!("failed to publish message: {e}"); } } GossipSubCmd::DecodeEcdh { src, data, rsp } => { @@ -206,7 +206,7 @@ async fn run( Ok(bytes) }; - rsp.send(r()); + let _ = rsp.send(r()); } } None => { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 8b6f05805..042b4dfe5 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -1,11 +1,6 @@ -mod call_initiation; -use call_initiation::run as handle_call_initiation; - mod data; use data::*; -mod webrtc_handler; - mod event_handler; mod gossipsub_listener; mod gossipsub_sender; @@ -56,9 +51,6 @@ pub struct BlinkImpl { own_id: Arc>>, ui_event_ch: broadcast::Sender, - // subscribes to IPFS topic to receive incoming calls - offer_handler: Arc>>, - gossipsub_listener: GossipSubListener, gossipsub_sender: GossipSubSender, event_handler: event_handler::EventHandler, @@ -81,18 +73,14 @@ impl BlinkImpl { pub async fn new(account: Box) -> anyhow::Result> { log::trace!("initializing WebRTC"); - let mut selected_speaker = None; - let mut selected_microphone = None; let cpal_host = cpal::default_host(); if let Some(input_device) = cpal_host.default_input_device() { - selected_microphone = input_device.name().ok(); host_media::change_audio_input(input_device).await?; } else { log::warn!("blink started with no input device"); } if let Some(output_device) = cpal_host.default_output_device() { - selected_speaker = output_device.name().ok(); host_media::change_audio_output(output_device).await?; } else { log::warn!("blink started with no output device"); @@ -122,7 +110,6 @@ impl BlinkImpl { let blink_impl = Self { own_id: Arc::new(warp::sync::RwLock::new(None)), ui_event_ch, - offer_handler: Arc::new(warp::sync::RwLock::new(tokio::spawn(async {}))), gossipsub_sender, gossipsub_listener, event_handler, @@ -130,8 +117,6 @@ impl BlinkImpl { }; let own_id = blink_impl.own_id.clone(); - let offer_handler = blink_impl.offer_handler.clone(); - let ui_event_ch = blink_impl.ui_event_ch.clone(); let gossipsub_listener = blink_impl.gossipsub_listener.clone(); tokio::spawn(async move { @@ -155,7 +140,7 @@ impl BlinkImpl { }; let _own_id = get_keypair_did(_ipfs.keypair()?)?; - let public_did = _own_id.clone(); + let public_did = identity.did_key(); // this one better not be cloned own_id_private.write().replace(_own_id); // this one is for blink and can be cloned. might not even be needed. @@ -275,9 +260,10 @@ impl Blink for BlinkImpl { }; let call_info = CallInfo::new(conversation_id, participants); + let call_id = call_info.call_id(); self.event_handler.offer_call(call_info).await?; - Ok(call_info.call_id()) + Ok(call_id) } /// accept/join a call. Automatically send and receive audio async fn answer_call(&mut self, call_id: Uuid) -> Result<(), Error> { @@ -286,12 +272,12 @@ impl Blink for BlinkImpl { } /// use the Leave signal as a courtesy, to let the group know not to expect you to join. async fn reject_call(&mut self, call_id: Uuid) -> Result<(), Error> { - self.event_handler.leave_call(Some(call_id)); + self.event_handler.leave_call(Some(call_id))?; Ok(()) } /// end/leave the current call async fn leave_call(&mut self) -> Result<(), Error> { - self.event_handler.leave_call(None); + self.event_handler.leave_call(None)?; Ok(()) } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs deleted file mode 100644 index 61dee56ac..000000000 --- a/extensions/warp-blink-wrtc/src/blink_impl/webrtc_handler.rs +++ /dev/null @@ -1,326 +0,0 @@ -use super::data::*; -use crate::host_media; -use crate::host_media::audio::AudioCodec; -use crate::simple_webrtc; -use futures::StreamExt; -use rust_ipfs::{Ipfs, SubscriptionStream}; -use std::sync::Arc; -use tokio::sync::{ - broadcast::{self}, - RwLock, -}; - -use warp::{blink::BlinkEventKind, crypto::DID, error::Error}; - -use crate::{ - host_media::audio::AudioHardwareConfig, - signaling::{ipfs_routes, CallSignal, PeerSignal}, - simple_webrtc::events::{EmittedEvents, WebRtcEventStream}, - store::{decode_gossipsub_msg_aes, decode_gossipsub_msg_ecdh, send_signal_ecdh, PeerIdExt}, -}; - -pub struct WebRtcHandlerParams { - pub own_id: Arc>>, - pub event_ch: broadcast::Sender, - pub ipfs: Ipfs, - pub active_call: Arc>>, - pub webrtc_controller: Arc>, - pub audio_sink_config: Arc>, - pub call_signaling_stream: SubscriptionStream, - pub peer_signaling_stream: SubscriptionStream, -} - -pub async fn run(params: WebRtcHandlerParams, mut webrtc_event_stream: WebRtcEventStream) { - let WebRtcHandlerParams { - own_id, - event_ch, - ipfs, - active_call, - webrtc_controller, - audio_sink_config: audio_sink_codec, - call_signaling_stream, - peer_signaling_stream, - } = params; - futures::pin_mut!(call_signaling_stream); - futures::pin_mut!(peer_signaling_stream); - - loop { - tokio::select! { - opt = call_signaling_stream.next() => { - let msg = match opt { - Some(m) => m, - None => continue - }; - let sender = match msg.source.and_then(|s| s.to_did().ok()) { - Some(id) => id, - None => { - log::error!("msg received without source"); - continue - } - }; - let mut lock = active_call.write().await; - let active_call = match lock.as_mut() { - Some(r) => r, - None => { - log::error!("received call signal without an active call"); - continue; - } - }; - let signal: CallSignal = match decode_gossipsub_msg_aes(&active_call.call.group_key(), &msg) { - Ok(s) => s, - Err(e) => { - log::error!("failed to decode msg from call signaling stream: {e}"); - continue; - }, - }; - match signal { - CallSignal::Join { call_id } => { - log::debug!("received signal: Join"); - match active_call.call_state.clone() { - CallState::Uninitialized => active_call.call_state = CallState::Started, - x => if x != CallState::Started { - log::error!("someone tried to join call with state: {:?}", active_call.call_state); - continue; - } - } - active_call.connected_participants.insert(sender.clone(), PeerState::Initializing); - // todo: properly hang up on error. - // emits CallInitiated Event, which returns the local sdp. will be sent to the peer with the dial signal - if let Err(e) = webrtc_controller.write().await.dial(&sender).await { - log::error!("failed to dial peer: {e}"); - continue; - } - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: sender }) { - log::error!("failed to send ParticipantJoined Event: {e}"); - } - - } - CallSignal::Leave { call_id } => { - log::debug!("received signal: Leave"); - if active_call.call_state == CallState::Closed { - log::error!("participant tried to leave a call which was already closed"); - continue; - } - if active_call.call.call_id() != call_id { - log::error!("participant tried to leave call which wasn't active"); - continue; - } - if !active_call.call.participants().contains(&sender) { - log::error!("participant tried to leave call who wasn't part of the call"); - continue; - } - webrtc_controller.write().await.hang_up(&sender).await; - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { - log::error!("failed to send ParticipantLeft event: {e}"); - } - }, - CallSignal::Muted => { - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { - log::error!("failed to send ParticipantMuted event: {e}"); - } - }, - CallSignal::Unmuted => { - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { - log::error!("failed to send ParticipantUnmuted event: {e}"); - } - } - CallSignal::Deafened => { - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { - log::error!("failed to send ParticipantDeafened event: {e}"); - } - }, - CallSignal::Undeafened => { - if let Err(e) = event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { - log::error!("failed to send ParticipantUndeafened event: {e}"); - } - }, - } - }, - opt = peer_signaling_stream.next() => { - let msg = match opt { - Some(m) => m, - None => continue - }; - let sender = match msg.source.and_then(|s| s.to_did().ok()) { - Some(id) => id, - None => { - log::error!("msg received without source"); - continue - } - }; - - let signal: PeerSignal = { - let lock = own_id.read().await; - let own_id = match lock.as_ref().ok_or(Error::BlinkNotInitialized) { - Ok(r) => r, - Err(e) => { - log::error!("{e}"); - continue; - } - }; - match decode_gossipsub_msg_ecdh(own_id, &sender, &msg) { - Ok(s) => s, - Err(e) => { - log::error!("failed to decode msg from call signaling stream: {e}"); - continue; - }, - } - }; - - let mut lock = active_call.write().await; - let active_call = match lock.as_mut() { - Some(r) => r, - None => { - log::error!("received a peer_signal when there is no active call"); - continue; - } - }; - if matches!(active_call.call_state, CallState::Closing | CallState::Closed) { - log::warn!("received a signal for a call which is being closed"); - continue; - } - if !active_call.call.participants().contains(&sender) { - log::error!("received a signal from a peer who isn't part of the call"); - continue; - } - - let mut webrtc_controller = webrtc_controller.write().await; - - match signal { - PeerSignal::Ice(ice) => { - if active_call.call_state != CallState::Started { - log::error!("ice received for uninitialized call"); - continue; - } - if let Err(e) = webrtc_controller.recv_ice(&sender, ice).await { - log::error!("failed to recv_ice {}", e); - } - } - PeerSignal::Sdp(sdp) => { - if active_call.call_state != CallState::Started { - log::error!("sdp received for uninitialized call"); - continue; - } - log::debug!("received signal: SDP"); - if let Err(e) = webrtc_controller.recv_sdp(&sender, sdp).await { - log::error!("failed to recv_sdp: {}", e); - } - } - PeerSignal::Dial(sdp) => { - if active_call.call_state == CallState::Uninitialized { - active_call.call_state = CallState::Started; - } - log::debug!("received signal: Dial"); - // emits the SDP Event, which is sent to the peer via the SDP signal - if let Err(e) = webrtc_controller.accept_call(&sender, sdp).await { - log::error!("failed to accept_call: {}", e); - } - } - } - }, - opt = webrtc_event_stream.next() => { - match opt { - Some(event) => { - if let EmittedEvents::Ice{ .. } = event { - // don't log this event. it is too noisy. - // would use matches! but the enum's fields don't implement PartialEq - } else { - log::debug!("webrtc event: {event}"); - } - let lock = own_id.read().await; - let own_id = match lock.as_ref().ok_or(Error::BlinkNotInitialized) { - Ok(r) => r, - Err(e) => { - log::error!("{e}"); - continue; - } - }; - let mut lock = active_call.write().await; - let active_call = match lock.as_mut() { - Some(ac) => ac, - None => { - log::error!("event emitted but no active call"); - continue; - } - }; - let mut webrtc_controller = webrtc_controller.write().await; - let call_id = active_call.call.call_id(); - match event { - EmittedEvents::TrackAdded { peer, track } => { - if peer == *own_id { - log::warn!("got TrackAdded event for own id"); - continue; - } - let audio_sink_codec = audio_sink_codec.read().await.clone(); - if let Err(e) = host_media::create_audio_sink_track(peer.clone(), event_ch.clone(), track, AudioCodec::default(), audio_sink_codec).await { - log::error!("failed to send media_track command: {e}"); - } - } - EmittedEvents::Connected { peer } => { - active_call.connected_participants.insert(peer.clone(), PeerState::Connected); - let event = BlinkEventKind::ParticipantJoined { call_id, peer_id: peer}; - let _ = event_ch.send(event); - } - EmittedEvents::ConnectionClosed { peer } => { - // sometimes this event triggers without Disconnected being triggered. - // need to hang_up here as well. - active_call.connected_participants.insert(peer.clone(), PeerState::Closed); - let all_closed = !active_call.connected_participants.iter().any(|(_k, v)| *v != PeerState::Closed); - if all_closed { - active_call.call_state = CallState::Closed; - } - // have to use data after active_call or there will be 2 mutable borrows, which isn't allowed - webrtc_controller.hang_up(&peer).await; - // only autoclose for 2-person calls (group or direct). - // library user should respond to CallTerminated event. - if all_closed && active_call.call.participants().len() == 2 { - log::info!("all participants have successfully been disconnected"); - if let Err(e) = webrtc_controller.deinit().await { - log::error!("webrtc deinit failed: {e}"); - } - //rtp_logger::deinit().await; - host_media::reset().await; - let event = BlinkEventKind::CallTerminated { call_id }; - let _ = event_ch.send(event); - // terminate the task on purpose. - return; - } - } - EmittedEvents::Disconnected { peer } - | EmittedEvents::ConnectionFailed { peer } => { - // todo: could need to retry - active_call.connected_participants.insert(peer.clone(), PeerState::Disconnected); - if let Err(e) = host_media::remove_sink_track(peer.clone()).await { - log::error!("failed to send media_track command: {e}"); - } - webrtc_controller.hang_up(&peer).await; - } - EmittedEvents::CallInitiated { dest, sdp } => { - let topic = ipfs_routes::peer_signal_route(&dest, &call_id); - let signal = PeerSignal::Dial(*sdp); - if let Err(e) = send_signal_ecdh(&ipfs, own_id, &dest, signal, topic).await { - log::error!("failed to send signal: {e}"); - } - } - EmittedEvents::Sdp { dest, sdp } => { - let topic = ipfs_routes::peer_signal_route(&dest, &call_id); - let signal = PeerSignal::Sdp(*sdp); - if let Err(e) = send_signal_ecdh(&ipfs, own_id, &dest, signal, topic).await { - log::error!("failed to send signal: {e}"); - } - } - EmittedEvents::Ice { dest, candidate } => { - let topic = ipfs_routes::peer_signal_route(&dest, &call_id); - let signal = PeerSignal::Ice(*candidate); - if let Err(e) = send_signal_ecdh(&ipfs, own_id, &dest, signal, topic).await { - log::error!("failed to send signal: {e}"); - } - } - } - } - None => todo!() - } - } - } - } -} From 87ed70fe4c58a480483e925c1ff8c52e7cdda768 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 13:55:24 -0500 Subject: [PATCH 08/42] fix clippy --- .../src/blink_impl/event_handler.rs | 14 +++++--------- .../src/blink_impl/gossipsub_listener.rs | 4 ++-- extensions/warp-blink-wrtc/src/blink_impl/mod.rs | 8 ++------ extensions/warp-blink-wrtc/src/host_media/mod.rs | 2 +- .../warp-blink-wrtc/src/simple_webrtc/mod.rs | 2 +- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs index 495e824d4..84650217a 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs @@ -1,21 +1,18 @@ use futures::channel::oneshot; use futures::StreamExt; -use serde::{de::DeserializeOwned, Serialize}; -use std::{collections::HashMap, fmt::Display, sync::Arc, time::Duration}; + +use std::{sync::Arc}; use tokio::{ sync::{ broadcast, mpsc::{self, UnboundedReceiver, UnboundedSender}, Notify, }, - time::Instant, }; use uuid::Uuid; use warp::{ - blink::{BlinkEventKind, CallInfo, CallState, ParticipantState}, - crypto::{cipher::Cipher, DID}, + blink::{BlinkEventKind, CallInfo, CallState}, error::Error, - sync::RwLock, }; use webrtc::{ rtp_transceiver::rtp_codec::RTCRtpCodecCapability, @@ -23,10 +20,9 @@ use webrtc::{ }; use crate::{ - blink_impl::data::CallData, host_media::{ self, - audio::{AudioCodec, AudioHardwareConfig}, + audio::{AudioCodec}, mp4_logger::Mp4LoggerConfig, }, signaling::{ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal}, @@ -294,7 +290,7 @@ async fn run( let _ = webrtc_controller.deinit().await; host_media::reset().await; } - call_data_map.add_call(call_info.clone(), &own_id); + call_data_map.add_call(call_info.clone(), own_id); // automatically add an audio track let webrtc_codec = AudioCodec::default(); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 1e1f5a9f6..987f109aa 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -172,7 +172,7 @@ async fn run( Ok(msg) => { let _ = ch.send(GossipSubSignal::Call{ sender, - call_id: call_id.clone(), + call_id, signal: msg }); }, @@ -231,7 +231,7 @@ async fn run( Ok(msg) => { let _ = ch.send(GossipSubSignal::Peer { sender, - call_id: call_id.clone(), + call_id, signal: msg }); }, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 042b4dfe5..3ea49a03a 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -1,5 +1,5 @@ mod data; -use data::*; + mod event_handler; mod gossipsub_listener; @@ -15,7 +15,6 @@ use tokio::{ broadcast::{self}, mpsc, }, - task::JoinHandle, }; use uuid::Uuid; use warp::{ @@ -26,18 +25,15 @@ use warp::{ multipass::MultiPass, Extension, SingleHandle, }; -use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability; + use crate::{ host_media::{ self, audio::{ automute::{AutoMuteCmd, AUDIO_CMD_CH}, - AudioCodec, }, - mp4_logger::Mp4LoggerConfig, }, - signaling::{ipfs_routes, CallSignal, InitiationSignal}, simple_webrtc::{self}, }; diff --git a/extensions/warp-blink-wrtc/src/host_media/mod.rs b/extensions/warp-blink-wrtc/src/host_media/mod.rs index e2eb1b521..d5417a57f 100644 --- a/extensions/warp-blink-wrtc/src/host_media/mod.rs +++ b/extensions/warp-blink-wrtc/src/host_media/mod.rs @@ -7,7 +7,7 @@ use cpal::traits::{DeviceTrait, HostTrait}; use once_cell::sync::Lazy; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{broadcast, RwLock}; -use warp::blink::{AudioDeviceConfig, BlinkEventKind}; +use warp::blink::{BlinkEventKind}; use warp::crypto::DID; use warp::error::Error; use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP; diff --git a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs index b1ea8259b..e0f8633af 100644 --- a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs +++ b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs @@ -17,7 +17,7 @@ //! use anyhow::{bail, Result}; -use futures::Stream; + use std::collections::HashMap; use std::sync::Arc; use tokio::sync::broadcast; From 99d34f97383b557d7075c42b782654dde128dd63 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 14:15:00 -0500 Subject: [PATCH 09/42] rename event_handler to blink_controller --- .../{event_handler.rs => blink_controller.rs} | 94 +++++++++---------- .../src/blink_impl/gossipsub_listener.rs | 2 +- .../warp-blink-wrtc/src/blink_impl/mod.rs | 48 +++++----- extensions/warp-blink-wrtc/src/signaling.rs | 3 +- 4 files changed, 65 insertions(+), 82 deletions(-) rename extensions/warp-blink-wrtc/src/blink_impl/{event_handler.rs => blink_controller.rs} (93%) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs similarity index 93% rename from extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs rename to extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 84650217a..3f78ad56f 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/event_handler.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -1,13 +1,11 @@ use futures::channel::oneshot; use futures::StreamExt; -use std::{sync::Arc}; -use tokio::{ - sync::{ - broadcast, - mpsc::{self, UnboundedReceiver, UnboundedSender}, - Notify, - }, +use std::sync::Arc; +use tokio::sync::{ + broadcast, + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, }; use uuid::Uuid; use warp::{ @@ -20,11 +18,7 @@ use webrtc::{ }; use crate::{ - host_media::{ - self, - audio::{AudioCodec}, - mp4_logger::Mp4LoggerConfig, - }, + host_media::{self, audio::AudioCodec, mp4_logger::Mp4LoggerConfig}, signaling::{ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal}, simple_webrtc::{self, events::WebRtcEventStream, MediaSourceId}, }; @@ -35,7 +29,7 @@ use super::{ gossipsub_sender::GossipSubSender, }; -enum EventHandlerCmd { +enum Cmd { OfferCall { call_info: CallInfo, rsp: oneshot::Sender>, @@ -82,12 +76,12 @@ enum EventHandlerCmd { } #[derive(Clone)] -pub struct EventHandler { - ch: UnboundedSender, +pub struct BlinkController { + ch: UnboundedSender, notify: Arc, } -impl EventHandler { +impl BlinkController { pub fn new( webrtc_controller: simple_webrtc::Controller, webrtc_event_stream: WebRtcEventStream, @@ -120,8 +114,7 @@ impl EventHandler { pub async fn offer_call(&self, call_info: CallInfo) -> anyhow::Result<()> { let (tx, rx) = oneshot::channel(); - self.ch - .send(EventHandlerCmd::OfferCall { call_info, rsp: tx })?; + self.ch.send(Cmd::OfferCall { call_info, rsp: tx })?; rx.await??; Ok(()) } @@ -129,7 +122,7 @@ impl EventHandler { pub async fn answer_call(&self, call_id: Uuid) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::AnswerCall { call_id, rsp: tx }) + .send(Cmd::AnswerCall { call_id, rsp: tx }) .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await .map_err(|x| Error::FailedToSendSignal(x.to_string()))? @@ -141,7 +134,7 @@ impl EventHandler { codec: RTCRtpCodecCapability, ) -> anyhow::Result> { let (tx, rx) = oneshot::channel(); - self.ch.send(EventHandlerCmd::AddMediaSource { + self.ch.send(Cmd::AddMediaSource { source_id, codec, rsp: tx, @@ -150,46 +143,43 @@ impl EventHandler { } pub fn remove_media_source(&self, source_id: MediaSourceId) -> anyhow::Result<()> { - self.ch - .send(EventHandlerCmd::RemoveMediaSource { source_id })?; + self.ch.send(Cmd::RemoveMediaSource { source_id })?; Ok(()) } pub async fn get_call_info(&self, call_id: Uuid) -> Option { let (tx, rx) = oneshot::channel(); - self.ch - .send(EventHandlerCmd::GetCallInfo { call_id, rsp: tx }) - .ok()?; + self.ch.send(Cmd::GetCallInfo { call_id, rsp: tx }).ok()?; rx.await.ok()? } pub fn leave_call(&self, call_id: Option) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::LeaveCall { call_id })?; + self.ch.send(Cmd::LeaveCall { call_id })?; Ok(()) } pub fn mute_self(&self) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::MuteSelf)?; + self.ch.send(Cmd::MuteSelf)?; Ok(()) } pub fn unmute_self(&self) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::UnmuteSelf)?; + self.ch.send(Cmd::UnmuteSelf)?; Ok(()) } pub fn silence_call(&self) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::SilenceCall)?; + self.ch.send(Cmd::SilenceCall)?; Ok(()) } pub fn unsilence_call(&self) -> anyhow::Result<()> { - self.ch.send(EventHandlerCmd::UnsilenceCall)?; + self.ch.send(Cmd::UnsilenceCall)?; Ok(()) } pub async fn get_pending_calls(&self) -> Result, Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::GetPendingCalls { rsp: tx }) + .send(Cmd::GetPendingCalls { rsp: tx }) .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) } @@ -197,7 +187,7 @@ impl EventHandler { pub async fn get_active_call_info(&self) -> Result, Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::GetActiveCallInfo { rsp: tx }) + .send(Cmd::GetActiveCallInfo { rsp: tx }) .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) } @@ -205,14 +195,14 @@ impl EventHandler { pub async fn get_active_call_state(&self) -> Result, Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::GetActiveCallState { rsp: tx }) + .send(Cmd::GetActiveCallState { rsp: tx }) .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await.map_err(|x| Error::OtherWithContext(x.to_string())) } pub async fn record_call(&self, output_dir: String) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::RecordCall { + .send(Cmd::RecordCall { output_dir, rsp: tx, }) @@ -224,7 +214,7 @@ impl EventHandler { pub async fn stop_recording(&self) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.ch - .send(EventHandlerCmd::StopRecording { rsp: tx }) + .send(Cmd::StopRecording { rsp: tx }) .map_err(|x| Error::OtherWithContext(x.to_string()))?; rx.await .map_err(|x| Error::OtherWithContext(x.to_string()))? @@ -236,7 +226,7 @@ async fn run( mut webrtc_event_stream: WebRtcEventStream, gossipsub_sender: GossipSubSender, gossipsub_listener: GossipSubListener, - mut cmd_rx: UnboundedReceiver, + mut cmd_rx: UnboundedReceiver, mut signal_rx: UnboundedReceiver, ui_event_ch: broadcast::Sender, notify: Arc, @@ -281,7 +271,7 @@ async fn run( } }; match cmd { - EventHandlerCmd::OfferCall { call_info, rsp } => { + Cmd::OfferCall { call_info, rsp } => { let prev_active = active_call.unwrap_or_default(); if let Some(data) = call_data_map.map.get_mut(&prev_active) { data.state.reset_self(); @@ -340,7 +330,7 @@ async fn run( } } }, - EventHandlerCmd::AnswerCall { call_id, rsp } => { + Cmd::AnswerCall { call_id, rsp } => { let call_info = match call_data_map.get_call_info(call_id) { Some(r) => r, None => { @@ -401,17 +391,17 @@ async fn run( } } } - EventHandlerCmd::AddMediaSource { source_id, codec, rsp } => { + Cmd::AddMediaSource { source_id, codec, rsp } => { let r = webrtc_controller.add_media_source(source_id, codec).await; let _ = rsp.send(r); }, - EventHandlerCmd::GetCallInfo { call_id, rsp } => { + Cmd::GetCallInfo { call_id, rsp } => { let _ = rsp.send(call_data_map.get_call_info(call_id)); } - EventHandlerCmd::RemoveMediaSource { source_id } => { + Cmd::RemoveMediaSource { source_id } => { let _ = webrtc_controller.remove_media_source(source_id).await; }, - EventHandlerCmd::LeaveCall { call_id } => { + Cmd::LeaveCall { call_id } => { let call_id = call_id.unwrap_or(active_call.unwrap_or_default()); match call_data_map.get_call_info(call_id) { Some(info) => { @@ -437,7 +427,7 @@ async fn run( } } }, - EventHandlerCmd::MuteSelf => { + Cmd::MuteSelf => { let call_id = active_call.unwrap_or_default(); if let Some(data) = call_data_map.map.get_mut(&call_id) { data.state.set_self_muted(true); @@ -453,7 +443,7 @@ async fn run( } } } - EventHandlerCmd::UnmuteSelf => { + Cmd::UnmuteSelf => { let call_id = active_call.unwrap_or_default(); if let Some(data) = call_data_map.map.get_mut(&call_id) { data.state.set_self_muted(false); @@ -469,7 +459,7 @@ async fn run( } } } - EventHandlerCmd::SilenceCall => { + Cmd::SilenceCall => { let call_id = active_call.unwrap_or_default(); if let Some(data) = call_data_map.map.get_mut(&call_id) { if let Err(e) = host_media::deafen().await { @@ -486,7 +476,7 @@ async fn run( } } } - EventHandlerCmd::UnsilenceCall => { + Cmd::UnsilenceCall => { let call_id = active_call.unwrap_or_default(); if let Some(data) = call_data_map.map.get_mut(&call_id) { if let Err(e) = host_media::undeafen().await { @@ -503,24 +493,24 @@ async fn run( } } } - EventHandlerCmd::GetPendingCalls { rsp } => { + Cmd::GetPendingCalls { rsp } => { let _ = rsp.send(call_data_map.get_pending_calls()); } - EventHandlerCmd::GetActiveCallState { rsp } => { + Cmd::GetActiveCallState { rsp } => { if active_call.is_none() { let _ = rsp.send(None); } else { let _ = rsp.send(call_data_map.get_call_state(active_call.unwrap_or_default())); } } - EventHandlerCmd::GetActiveCallInfo { rsp } => { + Cmd::GetActiveCallInfo { rsp } => { if active_call.is_none() { let _ = rsp.send(None); } else { let _ = rsp.send(call_data_map.get_call_info(active_call.unwrap_or_default())); } } - EventHandlerCmd::RecordCall { output_dir, rsp } => { + Cmd::RecordCall { output_dir, rsp } => { match active_call.and_then(|call_id| call_data_map.get_call_info(call_id)) { Some(call) => { let r = host_media::init_recording(Mp4LoggerConfig { @@ -537,7 +527,7 @@ async fn run( } } } - EventHandlerCmd::StopRecording { rsp } => { + Cmd::StopRecording { rsp } => { if active_call.is_none() { let _ = rsp.send(Err(Error::CallNotInProgress)); } else { @@ -556,7 +546,7 @@ async fn run( } }; match cmd { - GossipSubSignal::Peer { sender, call_id, signal } => match signal { + GossipSubSignal::Peer { sender, call_id, signal } => match *signal { _ if !active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() => { log::debug!("received webrtc signal for non-active call"); continue; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 987f109aa..147d28890 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -232,7 +232,7 @@ async fn run( let _ = ch.send(GossipSubSignal::Peer { sender, call_id, - signal: msg + signal: Box::new(msg) }); }, Err(e) => { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 3ea49a03a..4fed0aca0 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -1,7 +1,6 @@ mod data; - -mod event_handler; +mod blink_controller; mod gossipsub_listener; mod gossipsub_sender; @@ -10,11 +9,9 @@ use async_trait::async_trait; use cpal::traits::{DeviceTrait, HostTrait}; use rust_ipfs::{Ipfs, Keypair}; use std::{any::Any, str::FromStr, sync::Arc, time::Duration}; -use tokio::{ - sync::{ - broadcast::{self}, - mpsc, - }, +use tokio::sync::{ + broadcast::{self}, + mpsc, }; use uuid::Uuid; use warp::{ @@ -26,13 +23,10 @@ use warp::{ Extension, SingleHandle, }; - use crate::{ host_media::{ self, - audio::{ - automute::{AutoMuteCmd, AUDIO_CMD_CH}, - }, + audio::automute::{AutoMuteCmd, AUDIO_CMD_CH}, }, simple_webrtc::{self}, }; @@ -49,7 +43,7 @@ pub struct BlinkImpl { gossipsub_listener: GossipSubListener, gossipsub_sender: GossipSubSender, - event_handler: event_handler::EventHandler, + blink_controller: blink_controller::BlinkController, drop_handler: Arc, } @@ -94,7 +88,7 @@ impl BlinkImpl { let webrtc_controller = simple_webrtc::Controller::new()?; let webrtc_event_stream = webrtc_controller.get_event_stream(); - let event_handler = event_handler::EventHandler::new( + let blink_controller = blink_controller::BlinkController::new( webrtc_controller, webrtc_event_stream, gossipsub_sender.clone(), @@ -108,7 +102,7 @@ impl BlinkImpl { ui_event_ch, gossipsub_sender, gossipsub_listener, - event_handler, + blink_controller, drop_handler: Arc::new(DropHandler {}), }; @@ -257,23 +251,23 @@ impl Blink for BlinkImpl { let call_info = CallInfo::new(conversation_id, participants); let call_id = call_info.call_id(); - self.event_handler.offer_call(call_info).await?; + self.blink_controller.offer_call(call_info).await?; Ok(call_id) } /// accept/join a call. Automatically send and receive audio async fn answer_call(&mut self, call_id: Uuid) -> Result<(), Error> { - self.event_handler.answer_call(call_id).await + self.blink_controller.answer_call(call_id).await // todo: periodically re-send join signals } /// use the Leave signal as a courtesy, to let the group know not to expect you to join. async fn reject_call(&mut self, call_id: Uuid) -> Result<(), Error> { - self.event_handler.leave_call(Some(call_id))?; + self.blink_controller.leave_call(Some(call_id))?; Ok(()) } /// end/leave the current call async fn leave_call(&mut self) -> Result<(), Error> { - self.event_handler.leave_call(None)?; + self.blink_controller.leave_call(None)?; Ok(()) } @@ -307,24 +301,24 @@ impl Blink for BlinkImpl { // ------ Media controls ------ async fn mute_self(&mut self) -> Result<(), Error> { - self.event_handler.mute_self()?; + self.blink_controller.mute_self()?; Ok(()) } async fn unmute_self(&mut self) -> Result<(), Error> { - self.event_handler.unmute_self()?; + self.blink_controller.unmute_self()?; Ok(()) } async fn silence_call(&mut self) -> Result<(), Error> { - self.event_handler.silence_call()?; + self.blink_controller.silence_call()?; Ok(()) } async fn unsilence_call(&mut self) -> Result<(), Error> { - self.event_handler.unsilence_call()?; + self.blink_controller.unsilence_call()?; Ok(()) } async fn get_call_state(&self) -> Result, Error> { - self.event_handler.get_active_call_state().await + self.blink_controller.get_active_call_state().await } async fn enable_camera(&mut self) -> Result<(), Error> { @@ -334,10 +328,10 @@ impl Blink for BlinkImpl { Err(Error::Unimplemented) } async fn record_call(&mut self, output_dir: &str) -> Result<(), Error> { - self.event_handler.record_call(output_dir.into()).await + self.blink_controller.record_call(output_dir.into()).await } async fn stop_recording(&mut self) -> Result<(), Error> { - self.event_handler.stop_recording().await + self.blink_controller.stop_recording().await } fn enable_automute(&mut self) -> Result<(), Error> { @@ -359,7 +353,7 @@ impl Blink for BlinkImpl { // ------ Utility Functions ------ async fn pending_calls(&self) -> Vec { - match self.event_handler.get_pending_calls().await { + match self.blink_controller.get_pending_calls().await { Ok(r) => r, Err(e) => { log::error!("{e}"); @@ -368,7 +362,7 @@ impl Blink for BlinkImpl { } } async fn current_call(&self) -> Option { - match self.event_handler.get_active_call_info().await { + match self.blink_controller.get_active_call_info().await { Ok(r) => r, Err(e) => { log::error!("{e}"); diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/signaling.rs index 085851cc3..f08aaa3aa 100644 --- a/extensions/warp-blink-wrtc/src/signaling.rs +++ b/extensions/warp-blink-wrtc/src/signaling.rs @@ -7,13 +7,12 @@ use webrtc::{ ice_transport::ice_candidate::RTCIceCandidate, peer_connection::sdp::session_description::RTCSessionDescription, }; - #[derive(Clone)] pub enum GossipSubSignal { Peer { sender: DID, call_id: Uuid, - signal: PeerSignal, + signal: Box, }, Call { sender: DID, From 7382c2d8ba44dfc2560b1b27dde8e057bee26d05 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 14:18:28 -0500 Subject: [PATCH 10/42] move some stuff into blink_impl --- .../src/blink_impl/blink_controller.rs | 25 +++++++++++-------- .../src/blink_impl/gossipsub_listener.rs | 2 +- .../src/blink_impl/gossipsub_sender.rs | 2 +- .../warp-blink-wrtc/src/blink_impl/mod.rs | 2 ++ .../src/{ => blink_impl}/signaling.rs | 0 .../src/{ => blink_impl}/store.rs | 0 extensions/warp-blink-wrtc/src/lib.rs | 2 -- 7 files changed, 18 insertions(+), 15 deletions(-) rename extensions/warp-blink-wrtc/src/{ => blink_impl}/signaling.rs (100%) rename extensions/warp-blink-wrtc/src/{ => blink_impl}/store.rs (100%) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 3f78ad56f..73008ea34 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -1,6 +1,10 @@ use futures::channel::oneshot; use futures::StreamExt; +use super::signaling::{ + self, ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal, +}; + use std::sync::Arc; use tokio::sync::{ broadcast, @@ -19,7 +23,6 @@ use webrtc::{ use crate::{ host_media::{self, audio::AudioCodec, mp4_logger::Mp4LoggerConfig}, - signaling::{ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal}, simple_webrtc::{self, events::WebRtcEventStream, MediaSourceId}, }; @@ -555,18 +558,18 @@ async fn run( log::debug!("received signal from someone who isn't part of the call"); continue; } - crate::signaling::PeerSignal::Ice(ice) => { + signaling::PeerSignal::Ice(ice) => { if let Err(e) = webrtc_controller.recv_ice(&sender, ice).await { log::error!("failed to recv_ice {}", e); } }, - crate::signaling::PeerSignal::Sdp(sdp) => { + signaling::PeerSignal::Sdp(sdp) => { log::debug!("received signal: SDP"); if let Err(e) = webrtc_controller.recv_sdp(&sender, sdp).await { log::error!("failed to recv_sdp: {}", e); } }, - crate::signaling::PeerSignal::Dial(sdp) => { + signaling::PeerSignal::Dial(sdp) => { log::debug!("received signal: Dial"); // emits the SDP Event, which is sent to the peer via the SDP signal if let Err(e) = webrtc_controller.accept_call(&sender, sdp).await { @@ -579,7 +582,7 @@ async fn run( log::debug!("received signal from someone who isn't part of the call"); continue; } - crate::signaling::CallSignal::Join => { + signaling::CallSignal::Join => { call_data_map.add_participant(call_id, &sender); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -592,7 +595,7 @@ async fn run( } } }, - crate::signaling::CallSignal::Leave => { + signaling::CallSignal::Leave => { call_data_map.remove_participant(call_id, &sender); let is_call_empty = call_data_map.call_empty(call_id); @@ -608,7 +611,7 @@ async fn run( } } }, - crate::signaling::CallSignal::Muted => { + signaling::CallSignal::Muted => { call_data_map.set_muted(call_id, &sender, true); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -617,7 +620,7 @@ async fn run( } } }, - crate::signaling::CallSignal::Unmuted => { + signaling::CallSignal::Unmuted => { call_data_map.set_muted(call_id, &sender, false); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -626,7 +629,7 @@ async fn run( } } }, - crate::signaling::CallSignal::Deafened => { + signaling::CallSignal::Deafened => { call_data_map.set_deafened(call_id, &sender, true); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -635,7 +638,7 @@ async fn run( } } }, - crate::signaling::CallSignal::Undeafened => { + signaling::CallSignal::Undeafened => { call_data_map.set_deafened(call_id, &sender, false); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -646,7 +649,7 @@ async fn run( }, }, GossipSubSignal::Initiation { sender, signal } => match signal { - crate::signaling::InitiationSignal::Offer { call_info } => { + signaling::InitiationSignal::Offer { call_info } => { let call_id = call_info.call_id(); let conversation_id = call_info.conversation_id(); let participants = call_info.participants(); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 147d28890..0db6edc48 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -12,7 +12,7 @@ use tokio::{ use uuid::Uuid; use warp::{crypto::DID, sync::RwLock}; -use crate::{ +use super::{ signaling::{ ipfs_routes::{call_initiation_route, call_signal_route, peer_signal_route}, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index a1c883404..b54fda7c2 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -15,7 +15,7 @@ use warp::{ sync::RwLock, }; -use crate::store::{ecdh_decrypt, ecdh_encrypt}; +use super::store::{ecdh_decrypt, ecdh_encrypt}; use super::data::NotifyWrapper; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 4fed0aca0..655b99c68 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -3,6 +3,8 @@ mod data; mod blink_controller; mod gossipsub_listener; mod gossipsub_sender; +mod signaling; +mod store; use anyhow::bail; use async_trait::async_trait; diff --git a/extensions/warp-blink-wrtc/src/signaling.rs b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs similarity index 100% rename from extensions/warp-blink-wrtc/src/signaling.rs rename to extensions/warp-blink-wrtc/src/blink_impl/signaling.rs diff --git a/extensions/warp-blink-wrtc/src/store.rs b/extensions/warp-blink-wrtc/src/blink_impl/store.rs similarity index 100% rename from extensions/warp-blink-wrtc/src/store.rs rename to extensions/warp-blink-wrtc/src/blink_impl/store.rs diff --git a/extensions/warp-blink-wrtc/src/lib.rs b/extensions/warp-blink-wrtc/src/lib.rs index 29e4327a5..0d8c8b915 100644 --- a/extensions/warp-blink-wrtc/src/lib.rs +++ b/extensions/warp-blink-wrtc/src/lib.rs @@ -12,8 +12,6 @@ mod host_media; // mod rtp_logger; mod blink_impl; -mod signaling; mod simple_webrtc; -mod store; pub use blink_impl::*; From a3b04e5cf5cf5c24786fafce74ac10541eb9a359 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 14:22:22 -0500 Subject: [PATCH 11/42] fix clippy --- .../src/blink_impl/blink_controller.rs | 29 +++++++++++++------ .../warp-blink-wrtc/src/blink_impl/mod.rs | 11 +++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 73008ea34..5f13be11e 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -84,15 +84,26 @@ pub struct BlinkController { notify: Arc, } +pub struct Args { + pub webrtc_controller: simple_webrtc::Controller, + pub webrtc_event_stream: WebRtcEventStream, + pub gossipsub_sender: GossipSubSender, + pub gossipsub_listener: GossipSubListener, + pub signal_rx: UnboundedReceiver, + pub ui_event_ch: broadcast::Sender, +} + impl BlinkController { - pub fn new( - webrtc_controller: simple_webrtc::Controller, - webrtc_event_stream: WebRtcEventStream, - gossipsub_sender: GossipSubSender, - gossipsub_listener: GossipSubListener, - signal_rx: UnboundedReceiver, - event_ch: broadcast::Sender, - ) -> Self { + pub fn new(args: Args) -> Self { + let Args { + webrtc_controller, + webrtc_event_stream, + gossipsub_sender, + gossipsub_listener, + signal_rx, + ui_event_ch, + } = args; + let (tx, cmd_rx) = mpsc::unbounded_channel(); let notify = Arc::new(Notify::new()); let notify2 = notify.clone(); @@ -104,7 +115,7 @@ impl BlinkController { gossipsub_listener, cmd_rx, signal_rx, - event_ch, + ui_event_ch, notify2, ) .await; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 655b99c68..75df5fced 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -26,6 +26,7 @@ use warp::{ }; use crate::{ + blink_impl::blink_controller::BlinkController, host_media::{ self, audio::automute::{AutoMuteCmd, AUDIO_CMD_CH}, @@ -90,14 +91,14 @@ impl BlinkImpl { let webrtc_controller = simple_webrtc::Controller::new()?; let webrtc_event_stream = webrtc_controller.get_event_stream(); - let blink_controller = blink_controller::BlinkController::new( + let blink_controller = BlinkController::new(blink_controller::Args { webrtc_controller, webrtc_event_stream, - gossipsub_sender.clone(), - gossipsub_listener.clone(), + gossipsub_sender: gossipsub_sender.clone(), + gossipsub_listener: gossipsub_listener.clone(), signal_rx, - ui_event_ch.clone(), - ); + ui_event_ch: ui_event_ch.clone(), + }); let blink_impl = Self { own_id: Arc::new(warp::sync::RwLock::new(None)), From b344d7085d2541b6d228160611fa8a57615bd5dc Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 14:23:15 -0500 Subject: [PATCH 12/42] fix(fmt) --- extensions/warp-blink-wrtc/src/host_media/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/host_media/mod.rs b/extensions/warp-blink-wrtc/src/host_media/mod.rs index d5417a57f..c92f8d612 100644 --- a/extensions/warp-blink-wrtc/src/host_media/mod.rs +++ b/extensions/warp-blink-wrtc/src/host_media/mod.rs @@ -7,7 +7,7 @@ use cpal::traits::{DeviceTrait, HostTrait}; use once_cell::sync::Lazy; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{broadcast, RwLock}; -use warp::blink::{BlinkEventKind}; +use warp::blink::BlinkEventKind; use warp::crypto::DID; use warp::error::Error; use webrtc::track::track_local::track_local_static_rtp::TrackLocalStaticRTP; From b027b8681085732daa00468aeb897cc44e4ab207 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:08:36 -0500 Subject: [PATCH 13/42] fix clippy --- .../src/blink_impl/blink_controller.rs | 85 +++++++++++++++---- .../src/blink_impl/data/mod.rs | 6 ++ .../src/blink_impl/signaling.rs | 4 + warp/src/blink/call_state.rs | 11 +++ warp/src/blink/mod.rs | 4 + 5 files changed, 94 insertions(+), 16 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 5f13be11e..183a4cf5a 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -113,9 +113,9 @@ impl BlinkController { webrtc_event_stream, gossipsub_sender, gossipsub_listener, - cmd_rx, signal_rx, ui_event_ch, + cmd_rx, notify2, ) .await; @@ -235,14 +235,15 @@ impl BlinkController { } } +#[allow(clippy::too_many_arguments)] async fn run( mut webrtc_controller: simple_webrtc::Controller, mut webrtc_event_stream: WebRtcEventStream, gossipsub_sender: GossipSubSender, gossipsub_listener: GossipSubListener, - mut cmd_rx: UnboundedReceiver, mut signal_rx: UnboundedReceiver, ui_event_ch: broadcast::Sender, + mut cmd_rx: UnboundedReceiver, notify: Arc, ) { let own_id = { @@ -525,28 +526,64 @@ async fn run( } } Cmd::RecordCall { output_dir, rsp } => { - match active_call.and_then(|call_id| call_data_map.get_call_info(call_id)) { - Some(call) => { - let r = host_media::init_recording(Mp4LoggerConfig { - call_id: call.call_id(), - participants: call.participants(), + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + let info = data.get_info(); + match + host_media::init_recording(Mp4LoggerConfig { + call_id: info.call_id(), + participants: info.participants(), audio_codec: AudioCodec::default(), log_path: output_dir.into(), }) - .await; - let _ = rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); - } - None => { - let _ = rsp.send(Err(Error::CallNotInProgress)); + .await + { + Ok(_) => { + data.state.set_self_recording(true); + let topic = ipfs_routes::call_signal_route(&info.call_id()); + let signal = CallSignal::Recording; + if let Err(e) = + gossipsub_sender + .send_signal_aes(info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } + let _ = rsp.send(Ok(())); + } + Err(e) => { + let _ = rsp.send(Err(Error::OtherWithContext(e.to_string()))); + } } + } else { + let _ = rsp.send(Err(Error::CallNotInProgress)); } } Cmd::StopRecording { rsp } => { - if active_call.is_none() { - let _ = rsp.send(Err(Error::CallNotInProgress)); + let call_id = active_call.unwrap_or_default(); + if let Some(data) = call_data_map.map.get_mut(&call_id) { + let info = data.get_info(); + match + host_media::pause_recording() + .await + { + Ok(_) => { + data.state.set_self_recording(false); + let topic = ipfs_routes::call_signal_route(&info.call_id()); + let signal = CallSignal::NotRecording; + if let Err(e) = + gossipsub_sender + .send_signal_aes(info.group_key(), signal, topic) + { + log::error!("failed to send signal: {e}"); + } + let _ = rsp.send(Ok(())); + } + Err(e) => { + let _ = rsp.send(Err(Error::OtherWithContext(e.to_string()))); + } + } } else { - let r = host_media::pause_recording().await; - let _ = rsp.send(r.map_err(|x| Error::OtherWithContext(x.to_string()))); + let _ = rsp.send(Err(Error::CallNotInProgress)); } } } @@ -658,6 +695,22 @@ async fn run( } } }, + signaling::CallSignal::Recording => { + call_data_map.set_recording(call_id, &sender, true); + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantRecording { peer_id: sender }) { + log::error!("failed to send ParticipantRecording event: {e}"); + } + } + } + signaling::CallSignal::NotRecording => { + call_data_map.set_recording(call_id, &sender, false); + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantNotRecording { peer_id: sender }) { + log::error!("failed to send ParticipantNotRecording event: {e}"); + } + } + } }, GossipSubSignal::Initiation { sender, signal } => match signal { signaling::InitiationSignal::Offer { call_info } => { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index ce599d0f8..25d172df4 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -134,4 +134,10 @@ impl CallDataMap { data.state.set_deafened(participant, value); } } + + pub fn set_recording(&mut self, call_id: Uuid, participant: &DID, value: bool) { + if let Some(data) = self.map.get_mut(&call_id) { + data.state.set_recording(participant, value); + } + } } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs index f08aaa3aa..2e890b5a6 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs @@ -53,6 +53,10 @@ pub enum CallSignal { Deafened, #[display(fmt = "Undeafened")] Undeafened, + #[display(fmt = "Recording")] + Recording, + #[display(fmt = "NotRecording")] + NotRecording, } #[derive(Serialize, Deserialize, Display, Clone)] diff --git a/warp/src/blink/call_state.rs b/warp/src/blink/call_state.rs index f0ef6abbe..349584df2 100644 --- a/warp/src/blink/call_state.rs +++ b/warp/src/blink/call_state.rs @@ -50,6 +50,12 @@ impl CallState { } } + pub fn set_recording(&mut self, id: &DID, recording: bool) { + if let Some(participant) = self.participants_joined.get_mut(id) { + participant.recording = recording; + } + } + pub fn set_self_muted(&mut self, muted: bool) { let own_id = self.own_id.clone(); self.set_muted(&own_id, muted); @@ -60,6 +66,11 @@ impl CallState { self.set_deafened(&own_id, deafened); } + pub fn set_self_recording(&mut self, recording: bool) { + let own_id = self.own_id.clone(); + self.set_recording(&own_id, recording); + } + pub fn reset_self(&mut self) { self.participants_joined.remove(&self.own_id); } diff --git a/warp/src/blink/mod.rs b/warp/src/blink/mod.rs index 55656eb8c..11fd821b2 100644 --- a/warp/src/blink/mod.rs +++ b/warp/src/blink/mod.rs @@ -141,6 +141,10 @@ pub enum BlinkEventKind { ParticipantDeafened { peer_id: DID }, #[display(fmt = "ParticipantUndeafened")] ParticipantUndeafened { peer_id: DID }, + #[display(fmt = "ParticipantRecording")] + ParticipantRecording { peer_id: DID }, + #[display(fmt = "ParticipantNotRecording")] + ParticipantNotRecording { peer_id: DID }, /// audio packets were dropped for the peer #[display(fmt = "AudioDegradation")] AudioDegradation { peer_id: DID }, From 94a5dff07ced65e929af36ffb03f3f0eef5b0398 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:20:39 -0500 Subject: [PATCH 14/42] wire up connectionclosed --- .../src/blink_impl/blink_controller.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 183a4cf5a..3f8886117 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -754,7 +754,8 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::Disconnected { peer } - | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { + | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } + | simple_webrtc::events::EmittedEvents::ConnectionClosed { peer }=> { let ac = active_call.unwrap_or_default(); call_data_map.remove_participant(ac, &peer); @@ -762,9 +763,19 @@ async fn run( log::error!("failed to send media_track command: {e}"); } webrtc_controller.hang_up(&peer).await; - }, - simple_webrtc::events::EmittedEvents::ConnectionClosed { peer: _ } => { - // todo + + if let Some(data) = call_data_map.map.get(&ac) { + if data.info.participants().len() == 2 && data.state.participants_joined.iter().count() <= 1 { + log::info!("all participants have successfully been disconnected"); + if let Err(e) = webrtc_controller.deinit().await { + log::error!("webrtc deinit failed: {e}"); + } + //rtp_logger::deinit().await; + host_media::reset().await; + let event = BlinkEventKind::CallTerminated { call_id: ac }; + let _ = ui_event_ch.send(event); + } + } }, simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); From 0865bb6fb6509b6a98d9fc70789faf99a1183ab0 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:22:02 -0500 Subject: [PATCH 15/42] fix clippy --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 3f8886117..8997955d6 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -765,7 +765,7 @@ async fn run( webrtc_controller.hang_up(&peer).await; if let Some(data) = call_data_map.map.get(&ac) { - if data.info.participants().len() == 2 && data.state.participants_joined.iter().count() <= 1 { + if data.info.participants().len() == 2 && data.state.participants_joined.len() <= 1 { log::info!("all participants have successfully been disconnected"); if let Err(e) = webrtc_controller.deinit().await { log::error!("webrtc deinit failed: {e}"); From 6377a4bec1db443bf3d6f4088b97a2836399a143 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:31:17 -0500 Subject: [PATCH 16/42] change ConnectionClosedEvent --- .../src/blink_impl/blink_controller.rs | 14 ++++++++++---- .../warp-blink-wrtc/src/blink_impl/data/mod.rs | 7 ------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 8997955d6..99c238639 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -602,7 +602,7 @@ async fn run( log::debug!("received webrtc signal for non-active call"); continue; } - _ if !call_data_map.participant_in_call(call_id, &sender) => { + _ if !call_data_map.contains_participant(call_id, &sender) => { log::debug!("received signal from someone who isn't part of the call"); continue; } @@ -626,7 +626,7 @@ async fn run( }, }, GossipSubSignal::Call { sender, call_id, signal } => match signal { - _ if !call_data_map.participant_in_call(call_id, &sender) => { + _ if !call_data_map.contains_participant(call_id, &sender) => { log::debug!("received signal from someone who isn't part of the call"); continue; } @@ -754,14 +754,16 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::Disconnected { peer } - | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } - | simple_webrtc::events::EmittedEvents::ConnectionClosed { peer }=> { + | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { + // todo: dont' call this multiple times per peer let ac = active_call.unwrap_or_default(); call_data_map.remove_participant(ac, &peer); if let Err(e) = host_media::remove_sink_track(peer.clone()).await { log::error!("failed to send media_track command: {e}"); } + + // possibly redundant webrtc_controller.hang_up(&peer).await; if let Some(data) = call_data_map.map.get(&ac) { @@ -777,6 +779,10 @@ async fn run( } } }, + simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => { + // hoping this will trigger the Disconnected event. + webrtc_controller.hang_up(&peer).await; + } simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Sdp(*sdp); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index 25d172df4..07631bda4 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -102,13 +102,6 @@ impl CallDataMap { } } - pub fn participant_in_call(&self, call_id: Uuid, peer_id: &DID) -> bool { - self.map - .get(&call_id) - .map(|x| x.info.contains_participant(peer_id)) - .unwrap_or_default() - } - pub fn remove_call(&mut self, call_id: Uuid) { self.map.remove(&call_id); } From a346e9db2f71a80dfcfd653de1a0d71105d85c42 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:33:53 -0500 Subject: [PATCH 17/42] add logging --- .../warp-blink-wrtc/src/blink_impl/blink_controller.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 99c238639..a8f4482ee 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -285,6 +285,7 @@ async fn run( break; } }; + log::debug!("blink cmd: {:?}", std::mem::discriminant(&cmd)); match cmd { Cmd::OfferCall { call_info, rsp } => { let prev_active = active_call.unwrap_or_default(); @@ -589,14 +590,15 @@ async fn run( } }, opt = signal_rx.recv() => { - let cmd = match opt { + let signal = match opt { Some(r) => r, None => { log::debug!("blink handler signal_rx channel is closed. quitting"); break; } }; - match cmd { + log::debug!("gossipsub signal: {:?}", std::mem::discriminant(&signal)); + match signal { GossipSubSignal::Peer { sender, call_id, signal } => match *signal { _ if !active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() => { log::debug!("received webrtc signal for non-active call"); @@ -735,7 +737,7 @@ async fn run( continue; } }; - + log::debug!("webrtc event: {:?}", std::mem::discriminant(&event)); match event { simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); From 9fafe0ecfd1b82d42fd830a5311d4a4f6ea86e58 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:39:05 -0500 Subject: [PATCH 18/42] remove useless logging --- .../warp-blink-wrtc/src/blink_impl/blink_controller.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index a8f4482ee..da305d247 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -32,6 +32,7 @@ use super::{ gossipsub_sender::GossipSubSender, }; +#[derive(Debug)] enum Cmd { OfferCall { call_info: CallInfo, @@ -285,7 +286,6 @@ async fn run( break; } }; - log::debug!("blink cmd: {:?}", std::mem::discriminant(&cmd)); match cmd { Cmd::OfferCall { call_info, rsp } => { let prev_active = active_call.unwrap_or_default(); @@ -597,7 +597,6 @@ async fn run( break; } }; - log::debug!("gossipsub signal: {:?}", std::mem::discriminant(&signal)); match signal { GossipSubSignal::Peer { sender, call_id, signal } => match *signal { _ if !active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() => { @@ -737,7 +736,7 @@ async fn run( continue; } }; - log::debug!("webrtc event: {:?}", std::mem::discriminant(&event)); + log::debug!("webrtc event: {:?}", &event); match event { simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); From 8ae71daf437afafb6c6885fab7e19cd2f23fff27 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:45:22 -0500 Subject: [PATCH 19/42] change logging --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index da305d247..54bcd7db8 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -736,7 +736,6 @@ async fn run( continue; } }; - log::debug!("webrtc event: {:?}", &event); match event { simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); @@ -756,6 +755,7 @@ async fn run( }, simple_webrtc::events::EmittedEvents::Disconnected { peer } | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { + log::debug!("webrtc: disconnected or connection failed"); // todo: dont' call this multiple times per peer let ac = active_call.unwrap_or_default(); call_data_map.remove_participant(ac, &peer); @@ -781,6 +781,7 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => { + log::debug!("webrtc: connection closed"); // hoping this will trigger the Disconnected event. webrtc_controller.hang_up(&peer).await; } From d7217142193f7805da6b1d63deabebf3c4370604 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 15:48:30 -0500 Subject: [PATCH 20/42] handle connectionclosed with other events again --- .../src/blink_impl/blink_controller.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 54bcd7db8..bb3fd2c14 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -754,9 +754,9 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::Disconnected { peer } - | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } => { - log::debug!("webrtc: disconnected or connection failed"); - // todo: dont' call this multiple times per peer + | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } + | simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => { + log::debug!("webrtc: closed, disconnected or connection failed"); let ac = active_call.unwrap_or_default(); call_data_map.remove_participant(ac, &peer); @@ -780,11 +780,6 @@ async fn run( } } }, - simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => { - log::debug!("webrtc: connection closed"); - // hoping this will trigger the Disconnected event. - webrtc_controller.hang_up(&peer).await; - } simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Sdp(*sdp); From d68e01e95d4625e12f3762b448af4a780acaace8 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 16:27:37 -0500 Subject: [PATCH 21/42] try to fix blink --- .../src/blink_impl/blink_controller.rs | 37 ++++++++++++------- .../src/blink_impl/gossipsub_listener.rs | 12 +++--- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index bb3fd2c14..b3d6959b5 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -292,10 +292,12 @@ async fn run( if let Some(data) = call_data_map.map.get_mut(&prev_active) { data.state.reset_self(); } - if active_call.replace(call_info.call_id()).is_some() { + let call_id = call_info.call_id(); + if active_call.as_ref().map(|x| x != &call_id).unwrap_or_default() { let _ = webrtc_controller.deinit().await; host_media::reset().await; } + active_call.replace(call_id); call_data_map.add_call(call_info.clone(), own_id); // automatically add an audio track @@ -318,7 +320,7 @@ async fn run( gossipsub_listener .subscribe_call(call_info.call_id(), call_info.group_key()); gossipsub_listener - .connect_webrtc(call_info.call_id(), own_id.clone()); + .subscribe_webrtc(call_info.call_id(), own_id.clone()); // todo: resend periodically. perhaps somewhere else let mut participants = call_info.participants(); @@ -359,10 +361,11 @@ async fn run( if let Some(data) = call_data_map.map.get_mut(&prev_active) { data.state.reset_self(); } - if active_call.replace(call_id).is_some() { + if active_call.as_ref().map(|x| x != &call_id).unwrap_or_default() { let _ = webrtc_controller.deinit().await; host_media::reset().await; } + active_call.replace(call_id); // automatically add an audio track let webrtc_codec = AudioCodec::default(); @@ -382,7 +385,7 @@ async fn run( match r { Ok(_) => { gossipsub_listener.subscribe_call(call_id, call_info.group_key()); - gossipsub_listener.connect_webrtc(call_id, own_id.clone()); + gossipsub_listener.subscribe_webrtc(call_id, own_id.clone()); let topic = ipfs_routes::call_signal_route(&call_id); // todo? periodically re-send join signals. perhaps somewhere else @@ -419,7 +422,18 @@ async fn run( }, Cmd::LeaveCall { call_id } => { let call_id = call_id.unwrap_or(active_call.unwrap_or_default()); - match call_data_map.get_call_info(call_id) { + let info = call_data_map.get_call_info(call_id); + if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + call_data_map.leave_call(call_id); + let _ = active_call.take(); + let _ = webrtc_controller.deinit().await; + host_media::reset().await; + if let Err(e) = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id }) { + log::error!("failed to send CallTerminated Event: {e}"); + } + } + + match info { Some(info) => { let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Leave; @@ -433,15 +447,6 @@ async fn run( log::error!("failed to leave call - not found"); } } - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { - call_data_map.leave_call(call_id); - let _ = active_call.take(); - let _ = webrtc_controller.deinit().await; - host_media::reset().await; - if let Err(e) = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id }) { - log::error!("failed to send CallTerminated Event: {e}"); - } - } }, Cmd::MuteSelf => { let call_id = active_call.unwrap_or_default(); @@ -655,6 +660,7 @@ async fn run( } } else if is_call_empty { call_data_map.remove_call(call_id); + gossipsub_listener.unsubscribe_call(call_id); if let Err(e) = ui_event_ch.send(BlinkEventKind::CallCancelled { call_id }) { log::error!("failed to send CallCancelled event: {e}"); } @@ -777,6 +783,9 @@ async fn run( host_media::reset().await; let event = BlinkEventKind::CallTerminated { call_id: ac }; let _ = ui_event_ch.send(event); + + gossipsub_listener.unsubscribe_call(ac); + gossipsub_listener.unsubscribe_webrtc(ac); } } }, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 0db6edc48..46723f239 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -73,7 +73,7 @@ impl GossipSubListener { .send(GossipSubCmd::SubscribeCall { call_id, group_key }); } - pub fn connect_webrtc(&self, call_id: Uuid, peer: DID) { + pub fn subscribe_webrtc(&self, call_id: Uuid, peer: DID) { let _ = self.ch.send(GossipSubCmd::ConnectWebRtc { call_id, peer }); } @@ -112,8 +112,8 @@ async fn run( let mut current_call: Option = None; let mut subscribed_calls: HashMap> = HashMap::new(); - let webrtc_notify = Arc::new(Notify::new()); - let call_signal_notify = Arc::new(Notify::new()); + // replace webrtc_notify after notifying waiters + let mut webrtc_notify = Arc::new(Notify::new()); let call_offer_notify = Arc::new(Notify::new()); loop { tokio::select! { @@ -126,11 +126,13 @@ async fn run( if current_call.as_ref().map(|x| x == &call_id).unwrap_or_default(){ let _ = current_call.take(); webrtc_notify.notify_waiters(); + webrtc_notify = Arc::new(Notify::new()); } } GossipSubCmd::DisconnectWebrtc { call_id } => { if current_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { webrtc_notify.notify_waiters(); + webrtc_notify = Arc::new(Notify::new()); } } GossipSubCmd::SubscribeCall { call_id, group_key } => { @@ -194,6 +196,7 @@ async fn run( if !current_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { if current_call.is_some() { webrtc_notify.notify_waiters(); + webrtc_notify = Arc::new(Notify::new()); } current_call.replace(call_id); } @@ -250,7 +253,6 @@ async fn run( }); }, GossipSubCmd::ReceiveCalls { own_id } => { - call_offer_notify.notify_waiters(); let mut call_offer_stream = match ipfs .pubsub_subscribe(call_initiation_route(&own_id)) .await @@ -313,8 +315,8 @@ async fn run( } } + log::debug!("quitting gossipsub listener"); webrtc_notify.notify_waiters(); - call_signal_notify.notify_waiters(); call_offer_notify.notify_waiters(); } } From 652075d7f77a72d6913fd300e8f19a09170c05e6 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 16:30:28 -0500 Subject: [PATCH 22/42] fix gossipsub listener --- .../warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 46723f239..8b9df8c28 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -314,9 +314,9 @@ async fn run( break; } } - - log::debug!("quitting gossipsub listener"); - webrtc_notify.notify_waiters(); - call_offer_notify.notify_waiters(); } + + log::debug!("quitting gossipsub listener"); + webrtc_notify.notify_waiters(); + call_offer_notify.notify_waiters(); } From c551e8d1d131be9977c27b270c2757ceea21902e Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 16:43:16 -0500 Subject: [PATCH 23/42] fix some stuff --- .../src/blink_impl/blink_controller.rs | 24 ++++++++++++++----- .../src/blink_impl/gossipsub_listener.rs | 4 ++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index b3d6959b5..1597a6cb0 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -293,9 +293,17 @@ async fn run( data.state.reset_self(); } let call_id = call_info.call_id(); - if active_call.as_ref().map(|x| x != &call_id).unwrap_or_default() { + if let Some(prev_id) = active_call.as_ref() { + if prev_id == &call_id { + log::debug!("tried to offer call which is already in progress"); + continue; + } + let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id: *prev_id }); let _ = webrtc_controller.deinit().await; host_media::reset().await; + if let Some(data) = call_data_map.map.get_mut(prev_id) { + data.state.reset_self(); + } } active_call.replace(call_id); call_data_map.add_call(call_info.clone(), own_id); @@ -357,13 +365,17 @@ async fn run( } }; - let prev_active = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&prev_active) { - data.state.reset_self(); - } - if active_call.as_ref().map(|x| x != &call_id).unwrap_or_default() { + if let Some(prev_id) = active_call.as_ref() { + if prev_id == &call_id { + log::debug!("tried to answer call which is already in progress"); + continue; + } + let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id: *prev_id }); let _ = webrtc_controller.deinit().await; host_media::reset().await; + if let Some(data) = call_data_map.map.get_mut(prev_id) { + data.state.reset_self(); + } } active_call.replace(call_id); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs index 8b9df8c28..e3f90154f 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_listener.rs @@ -244,7 +244,7 @@ async fn run( }; } None => { - log::debug!("peer signal stream terminated!"); + log::debug!("peer signal stream closed!"); break; } } @@ -295,7 +295,7 @@ async fn run( }; } None => { - log::debug!("call offer stream terminated!"); + log::debug!("call offer stream closed!"); break; } } From 32412e2a451ab348b81bf41393d798782f7aeff3 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 20:09:20 -0500 Subject: [PATCH 24/42] added logging --- .../warp-blink-wrtc/src/blink_impl/blink_controller.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 1597a6cb0..e03c7c9e7 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -330,6 +330,7 @@ async fn run( gossipsub_listener .subscribe_webrtc(call_info.call_id(), own_id.clone()); + log::debug!("sending offer signal"); // todo: resend periodically. perhaps somewhere else let mut participants = call_info.participants(); participants.retain(|x| x != own_id); @@ -400,6 +401,7 @@ async fn run( gossipsub_listener.subscribe_webrtc(call_id, own_id.clone()); let topic = ipfs_routes::call_signal_route(&call_id); + log::debug!("answering call. sending join signal"); // todo? periodically re-send join signals. perhaps somewhere else let signal = CallSignal::Join; if let Err(e) = @@ -650,8 +652,8 @@ async fn run( } signaling::CallSignal::Join => { call_data_map.add_participant(call_id, &sender); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + log::debug!("dialing peer"); if let Err(e) = webrtc_controller.dial(&sender).await { log::error!("failed to dial peer: {e}"); continue; @@ -809,6 +811,7 @@ async fn run( } }, simple_webrtc::events::EmittedEvents::CallInitiated { dest, sdp } => { + log::debug!("sending dial signal"); let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Dial(*sdp); if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { From 2d2fb43ed0112dcaafc9693a1f348040ea9d33ba Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 20:45:27 -0500 Subject: [PATCH 25/42] make gossipsub listener's send signal functions async --- .../src/blink_impl/blink_controller.rs | 47 +++++++------------ .../src/blink_impl/gossipsub_sender.rs | 26 +++++++--- .../warp-blink-wrtc/src/blink_impl/mod.rs | 1 + .../src/blink_impl/sender_queue.rs | 0 4 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index e03c7c9e7..c774f063d 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -247,25 +247,14 @@ async fn run( mut cmd_rx: UnboundedReceiver, notify: Arc, ) { - let own_id = { - let notify2 = notify.clone(); - let fut = gossipsub_sender.get_own_id(); - tokio::select! { - _ = notify2.notified() => { - log::debug!("quitting blink event handler"); - return; - } - r = fut => { - match r { - Ok(r) => r, - Err(e) => { - log::debug!("failed to get own id. quitting blink event handler: {e}"); - return; - } - } - } + let own_id = match gossipsub_sender.get_own_id().await { + Ok(r) => r, + Err(e) => { + log::error!("failed to get own id. quitting blnk controller"); + return; } }; + // prevent accidental moves let own_id = &own_id; @@ -340,7 +329,7 @@ async fn run( call_info: call_info.clone(), }; - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { log::error!("failed to send signal: {e}"); } } @@ -406,7 +395,7 @@ async fn run( let signal = CallSignal::Join; if let Err(e) = gossipsub_sender - .send_signal_aes(call_info.group_key(), signal, topic) + .send_signal_aes(call_info.group_key(), signal, topic).await { let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { @@ -452,7 +441,7 @@ async fn run( let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Leave; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic) + .send_signal_aes(info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } @@ -470,7 +459,7 @@ async fn run( let signal = CallSignal::Muted; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + .send_signal_aes(data.info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } else { @@ -486,7 +475,7 @@ async fn run( let signal = CallSignal::Unmuted; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + .send_signal_aes(data.info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } else { @@ -505,7 +494,7 @@ async fn run( let signal = CallSignal::Deafened; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + .send_signal_aes(data.info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } @@ -522,7 +511,7 @@ async fn run( let signal = CallSignal::Undeafened; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + .send_signal_aes(data.info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } @@ -564,7 +553,7 @@ async fn run( let signal = CallSignal::Recording; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic) + .send_signal_aes(info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } @@ -592,7 +581,7 @@ async fn run( let signal = CallSignal::NotRecording; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic) + .send_signal_aes(info.group_key(), signal, topic).await { log::error!("failed to send signal: {e}"); } @@ -760,7 +749,7 @@ async fn run( simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Ice(*candidate); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { log::error!("failed to send signal: {e}"); } }, @@ -806,7 +795,7 @@ async fn run( simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Sdp(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { log::error!("failed to send signal: {e}"); } }, @@ -814,7 +803,7 @@ async fn run( log::debug!("sending dial signal"); let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Dial(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { log::error!("failed to send signal: {e}"); } }, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index b54fda7c2..9aa5323cc 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -24,11 +24,13 @@ enum GossipSubCmd { group_key: Vec, signal: Vec, topic: String, + rsp: oneshot::Sender>, }, SendEcdh { dest: DID, signal: Vec, topic: String, + rsp: oneshot::Sender>, }, DecodeEcdh { src: DID, @@ -70,35 +72,39 @@ impl GossipSubSender { Ok(id) } - pub fn send_signal_aes( + pub async fn send_signal_aes( &self, group_key: Vec, signal: T, topic: String, ) -> anyhow::Result<()> { + let (tx, rx) = oneshot::channel(); let signal = serde_cbor::to_vec(&signal)?; self.ch.send(GossipSubCmd::SendAes { group_key, signal, topic, + rsp: tx, })?; - + rx.await??; Ok(()) } - pub fn send_signal_ecdh( + pub async fn send_signal_ecdh( &self, dest: DID, signal: T, topic: String, ) -> anyhow::Result<()> { + let (tx, rx) = oneshot::channel(); let signal = serde_cbor::to_vec(&signal)?; self.ch.send(GossipSubCmd::SendEcdh { dest, signal, topic, + rsp: tx, })?; - + rx.await??; Ok(()) } @@ -176,28 +182,36 @@ async fn run( GossipSubCmd::GetOwnId { rsp } => { let _ = rsp.send(own_id.clone()); } - GossipSubCmd::SendAes { group_key, signal, topic } => { + GossipSubCmd::SendAes { group_key, signal, topic, rsp } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { Ok(r) => r, Err(e) => { log::error!("failed to encrypt aes message: {e}"); + let _ = rsp.send(Err(anyhow::anyhow!(e))); continue; } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { log::error!("failed to publish message: {e}"); + let _ = rsp.send(Err(anyhow::anyhow!(e))); + } else { + let _ = rsp.send(Ok(())); } }, - GossipSubCmd::SendEcdh { dest, signal, topic } => { + GossipSubCmd::SendEcdh { dest, signal, topic, rsp } => { let encrypted = match ecdh_encrypt(&own_id, &dest, signal) { Ok(r) => r, Err(e) => { log::error!("failed to encrypt ecdh message: {e}"); + let _ = rsp.send(Err(anyhow::anyhow!(e))); continue; } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { log::error!("failed to publish message: {e}"); + let _ = rsp.send(Err(anyhow::anyhow!(e))); + }else { + let _ = rsp.send(Ok(())); } } GossipSubCmd::DecodeEcdh { src, data, rsp } => { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 75df5fced..56a77d2ff 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -3,6 +3,7 @@ mod data; mod blink_controller; mod gossipsub_listener; mod gossipsub_sender; +mod sender_queue; mod signaling; mod store; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs b/extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs new file mode 100644 index 000000000..e69de29bb From d05a1310f978f2f3a7ec6b1be9a77c0a5e570fb7 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 21:07:02 -0500 Subject: [PATCH 26/42] add queue to gossipsub sender --- .../src/blink_impl/blink_controller.rs | 26 ++++--- .../src/blink_impl/gossipsub_sender.rs | 77 +++++++++++-------- .../warp-blink-wrtc/src/blink_impl/mod.rs | 1 - .../src/blink_impl/sender_queue.rs | 0 4 files changed, 60 insertions(+), 44 deletions(-) delete mode 100644 extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index c774f063d..5f7d5f69d 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -329,7 +329,7 @@ async fn run( call_info: call_info.clone(), }; - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { log::error!("failed to send signal: {e}"); } } @@ -395,7 +395,7 @@ async fn run( let signal = CallSignal::Join; if let Err(e) = gossipsub_sender - .send_signal_aes(call_info.group_key(), signal, topic).await + .send_signal_aes(call_info.group_key(), signal, topic) { let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { @@ -428,6 +428,7 @@ async fn run( let info = call_data_map.get_call_info(call_id); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { call_data_map.leave_call(call_id); + let _ = gossipsub_sender.empty_queue(); let _ = active_call.take(); let _ = webrtc_controller.deinit().await; host_media::reset().await; @@ -441,7 +442,7 @@ async fn run( let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Leave; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic).await + .send_signal_aes(info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } @@ -459,7 +460,7 @@ async fn run( let signal = CallSignal::Muted; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic).await + .send_signal_aes(data.info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } else { @@ -475,7 +476,7 @@ async fn run( let signal = CallSignal::Unmuted; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic).await + .send_signal_aes(data.info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } else { @@ -494,7 +495,7 @@ async fn run( let signal = CallSignal::Deafened; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic).await + .send_signal_aes(data.info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } @@ -511,7 +512,7 @@ async fn run( let signal = CallSignal::Undeafened; if let Err(e) = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic).await + .send_signal_aes(data.info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } @@ -553,7 +554,7 @@ async fn run( let signal = CallSignal::Recording; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic).await + .send_signal_aes(info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } @@ -581,7 +582,7 @@ async fn run( let signal = CallSignal::NotRecording; if let Err(e) = gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic).await + .send_signal_aes(info.group_key(), signal, topic) { log::error!("failed to send signal: {e}"); } @@ -749,7 +750,7 @@ async fn run( simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Ice(*candidate); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { log::error!("failed to send signal: {e}"); } }, @@ -789,13 +790,14 @@ async fn run( gossipsub_listener.unsubscribe_call(ac); gossipsub_listener.unsubscribe_webrtc(ac); + let _ = gossipsub_sender.empty_queue(); } } }, simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Sdp(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { log::error!("failed to send signal: {e}"); } }, @@ -803,7 +805,7 @@ async fn run( log::debug!("sending dial signal"); let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); let signal = PeerSignal::Dial(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic).await { + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { log::error!("failed to send signal: {e}"); } }, diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 9aa5323cc..98850a1fe 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -1,4 +1,9 @@ -use std::{fmt::Display, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, VecDeque}, + fmt::Display, + sync::Arc, + time::Duration, +}; use futures::channel::oneshot; use rust_ipfs::Ipfs; @@ -24,13 +29,11 @@ enum GossipSubCmd { group_key: Vec, signal: Vec, topic: String, - rsp: oneshot::Sender>, }, SendEcdh { dest: DID, signal: Vec, topic: String, - rsp: oneshot::Sender>, }, DecodeEcdh { src: DID, @@ -40,6 +43,7 @@ enum GossipSubCmd { GetOwnId { rsp: oneshot::Sender, }, + EmptyQueue, } #[derive(Clone)] @@ -72,39 +76,33 @@ impl GossipSubSender { Ok(id) } - pub async fn send_signal_aes( + pub fn send_signal_aes( &self, group_key: Vec, signal: T, topic: String, ) -> anyhow::Result<()> { - let (tx, rx) = oneshot::channel(); let signal = serde_cbor::to_vec(&signal)?; self.ch.send(GossipSubCmd::SendAes { group_key, signal, topic, - rsp: tx, })?; - rx.await??; Ok(()) } - pub async fn send_signal_ecdh( + pub fn send_signal_ecdh( &self, dest: DID, signal: T, topic: String, ) -> anyhow::Result<()> { - let (tx, rx) = oneshot::channel(); let signal = serde_cbor::to_vec(&signal)?; self.ch.send(GossipSubCmd::SendEcdh { dest, signal, topic, - rsp: tx, })?; - rx.await??; Ok(()) } @@ -134,6 +132,11 @@ impl GossipSubSender { let data: T = serde_cbor::from_slice(&bytes)?; Ok(data) } + + pub fn empty_queue(&self) -> anyhow::Result<()> { + self.ch.send(GossipSubCmd::EmptyQueue)?; + Ok(()) + } } async fn run( @@ -175,44 +178,56 @@ async fn run( } }; + let mut ecdh_queue: HashMap> = HashMap::new(); + loop { tokio::select! { + _ = timer.tick() => { + for (_dest, queue) in ecdh_queue.iter_mut() { + while let Some(cmd) = queue.pop_front() { + match cmd { + GossipSubCmd::SendEcdh { dest, signal, topic } => { + let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt ecdh message: {e}"); + break; + } + }; + if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { + queue.push_front(GossipSubCmd::SendEcdh { dest, signal, topic }); + break; + } + } + _ => {} + } + } + } + } opt = ch.recv() => match opt { Some(cmd) => match cmd { + GossipSubCmd::EmptyQueue => { + ecdh_queue.clear(); + } GossipSubCmd::GetOwnId { rsp } => { let _ = rsp.send(own_id.clone()); } - GossipSubCmd::SendAes { group_key, signal, topic, rsp } => { + GossipSubCmd::SendAes { group_key, signal, topic } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { Ok(r) => r, Err(e) => { log::error!("failed to encrypt aes message: {e}"); - let _ = rsp.send(Err(anyhow::anyhow!(e))); continue; } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { log::error!("failed to publish message: {e}"); - let _ = rsp.send(Err(anyhow::anyhow!(e))); - } else { - let _ = rsp.send(Ok(())); + } }, - GossipSubCmd::SendEcdh { dest, signal, topic, rsp } => { - let encrypted = match ecdh_encrypt(&own_id, &dest, signal) { - Ok(r) => r, - Err(e) => { - log::error!("failed to encrypt ecdh message: {e}"); - let _ = rsp.send(Err(anyhow::anyhow!(e))); - continue; - } - }; - if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { - log::error!("failed to publish message: {e}"); - let _ = rsp.send(Err(anyhow::anyhow!(e))); - }else { - let _ = rsp.send(Ok(())); - } + GossipSubCmd::SendEcdh { dest, signal, topic } => { + let queue = ecdh_queue.entry(dest.clone()).or_insert(VecDeque::new()); + queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); } GossipSubCmd::DecodeEcdh { src, data, rsp } => { let r = || { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 56a77d2ff..75df5fced 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -3,7 +3,6 @@ mod data; mod blink_controller; mod gossipsub_listener; mod gossipsub_sender; -mod sender_queue; mod signaling; mod store; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs b/extensions/warp-blink-wrtc/src/blink_impl/sender_queue.rs deleted file mode 100644 index e69de29bb..000000000 From 5218ead9a34273f5be2904803dde74346ca26b6f Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 21:10:18 -0500 Subject: [PATCH 27/42] fix clippy --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 5f7d5f69d..4fc732401 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -250,7 +250,7 @@ async fn run( let own_id = match gossipsub_sender.get_own_id().await { Ok(r) => r, Err(e) => { - log::error!("failed to get own id. quitting blnk controller"); + log::error!("failed to get own id. quitting bilnk controller: {e}"); return; } }; From 2203293f20c6395ceff13c8bd3c641622600f4ec Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 21:10:38 -0500 Subject: [PATCH 28/42] fix typo --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 4fc732401..1fa56834c 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -250,7 +250,7 @@ async fn run( let own_id = match gossipsub_sender.get_own_id().await { Ok(r) => r, Err(e) => { - log::error!("failed to get own id. quitting bilnk controller: {e}"); + log::error!("failed to get own id. quitting blink controller: {e}"); return; } }; From 9594e8d60567313c2432813908259b3d40d0a924 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Wed, 8 Nov 2023 21:38:32 -0500 Subject: [PATCH 29/42] resend join signal. might be a bug --- .../src/blink_impl/blink_controller.rs | 45 ++++++++++++++++--- .../src/blink_impl/gossipsub_sender.rs | 6 ++- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 1fa56834c..3b4101182 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -5,11 +5,14 @@ use super::signaling::{ self, ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal, }; -use std::sync::Arc; -use tokio::sync::{ - broadcast, - mpsc::{self, UnboundedReceiver, UnboundedSender}, - Notify, +use std::{sync::Arc, time::Duration}; +use tokio::{ + sync::{ + broadcast, + mpsc::{self, UnboundedReceiver, UnboundedSender}, + Notify, + }, + time::Instant, }; use uuid::Uuid; use warp::{ @@ -260,6 +263,12 @@ async fn run( let mut call_data_map = CallDataMap::new(own_id.clone()); let mut active_call: Option = None; + // if you aren't the one to offer the call, then this will get set to true. + let mut resend_join = false; + let mut timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(5000), + Duration::from_millis(5000), + ); loop { tokio::select! { @@ -267,6 +276,24 @@ async fn run( log::debug!("quitting blink event handler"); break; }, + _ = timer.tick() => { + if !resend_join { + continue; + } + if let Some(call_id) = active_call.as_ref() { + let topic = ipfs_routes::call_signal_route(call_id); + if let Some(data) = call_data_map.map.get(call_id) { + if data.state.participants_joined.len() == data.info.participants().len() { + continue; + } + let signal = CallSignal::Join; + let _ = gossipsub_sender + .send_signal_aes(data.info.group_key(), signal, topic); + } + } + + + } opt = cmd_rx.recv() => { let cmd = match opt { Some(r) => r, @@ -391,7 +418,6 @@ async fn run( let topic = ipfs_routes::call_signal_route(&call_id); log::debug!("answering call. sending join signal"); - // todo? periodically re-send join signals. perhaps somewhere else let signal = CallSignal::Join; if let Err(e) = gossipsub_sender @@ -399,6 +425,7 @@ async fn run( { let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { + resend_join = true; let _ = rsp.send(Ok(())); } } @@ -424,6 +451,7 @@ async fn run( let _ = webrtc_controller.remove_media_source(source_id).await; }, Cmd::LeaveCall { call_id } => { + resend_join = false; let call_id = call_id.unwrap_or(active_call.unwrap_or_default()); let info = call_data_map.get_call_info(call_id); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { @@ -641,6 +669,10 @@ async fn run( continue; } signaling::CallSignal::Join => { + // ignore extra join signals + if call_data_map.contains_participant(call_id, &sender) { + continue; + } call_data_map.add_participant(call_id, &sender); if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { log::debug!("dialing peer"); @@ -780,6 +812,7 @@ async fn run( if let Some(data) = call_data_map.map.get(&ac) { if data.info.participants().len() == 2 && data.state.participants_joined.len() <= 1 { log::info!("all participants have successfully been disconnected"); + resend_join = false; if let Err(e) = webrtc_controller.deinit().await { log::error!("webrtc deinit failed: {e}"); } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 98850a1fe..d339f32f9 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -179,6 +179,10 @@ async fn run( }; let mut ecdh_queue: HashMap> = HashMap::new(); + let mut timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(2000), + Duration::from_millis(2000), + ); loop { tokio::select! { @@ -221,7 +225,7 @@ async fn run( } }; if let Err(e) = ipfs.pubsub_publish(topic, encrypted).await { - log::error!("failed to publish message: {e}"); + log::error!("failed to publish aes message: {e}"); } }, From e3368cfcdd823960766932255e318b1d2b19eb71 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:29:18 -0500 Subject: [PATCH 30/42] change signaling to periodically announce presence and then dial peers who aren't connected and have a lesser DID --- .../src/blink_impl/blink_controller.rs | 256 +++++++++--------- .../src/blink_impl/data/mod.rs | 34 +++ .../src/blink_impl/gossipsub_sender.rs | 82 +++++- .../src/blink_impl/signaling.rs | 2 +- .../warp-blink-wrtc/src/simple_webrtc/mod.rs | 4 + warp/src/error.rs | 2 + 6 files changed, 245 insertions(+), 135 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 3b4101182..a4efb353d 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -260,14 +260,12 @@ async fn run( // prevent accidental moves let own_id = &own_id; + let own_id_str = own_id.to_string(); let mut call_data_map = CallDataMap::new(own_id.clone()); - let mut active_call: Option = None; - // if you aren't the one to offer the call, then this will get set to true. - let mut resend_join = false; - let mut timer = tokio::time::interval_at( - Instant::now() + Duration::from_millis(5000), - Duration::from_millis(5000), + let mut dial_timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(1000), + Duration::from_millis(1000), ); loop { @@ -276,23 +274,26 @@ async fn run( log::debug!("quitting blink event handler"); break; }, - _ = timer.tick() => { - if !resend_join { - continue; - } - if let Some(call_id) = active_call.as_ref() { - let topic = ipfs_routes::call_signal_route(call_id); - if let Some(data) = call_data_map.map.get(call_id) { - if data.state.participants_joined.len() == data.info.participants().len() { + _ = dial_timer.tick() => { + if let Some(data) = call_data_map.get_active() { + let call_id = data.info.call_id(); + for (peer_id, _) in data.state.participants_joined.iter() { + if webrtc_controller.is_connected(peer_id) { continue; } - let signal = CallSignal::Join; - let _ = gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic); + let peer_str = peer_id.to_string(); + if peer_str < own_id_str { + log::debug!("dialing peer"); + if let Err(e) = webrtc_controller.dial(&peer_id).await { + log::error!("failed to dial peer: {e}"); + continue; + } + if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: peer_id.clone() }) { + log::error!("failed to send ParticipantJoined Event: {e}"); + } + } } } - - } opt = cmd_rx.recv() => { let cmd = match opt { @@ -304,25 +305,21 @@ async fn run( }; match cmd { Cmd::OfferCall { call_info, rsp } => { - let prev_active = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&prev_active) { - data.state.reset_self(); + if call_data_map.is_active_call(call_info.call_id()) { + log::debug!("tried to offer call which is already in progress"); + let _ = rsp.send(Err(Error::CallAlreadyInProgress)); + continue; } - let call_id = call_info.call_id(); - if let Some(prev_id) = active_call.as_ref() { - if prev_id == &call_id { - log::debug!("tried to offer call which is already in progress"); - continue; - } - let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id: *prev_id }); + if let Some(data) = call_data_map.get_active_mut() { + data.state.reset_self(); + let call_id = data.info.call_id(); + let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id}); let _ = webrtc_controller.deinit().await; host_media::reset().await; - if let Some(data) = call_data_map.map.get_mut(prev_id) { - data.state.reset_self(); - } } - active_call.replace(call_id); + let call_id = call_info.call_id(); call_data_map.add_call(call_info.clone(), own_id); + call_data_map.set_active(call_id); // automatically add an audio track let webrtc_codec = AudioCodec::default(); @@ -341,13 +338,13 @@ async fn run( webrtc_codec).await { Ok(_) => { + log::debug!("sending offer signal"); + let call_id = call_info.call_id(); gossipsub_listener - .subscribe_call(call_info.call_id(), call_info.group_key()); + .subscribe_call(call_id, call_info.group_key()); gossipsub_listener - .subscribe_webrtc(call_info.call_id(), own_id.clone()); + .subscribe_webrtc(call_id, own_id.clone()); - log::debug!("sending offer signal"); - // todo: resend periodically. perhaps somewhere else let mut participants = call_info.participants(); participants.retain(|x| x != own_id); for dest in participants { @@ -360,6 +357,16 @@ async fn run( log::error!("failed to send signal: {e}"); } } + + // todo: what to do if the signal fails to send? only happens if there's a problem with gossipsub sender + let topic = ipfs_routes::call_signal_route(&call_id); + let signal = CallSignal::Announce; + if let Err(e) = + gossipsub_sender + .send_signal_aes(call_info.group_key(), signal, topic) + { + log::error!("failed to send announce signal: {e}"); + } let _ = rsp.send(Ok(())); } Err(e) => { @@ -374,6 +381,12 @@ async fn run( } }, Cmd::AnswerCall { call_id, rsp } => { + if call_data_map.is_active_call(call_id) { + log::debug!("tried to answer call which is already in progress"); + let _ = rsp.send(Err(Error::CallAlreadyInProgress)); + continue; + } + let call_info = match call_data_map.get_call_info(call_id) { Some(r) => r, None => { @@ -382,19 +395,14 @@ async fn run( } }; - if let Some(prev_id) = active_call.as_ref() { - if prev_id == &call_id { - log::debug!("tried to answer call which is already in progress"); - continue; - } - let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id: *prev_id }); + if let Some(data) = call_data_map.get_active_mut() { + data.state.reset_self(); + let _ = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id: data.info.call_id() }); let _ = webrtc_controller.deinit().await; host_media::reset().await; - if let Some(data) = call_data_map.map.get_mut(prev_id) { - data.state.reset_self(); - } } - active_call.replace(call_id); + + call_data_map.set_active(call_id); // automatically add an audio track let webrtc_codec = AudioCodec::default(); @@ -413,19 +421,17 @@ async fn run( webrtc_codec).await; match r { Ok(_) => { + log::debug!("answering call. sending join signal"); gossipsub_listener.subscribe_call(call_id, call_info.group_key()); gossipsub_listener.subscribe_webrtc(call_id, own_id.clone()); let topic = ipfs_routes::call_signal_route(&call_id); - - log::debug!("answering call. sending join signal"); - let signal = CallSignal::Join; + let signal = CallSignal::Announce; if let Err(e) = gossipsub_sender .send_signal_aes(call_info.group_key(), signal, topic) { let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { - resend_join = true; let _ = rsp.send(Ok(())); } } @@ -451,13 +457,10 @@ async fn run( let _ = webrtc_controller.remove_media_source(source_id).await; }, Cmd::LeaveCall { call_id } => { - resend_join = false; - let call_id = call_id.unwrap_or(active_call.unwrap_or_default()); - let info = call_data_map.get_call_info(call_id); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + let call_id = call_id.unwrap_or_default(); + if call_data_map.is_active_call(call_id) { call_data_map.leave_call(call_id); let _ = gossipsub_sender.empty_queue(); - let _ = active_call.take(); let _ = webrtc_controller.deinit().await; host_media::reset().await; if let Err(e) = ui_event_ch.send(BlinkEventKind::CallTerminated { call_id }) { @@ -465,7 +468,8 @@ async fn run( } } - match info { + // todo: if someone tries to dial you when you left the call, resend the leave signal + match call_data_map.get_call_info(call_id) { Some(info) => { let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Leave; @@ -481,8 +485,8 @@ async fn run( } }, Cmd::MuteSelf => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { + let call_id = data.info.call_id(); data.state.set_self_muted(true); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Muted; @@ -497,8 +501,8 @@ async fn run( } } Cmd::UnmuteSelf => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { + let call_id = data.info.call_id(); data.state.set_self_muted(false); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Unmuted; @@ -513,8 +517,8 @@ async fn run( } } Cmd::SilenceCall => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { + let call_id = data.info.call_id(); if let Err(e) = host_media::deafen().await { log::error!("{e}"); } @@ -530,8 +534,8 @@ async fn run( } } Cmd::UnsilenceCall => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { + let call_id = data.info.call_id(); if let Err(e) = host_media::undeafen().await { log::error!("{e}"); } @@ -550,22 +554,13 @@ async fn run( let _ = rsp.send(call_data_map.get_pending_calls()); } Cmd::GetActiveCallState { rsp } => { - if active_call.is_none() { - let _ = rsp.send(None); - } else { - let _ = rsp.send(call_data_map.get_call_state(active_call.unwrap_or_default())); - } + let _ = rsp.send(call_data_map.get_active().map(|data| data.get_state())); } Cmd::GetActiveCallInfo { rsp } => { - if active_call.is_none() { - let _ = rsp.send(None); - } else { - let _ = rsp.send(call_data_map.get_call_info(active_call.unwrap_or_default())); - } + let _ = rsp.send(call_data_map.get_active().map(|data| data.get_info())); } Cmd::RecordCall { output_dir, rsp } => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { let info = data.get_info(); match host_media::init_recording(Mp4LoggerConfig { @@ -597,8 +592,7 @@ async fn run( } } Cmd::StopRecording { rsp } => { - let call_id = active_call.unwrap_or_default(); - if let Some(data) = call_data_map.map.get_mut(&call_id) { + if let Some(data) = call_data_map.get_active_mut() { let info = data.get_info(); match host_media::pause_recording() @@ -636,7 +630,7 @@ async fn run( }; match signal { GossipSubSignal::Peer { sender, call_id, signal } => match *signal { - _ if !active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() => { + _ if !call_data_map.is_active_call(call_id) => { log::debug!("received webrtc signal for non-active call"); continue; } @@ -668,28 +662,17 @@ async fn run( log::debug!("received signal from someone who isn't part of the call"); continue; } - signaling::CallSignal::Join => { - // ignore extra join signals + signaling::CallSignal::Announce => { if call_data_map.contains_participant(call_id, &sender) { continue; } call_data_map.add_participant(call_id, &sender); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { - log::debug!("dialing peer"); - if let Err(e) = webrtc_controller.dial(&sender).await { - log::error!("failed to dial peer: {e}"); - continue; - } - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantJoined { call_id, peer_id: sender }) { - log::error!("failed to send ParticipantJoined Event: {e}"); - } - } }, signaling::CallSignal::Leave => { call_data_map.remove_participant(call_id, &sender); let is_call_empty = call_data_map.call_empty(call_id); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { webrtc_controller.hang_up(&sender).await; if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantLeft { call_id, peer_id: sender }) { log::error!("failed to send ParticipantLeft event: {e}"); @@ -705,7 +688,7 @@ async fn run( signaling::CallSignal::Muted => { call_data_map.set_muted(call_id, &sender, true); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { log::error!("failed to send ParticipantMuted event: {e}"); } @@ -714,7 +697,7 @@ async fn run( signaling::CallSignal::Unmuted => { call_data_map.set_muted(call_id, &sender, false); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { log::error!("failed to send ParticipantUnmuted event: {e}"); } @@ -723,7 +706,7 @@ async fn run( signaling::CallSignal::Deafened => { call_data_map.set_deafened(call_id, &sender, true); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { log::error!("failed to send ParticipantDeafened event: {e}"); } @@ -732,7 +715,7 @@ async fn run( signaling::CallSignal::Undeafened => { call_data_map.set_deafened(call_id, &sender, false); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { log::error!("failed to send ParticipantUndeafened event: {e}"); } @@ -740,7 +723,7 @@ async fn run( }, signaling::CallSignal::Recording => { call_data_map.set_recording(call_id, &sender, true); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantRecording { peer_id: sender }) { log::error!("failed to send ParticipantRecording event: {e}"); } @@ -748,7 +731,7 @@ async fn run( } signaling::CallSignal::NotRecording => { call_data_map.set_recording(call_id, &sender, false); - if active_call.as_ref().map(|x| x == &call_id).unwrap_or_default() { + if call_data_map.is_active_call(call_id) { if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantNotRecording { peer_id: sender }) { log::error!("failed to send ParticipantNotRecording event: {e}"); } @@ -780,66 +763,81 @@ async fn run( }; match event { simple_webrtc::events::EmittedEvents::Ice { dest, candidate } => { - let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); - let signal = PeerSignal::Ice(*candidate); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { - log::error!("failed to send signal: {e}"); + if let Some(data) = call_data_map.get_active() { + let topic = ipfs_routes::peer_signal_route(&dest, &data.info.call_id()); + let signal = PeerSignal::Ice(*candidate); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + } else { + log::warn!("received EmittedEvents::Ice without active call"); } }, simple_webrtc::events::EmittedEvents::Connected { peer } => { - let ac = active_call.unwrap_or_default(); - if call_data_map.contains_participant(ac, &peer) { - call_data_map.add_participant(ac, &peer); - } else { - log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); - webrtc_controller.hang_up(&peer).await; - } + if let Some(data) = call_data_map.get_active() { + let call_id = data.info.call_id(); + if call_data_map.contains_participant(call_id, &peer) { + call_data_map.add_participant(call_id, &peer); + } else { + log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); + webrtc_controller.hang_up(&peer).await; + } + } else { + log::warn!("received EmittedEvents::Connected without active call"); + } }, simple_webrtc::events::EmittedEvents::Disconnected { peer } | simple_webrtc::events::EmittedEvents::ConnectionFailed { peer } | simple_webrtc::events::EmittedEvents::ConnectionClosed { peer } => { log::debug!("webrtc: closed, disconnected or connection failed"); - let ac = active_call.unwrap_or_default(); - call_data_map.remove_participant(ac, &peer); + webrtc_controller.hang_up(&peer).await; if let Err(e) = host_media::remove_sink_track(peer.clone()).await { log::error!("failed to send media_track command: {e}"); } - // possibly redundant - webrtc_controller.hang_up(&peer).await; - - if let Some(data) = call_data_map.map.get(&ac) { + if let Some(data) = call_data_map.get_active_mut() { + let call_id = data.info.call_id(); + if data.info.contains_participant(&peer) { + data.state.remove_participant(&peer); + } if data.info.participants().len() == 2 && data.state.participants_joined.len() <= 1 { log::info!("all participants have successfully been disconnected"); - resend_join = false; if let Err(e) = webrtc_controller.deinit().await { log::error!("webrtc deinit failed: {e}"); } //rtp_logger::deinit().await; host_media::reset().await; - let event = BlinkEventKind::CallTerminated { call_id: ac }; + let event = BlinkEventKind::CallTerminated { call_id }; let _ = ui_event_ch.send(event); - gossipsub_listener.unsubscribe_call(ac); - gossipsub_listener.unsubscribe_webrtc(ac); + gossipsub_listener.unsubscribe_call(call_id); + gossipsub_listener.unsubscribe_webrtc(call_id); let _ = gossipsub_sender.empty_queue(); } } }, simple_webrtc::events::EmittedEvents::Sdp { dest, sdp } => { - let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); - let signal = PeerSignal::Sdp(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { - log::error!("failed to send signal: {e}"); + if let Some(data) = call_data_map.get_active() { + let topic = ipfs_routes::peer_signal_route(&dest, &data.info.call_id()); + let signal = PeerSignal::Sdp(*sdp); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + } else { + log::warn!("received EmittedEvents::Sdp without active call"); } }, simple_webrtc::events::EmittedEvents::CallInitiated { dest, sdp } => { - log::debug!("sending dial signal"); - let topic = ipfs_routes::peer_signal_route(&dest, &active_call.unwrap_or_default()); - let signal = PeerSignal::Dial(*sdp); - if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { - log::error!("failed to send signal: {e}"); + if let Some(data) = call_data_map.get_active() { + log::debug!("sending dial signal"); + let topic = ipfs_routes::peer_signal_route(&dest, &data.info.call_id()); + let signal = PeerSignal::Dial(*sdp); + if let Err(e) = gossipsub_sender.send_signal_ecdh(dest, signal, topic) { + log::error!("failed to send signal: {e}"); + } + } else { + log::warn!("dialing without active call"); } }, simple_webrtc::events::EmittedEvents::TrackAdded { peer, track } => { diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index 07631bda4..d6b36511b 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -30,6 +30,7 @@ impl CallData { pub struct CallDataMap { pub own_id: DID, + pub active_call: Option, pub map: HashMap, } @@ -37,6 +38,7 @@ impl CallDataMap { pub fn new(own_id: DID) -> Self { Self { own_id, + active_call: None, map: HashMap::default(), } } @@ -55,6 +57,35 @@ impl CallDataMap { pub fn get_pending_calls(&self) -> Vec { self.map.values().map(|x| x.get_info()).collect() } + + pub fn is_active_call(&self, call_id: Uuid) -> bool { + self.active_call + .as_ref() + .map(|x| x == &call_id) + .unwrap_or_default() + } + + pub fn get_mut(&mut self, call_id: Uuid) -> Option<&mut CallData> { + self.map.get_mut(&call_id) + } + + pub fn get_active_mut(&mut self) -> Option<&mut CallData> { + match self.active_call.clone() { + None => None, + Some(call_id) => self.map.get_mut(&call_id), + } + } + + pub fn get_active(&self) -> Option<&CallData> { + match self.active_call.clone() { + None => None, + Some(call_id) => self.map.get(&call_id), + } + } + + pub fn set_active(&mut self, call_id: Uuid) { + self.active_call.replace(call_id); + } } impl CallDataMap { @@ -97,6 +128,9 @@ impl CallDataMap { } pub fn leave_call(&mut self, call_id: Uuid) { + if self.is_active_call(call_id) { + self.active_call.take(); + } if let Some(data) = self.map.get_mut(&call_id) { data.state.reset_self(); } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index d339f32f9..4c88a55bb 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -30,11 +30,18 @@ enum GossipSubCmd { signal: Vec, topic: String, }, + // if this command fails, it will periodically be resent SendEcdh { dest: DID, signal: Vec, topic: String, }, + // when someone joins a call, they need to periodically announce their presence. + Announce { + group_key: Vec, + signal: Vec, + topic: String, + }, DecodeEcdh { src: DID, data: Vec, @@ -43,6 +50,7 @@ enum GossipSubCmd { GetOwnId { rsp: oneshot::Sender, }, + // drop resending signals (failed ECDH signals and the announce signal) EmptyQueue, } @@ -106,6 +114,21 @@ impl GossipSubSender { Ok(()) } + pub fn announce( + &self, + group_key: Vec, + signal: T, + topic: String, + ) -> anyhow::Result<()> { + let signal = serde_cbor::to_vec(&signal)?; + self.ch.send(GossipSubCmd::Announce { + group_key, + signal, + topic, + })?; + Ok(()) + } + // this one doesn't require access to own_id. it can be decrypted using just the group key. pub async fn decode_signal_aes( &self, @@ -178,15 +201,21 @@ async fn run( } }; + let mut to_announce: Option = None; let mut ecdh_queue: HashMap> = HashMap::new(); - let mut timer = tokio::time::interval_at( + let mut retry_timer = tokio::time::interval_at( Instant::now() + Duration::from_millis(2000), Duration::from_millis(2000), ); + let mut announce_timer = tokio::time::interval_at( + Instant::now() + Duration::from_millis(5000), + Duration::from_millis(5000), + ); + loop { tokio::select! { - _ = timer.tick() => { + _ = retry_timer.tick() => { for (_dest, queue) in ecdh_queue.iter_mut() { while let Some(cmd) = queue.pop_front() { match cmd { @@ -195,7 +224,7 @@ async fn run( Ok(r) => r, Err(e) => { log::error!("failed to encrypt ecdh message: {e}"); - break; + continue; } }; if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { @@ -208,10 +237,30 @@ async fn run( } } } + _ = announce_timer.tick() => { + if let Some(cmd) = to_announce.as_ref() { + match cmd { + GossipSubCmd::Announce { group_key, signal, topic } => { + let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt aes message: {e}"); + continue; + } + }; + if let Err(e) = ipfs.pubsub_publish(topic.clone(), encrypted).await { + log::error!("failed to publish aes message: {e}"); + } + } + _ => {} + } + } + } opt = ch.recv() => match opt { Some(cmd) => match cmd { GossipSubCmd::EmptyQueue => { ecdh_queue.clear(); + to_announce.take(); } GossipSubCmd::GetOwnId { rsp } => { let _ = rsp.send(own_id.clone()); @@ -230,9 +279,32 @@ async fn run( } }, GossipSubCmd::SendEcdh { dest, signal, topic } => { - let queue = ecdh_queue.entry(dest.clone()).or_insert(VecDeque::new()); - queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); + // only add to the queue if sending fails + let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt ecdh message: {e}"); + continue; + } + }; + if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { + let queue = ecdh_queue.entry(dest.clone()).or_insert(VecDeque::new()); + queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); + } } + GossipSubCmd::Announce { group_key, signal, topic } => { + let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt aes message: {e}"); + continue; + } + }; + if let Err(e) = ipfs.pubsub_publish(topic.clone(), encrypted).await { + log::error!("failed to publish aes message: {e}"); + } + to_announce.replace(GossipSubCmd::Announce { group_key, signal, topic }); + }, GossipSubCmd::DecodeEcdh { src, data, rsp } => { let r = || { let bytes = ecdh_decrypt(&own_id, &src, &data)?; diff --git a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs index 2e890b5a6..add236326 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs @@ -42,7 +42,7 @@ pub enum PeerSignal { #[derive(Serialize, Deserialize, Display, Clone)] pub enum CallSignal { #[display(fmt = "Join")] - Join, + Announce, #[display(fmt = "Leave")] Leave, #[display(fmt = "Muted")] diff --git a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs index e0f8633af..a59306648 100644 --- a/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs +++ b/extensions/warp-blink-wrtc/src/simple_webrtc/mod.rs @@ -223,6 +223,10 @@ impl Controller { } } + pub fn is_connected(&self, peer_id: &DID) -> bool { + self.peers.contains_key(peer_id) + } + /// Spawns a MediaWorker which will receive RTP packets and forward them to all peers /// todo: the peers may want to agree on the MimeType pub async fn add_media_source( diff --git a/warp/src/error.rs b/warp/src/error.rs index 3a45a7560..5a9e06ede 100644 --- a/warp/src/error.rs +++ b/warp/src/error.rs @@ -228,6 +228,8 @@ pub enum Error { CallNotFound, #[error("CallNotInProgress")] CallNotInProgress, + #[error("CallAlreadyInProgress")] + CallAlreadyInProgress, #[error("FailedToSendSignal: {_0}")] FailedToSendSignal(String), #[error("Invalid MIME type: {_0}")] From 96469af53e759c2ffd97bb74d025ddd8ecf05549 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:34:14 -0500 Subject: [PATCH 31/42] fix clippy --- .../src/blink_impl/blink_controller.rs | 2 +- .../src/blink_impl/data/mod.rs | 4 +- .../src/blink_impl/gossipsub_sender.rs | 48 ++++++++----------- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index a4efb353d..e8c347b6c 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -284,7 +284,7 @@ async fn run( let peer_str = peer_id.to_string(); if peer_str < own_id_str { log::debug!("dialing peer"); - if let Err(e) = webrtc_controller.dial(&peer_id).await { + if let Err(e) = webrtc_controller.dial(peer_id).await { log::error!("failed to dial peer: {e}"); continue; } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index d6b36511b..8872bbe9a 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -70,14 +70,14 @@ impl CallDataMap { } pub fn get_active_mut(&mut self) -> Option<&mut CallData> { - match self.active_call.clone() { + match self.active_call { None => None, Some(call_id) => self.map.get_mut(&call_id), } } pub fn get_active(&self) -> Option<&CallData> { - match self.active_call.clone() { + match self.active_call { None => None, Some(call_id) => self.map.get(&call_id), } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 4c88a55bb..31cac9de3 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -218,41 +218,33 @@ async fn run( _ = retry_timer.tick() => { for (_dest, queue) in ecdh_queue.iter_mut() { while let Some(cmd) = queue.pop_front() { - match cmd { - GossipSubCmd::SendEcdh { dest, signal, topic } => { - let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { - Ok(r) => r, - Err(e) => { - log::error!("failed to encrypt ecdh message: {e}"); - continue; - } - }; - if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { - queue.push_front(GossipSubCmd::SendEcdh { dest, signal, topic }); - break; + if let GossipSubCmd::SendEcdh { dest, signal, topic } = cmd { + let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt ecdh message: {e}"); + continue; } + }; + if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { + queue.push_front(GossipSubCmd::SendEcdh { dest, signal, topic }); + break; } - _ => {} } } } } _ = announce_timer.tick() => { - if let Some(cmd) = to_announce.as_ref() { - match cmd { - GossipSubCmd::Announce { group_key, signal, topic } => { - let encrypted = match Cipher::direct_encrypt(&signal, &group_key) { - Ok(r) => r, - Err(e) => { - log::error!("failed to encrypt aes message: {e}"); - continue; - } - }; - if let Err(e) = ipfs.pubsub_publish(topic.clone(), encrypted).await { - log::error!("failed to publish aes message: {e}"); - } + if let Some(GossipSubCmd::Announce { group_key, signal, topic }) = to_announce.as_ref() { + let encrypted = match Cipher::direct_encrypt(signal, group_key) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt aes message: {e}"); + continue; } - _ => {} + }; + if let Err(e) = ipfs.pubsub_publish(topic.clone(), encrypted).await { + log::error!("failed to publish aes message: {e}"); } } } @@ -288,7 +280,7 @@ async fn run( } }; if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { - let queue = ecdh_queue.entry(dest.clone()).or_insert(VecDeque::new()); + let queue = ecdh_queue.entry(dest.clone()).or_default(); queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); } } From 7c277cc9fbd3c15807be9b5c304487e379b049db Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:35:38 -0500 Subject: [PATCH 32/42] increase duration of dial timer --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index e8c347b6c..81acf7780 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -264,8 +264,8 @@ async fn run( let mut call_data_map = CallDataMap::new(own_id.clone()); let mut dial_timer = tokio::time::interval_at( - Instant::now() + Duration::from_millis(1000), - Duration::from_millis(1000), + Instant::now() + Duration::from_millis(3000), + Duration::from_millis(3000), ); loop { From 451397cbf01404accbebda0bf164ce63f0a9af5f Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:42:05 -0500 Subject: [PATCH 33/42] fix signalling - use announce --- .../warp-blink-wrtc/src/blink_impl/blink_controller.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 81acf7780..bc9c08a35 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -362,8 +362,7 @@ async fn run( let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce; if let Err(e) = - gossipsub_sender - .send_signal_aes(call_info.group_key(), signal, topic) + gossipsub_sender.announce(call_info.group_key(), signal, topic) { log::error!("failed to send announce signal: {e}"); } @@ -421,14 +420,13 @@ async fn run( webrtc_codec).await; match r { Ok(_) => { - log::debug!("answering call. sending join signal"); + log::debug!("answering call"); gossipsub_listener.subscribe_call(call_id, call_info.group_key()); gossipsub_listener.subscribe_webrtc(call_id, own_id.clone()); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce; if let Err(e) = - gossipsub_sender - .send_signal_aes(call_info.group_key(), signal, topic) + gossipsub_sender.announce(call_info.group_key(), signal, topic) { let _ = rsp.send(Err(Error::FailedToSendSignal(e.to_string()))); } else { From 2416ae8265f2cc57d0974d000e5dd89ce167a9b5 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:47:35 -0500 Subject: [PATCH 34/42] try to fix logic for leaving call --- extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index bc9c08a35..a5f25ebc8 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -455,7 +455,7 @@ async fn run( let _ = webrtc_controller.remove_media_source(source_id).await; }, Cmd::LeaveCall { call_id } => { - let call_id = call_id.unwrap_or_default(); + let call_id = call_id.unwrap_or(call_data_map.active_call.unwrap_or_default()); if call_data_map.is_active_call(call_id) { call_data_map.leave_call(call_id); let _ = gossipsub_sender.empty_queue(); From 54397935fe1a682e0d94a0a05dfa062800613407 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 10:57:19 -0500 Subject: [PATCH 35/42] add logging --- .../src/blink_impl/blink_controller.rs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index a5f25ebc8..8ccf53cb6 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -275,14 +275,27 @@ async fn run( break; }, _ = dial_timer.tick() => { + log::debug!("dial timer: tick"); if let Some(data) = call_data_map.get_active() { let call_id = data.info.call_id(); for (peer_id, _) in data.state.participants_joined.iter() { + if peer_id == own_id { + continue; + } if webrtc_controller.is_connected(peer_id) { continue; } let peer_str = peer_id.to_string(); - if peer_str < own_id_str { + let mut should_dial = false; + for (l, r) in std::iter::zip(peer_str.as_bytes(), own_id_str.as_bytes()) { + if l < r { + should_dial = true; + break; + } else if r > l { + break; + } + } + if should_dial { log::debug!("dialing peer"); if let Err(e) = webrtc_controller.dial(peer_id).await { log::error!("failed to dial peer: {e}"); @@ -358,7 +371,6 @@ async fn run( } } - // todo: what to do if the signal fails to send? only happens if there's a problem with gossipsub sender let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce; if let Err(e) = @@ -661,9 +673,7 @@ async fn run( continue; } signaling::CallSignal::Announce => { - if call_data_map.contains_participant(call_id, &sender) { - continue; - } + log::debug!("received announce from {}", &sender); call_data_map.add_participant(call_id, &sender); }, signaling::CallSignal::Leave => { From 38053c310c18dede88065e3c6351ef9fac20e400 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 11:01:00 -0500 Subject: [PATCH 36/42] remove some logging --- .../warp-blink-wrtc/src/blink_impl/blink_controller.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 8ccf53cb6..2d40b6d58 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -275,7 +275,7 @@ async fn run( break; }, _ = dial_timer.tick() => { - log::debug!("dial timer: tick"); + //log::trace!("dial timer: tick"); if let Some(data) = call_data_map.get_active() { let call_id = data.info.call_id(); for (peer_id, _) in data.state.participants_joined.iter() { @@ -296,7 +296,6 @@ async fn run( } } if should_dial { - log::debug!("dialing peer"); if let Err(e) = webrtc_controller.dial(peer_id).await { log::error!("failed to dial peer: {e}"); continue; @@ -673,7 +672,7 @@ async fn run( continue; } signaling::CallSignal::Announce => { - log::debug!("received announce from {}", &sender); + //log::trace!("received announce from {}", &sender); call_data_map.add_participant(call_id, &sender); }, signaling::CallSignal::Leave => { From 81ffb62da4c6d7cd6bbe128e7ad99670d228cd77 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 15:09:39 -0500 Subject: [PATCH 37/42] added readme --- .../warp-blink-wrtc/src/blink_impl/mod.rs | 6 +++--- .../warp-blink-wrtc/src/blink_impl/readme.md | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 extensions/warp-blink-wrtc/src/blink_impl/readme.md diff --git a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs index 75df5fced..75f0c61e4 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/mod.rs @@ -81,13 +81,13 @@ impl BlinkImpl { // todo: ensure rx doesn't get dropped let (ui_event_ch, _rx) = broadcast::channel(1024); - let (signal_tx, signal_rx) = mpsc::unbounded_channel(); + let (gossipsub_tx, gossipsub_rx) = mpsc::unbounded_channel(); let ipfs = Arc::new(warp::sync::RwLock::new(None)); let own_id_private = Arc::new(warp::sync::RwLock::new(None)); let gossipsub_sender = GossipSubSender::new(own_id_private.clone(), ipfs.clone()); let gossipsub_listener = - GossipSubListener::new(ipfs.clone(), signal_tx, gossipsub_sender.clone()); + GossipSubListener::new(ipfs.clone(), gossipsub_tx, gossipsub_sender.clone()); let webrtc_controller = simple_webrtc::Controller::new()?; let webrtc_event_stream = webrtc_controller.get_event_stream(); @@ -96,7 +96,7 @@ impl BlinkImpl { webrtc_event_stream, gossipsub_sender: gossipsub_sender.clone(), gossipsub_listener: gossipsub_listener.clone(), - signal_rx, + signal_rx: gossipsub_rx, ui_event_ch: ui_event_ch.clone(), }); diff --git a/extensions/warp-blink-wrtc/src/blink_impl/readme.md b/extensions/warp-blink-wrtc/src/blink_impl/readme.md new file mode 100644 index 000000000..8f12e73ed --- /dev/null +++ b/extensions/warp-blink-wrtc/src/blink_impl/readme.md @@ -0,0 +1,19 @@ +### BlinkImpl spawns three long running tasks +- `GossipSubListener`: receives messages via IPFS and forwards them to the `BlinkController` +- `GossipSubSender`: contains the user's full DID - both the public and private key - and is responsible for sending and decoding messages. GossipSubSender also can provide a clone of the DID (which returns just the public key) upon request. +- `BlinkController`: contains the instance of `SimpleWebrtc`. receives all the gossipsub messages, webrtc events, and user commands (invoked by BlinkImpl) + +### when BlinkImpl offers a call +- an Offer signal is sent (and retried) +- the sender subscribes to a gossip channel specific to that call +- all recipients subscribe to that gossip channel too, even if they don't join the call (this allows them to detect when all the other participants have left the call - in this case the call would be considered terminated). +- The sender automatically joins the call. + +### when someone joins the call +- they subscribe to another gossip channel, using the call id and their DID. This one is for receiving webrtc specific signals (SDP and ICE mostly). +- they broadcast an `Announce` signal on the call-wide channel periodically. +- they track all the `Announce` and `Leave` signals. +- they periodically go through a list of all participants who joined the call but to whom they aren't yet connected. they compare DIDs and based off of that, one of the peers will initiate a webrtc connection via the `Dial` signal. + +### in response to a dial signal +- the other side automatically accepts and proceeds with the webrtc connection process. \ No newline at end of file From 615d30f4cb604abdf1593aed5d2000b920767a00 Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 16:29:37 -0500 Subject: [PATCH 38/42] fix(signaling): send participant state in the announce event --- .../src/blink_impl/blink_controller.rs | 129 ++++++------------ .../src/blink_impl/data/mod.rs | 27 +++- .../src/blink_impl/signaling.rs | 21 +-- warp/src/blink/call_state.rs | 12 +- warp/src/blink/mod.rs | 17 +-- 5 files changed, 78 insertions(+), 128 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 2d40b6d58..89bd04495 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -16,7 +16,7 @@ use tokio::{ }; use uuid::Uuid; use warp::{ - blink::{BlinkEventKind, CallInfo, CallState}, + blink::{BlinkEventKind, CallInfo, CallState, ParticipantState}, error::Error, }; use webrtc::{ @@ -370,8 +370,9 @@ async fn run( } } + let own_state = call_data_map.get_own_state().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Announce; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = gossipsub_sender.announce(call_info.group_key(), signal, topic) { @@ -434,8 +435,10 @@ async fn run( log::debug!("answering call"); gossipsub_listener.subscribe_call(call_id, call_info.group_key()); gossipsub_listener.subscribe_webrtc(call_id, own_id.clone()); + + let own_state = call_data_map.get_own_state().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Announce; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = gossipsub_sender.announce(call_info.group_key(), signal, topic) { @@ -497,15 +500,13 @@ async fn run( if let Some(data) = call_data_map.get_active_mut() { let call_id = data.info.call_id(); data.state.set_self_muted(true); + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Muted; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to mute self"); + log::error!("failed to send announce signal: {e}"); } } } @@ -513,15 +514,13 @@ async fn run( if let Some(data) = call_data_map.get_active_mut() { let call_id = data.info.call_id(); data.state.set_self_muted(false); + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Unmuted; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); - } else { - log::debug!("sent signal to unmute self"); + log::error!("failed to send announce signal: {e}"); } } } @@ -532,13 +531,13 @@ async fn run( log::error!("{e}"); } data.state.set_deafened(own_id, true); + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Deafened; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); + log::error!("failed to send announce signal: {e}"); } } } @@ -549,13 +548,13 @@ async fn run( log::error!("{e}"); } data.state.set_deafened(own_id, false); + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); - let signal = CallSignal::Undeafened; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(data.info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); + log::error!("failed to send announce signal: {e}"); } } } @@ -582,13 +581,13 @@ async fn run( { Ok(_) => { data.state.set_self_recording(true); + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&info.call_id()); - let signal = CallSignal::Recording; + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); + log::error!("failed to send announce signal: {e}"); } let _ = rsp.send(Ok(())); } @@ -602,20 +601,19 @@ async fn run( } Cmd::StopRecording { rsp } => { if let Some(data) = call_data_map.get_active_mut() { - let info = data.get_info(); match host_media::pause_recording() .await { Ok(_) => { data.state.set_self_recording(false); - let topic = ipfs_routes::call_signal_route(&info.call_id()); - let signal = CallSignal::NotRecording; + let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let topic = ipfs_routes::call_signal_route(&data.info.call_id()); + let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = - gossipsub_sender - .send_signal_aes(info.group_key(), signal, topic) + gossipsub_sender.announce(data.info.group_key(), signal, topic) { - log::error!("failed to send signal: {e}"); + log::error!("failed to send announce signal: {e}"); } let _ = rsp.send(Ok(())); } @@ -671,9 +669,14 @@ async fn run( log::debug!("received signal from someone who isn't part of the call"); continue; } - signaling::CallSignal::Announce => { + signaling::CallSignal::Announce { participant_state } => { //log::trace!("received announce from {}", &sender); - call_data_map.add_participant(call_id, &sender); + let prev_state = call_data_map.get_participant_state(call_id, &sender); + let state_changed = prev_state.as_ref().map(|x| x != &participant_state).unwrap_or(true); + call_data_map.add_participant(call_id, &sender, participant_state.clone()); + if state_changed { + let _ = ui_event_ch.send(BlinkEventKind::ParticipantStateChanged { peer_id: sender, state: participant_state }); + } }, signaling::CallSignal::Leave => { call_data_map.remove_participant(call_id, &sender); @@ -692,58 +695,6 @@ async fn run( } } }, - signaling::CallSignal::Muted => { - call_data_map.set_muted(call_id, &sender, true); - - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantMuted { peer_id: sender }) { - log::error!("failed to send ParticipantMuted event: {e}"); - } - } - }, - signaling::CallSignal::Unmuted => { - call_data_map.set_muted(call_id, &sender, false); - - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUnmuted { peer_id: sender }) { - log::error!("failed to send ParticipantUnmuted event: {e}"); - } - } - }, - signaling::CallSignal::Deafened => { - call_data_map.set_deafened(call_id, &sender, true); - - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantDeafened { peer_id: sender }) { - log::error!("failed to send ParticipantDeafened event: {e}"); - } - } - }, - signaling::CallSignal::Undeafened => { - call_data_map.set_deafened(call_id, &sender, false); - - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantUndeafened { peer_id: sender }) { - log::error!("failed to send ParticipantUndeafened event: {e}"); - } - } - }, - signaling::CallSignal::Recording => { - call_data_map.set_recording(call_id, &sender, true); - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantRecording { peer_id: sender }) { - log::error!("failed to send ParticipantRecording event: {e}"); - } - } - } - signaling::CallSignal::NotRecording => { - call_data_map.set_recording(call_id, &sender, false); - if call_data_map.is_active_call(call_id) { - if let Err(e) = ui_event_ch.send(BlinkEventKind::ParticipantNotRecording { peer_id: sender }) { - log::error!("failed to send ParticipantNotRecording event: {e}"); - } - } - } }, GossipSubSignal::Initiation { sender, signal } => match signal { signaling::InitiationSignal::Offer { call_info } => { @@ -783,9 +734,7 @@ async fn run( simple_webrtc::events::EmittedEvents::Connected { peer } => { if let Some(data) = call_data_map.get_active() { let call_id = data.info.call_id(); - if call_data_map.contains_participant(call_id, &peer) { - call_data_map.add_participant(call_id, &peer); - } else { + if !call_data_map.contains_participant(call_id, &peer) { log::warn!("webrtc controller connected to a peer who wasn't in the list for the active call"); webrtc_controller.hang_up(&peer).await; } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs index 8872bbe9a..566c4fd7b 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/data/mod.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use uuid::Uuid; use warp::{ - blink::{CallInfo, CallState}, + blink::{CallInfo, CallState, ParticipantState}, crypto::DID, }; @@ -50,7 +50,7 @@ impl CallDataMap { } let mut state = CallState::new(self.own_id.clone()); - state.add_participant(sender); + state.add_participant(sender, ParticipantState::default()); self.map.insert(call_id, CallData::new(info, state)); } @@ -89,10 +89,15 @@ impl CallDataMap { } impl CallDataMap { - pub fn add_participant(&mut self, call_id: Uuid, peer_id: &DID) { + pub fn add_participant( + &mut self, + call_id: Uuid, + peer_id: &DID, + participant_state: ParticipantState, + ) { if let Some(data) = self.map.get_mut(&call_id) { if data.info.contains_participant(peer_id) { - data.state.add_participant(peer_id); + data.state.add_participant(peer_id, participant_state); } } } @@ -119,6 +124,20 @@ impl CallDataMap { self.map.get(&id).map(|x| x.get_state()) } + pub fn get_own_state(&self) -> Option { + self.get_active().cloned().and_then(|data| { + data.get_state() + .participants_joined + .get(&self.own_id) + .cloned() + }) + } + + pub fn get_participant_state(&self, call_id: Uuid, peer_id: &DID) -> Option { + self.get_call_state(call_id) + .and_then(|state| state.participants_joined.get(peer_id).cloned()) + } + pub fn insert(&mut self, id: Uuid, data: CallData) { self.map.insert(id, data); } diff --git a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs index add236326..48cabc7ec 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/signaling.rs @@ -2,7 +2,10 @@ use derive_more::Display; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use warp::{blink::CallInfo, crypto::DID}; +use warp::{ + blink::{CallInfo, ParticipantState}, + crypto::DID, +}; use webrtc::{ ice_transport::ice_candidate::RTCIceCandidate, peer_connection::sdp::session_description::RTCSessionDescription, @@ -41,22 +44,10 @@ pub enum PeerSignal { // it is somewhat redundant but for now i'll leave it in. #[derive(Serialize, Deserialize, Display, Clone)] pub enum CallSignal { - #[display(fmt = "Join")] - Announce, + #[display(fmt = "Announce")] + Announce { participant_state: ParticipantState }, #[display(fmt = "Leave")] Leave, - #[display(fmt = "Muted")] - Muted, - #[display(fmt = "Unmuted")] - Unmuted, - #[display(fmt = "Deafened")] - Deafened, - #[display(fmt = "Undeafened")] - Undeafened, - #[display(fmt = "Recording")] - Recording, - #[display(fmt = "NotRecording")] - NotRecording, } #[derive(Serialize, Deserialize, Display, Clone)] diff --git a/warp/src/blink/call_state.rs b/warp/src/blink/call_state.rs index 349584df2..1b160d7ce 100644 --- a/warp/src/blink/call_state.rs +++ b/warp/src/blink/call_state.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use serde::{Deserialize, Serialize}; + use crate::crypto::DID; #[derive(Debug, Clone)] @@ -8,7 +10,7 @@ pub struct CallState { pub participants_joined: HashMap, } -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] pub struct ParticipantState { pub muted: bool, pub deafened: bool, @@ -22,12 +24,8 @@ impl CallState { participants_joined: HashMap::default(), } } - pub fn add_participant(&mut self, id: &DID) { - if self.participants_joined.contains_key(id) { - return; - } - self.participants_joined - .insert(id.clone(), ParticipantState::default()); + pub fn add_participant(&mut self, id: &DID, state: ParticipantState) { + self.participants_joined.insert(id.clone(), state); } pub fn is_call_empty(&self) -> bool { diff --git a/warp/src/blink/mod.rs b/warp/src/blink/mod.rs index 11fd821b2..5ee3a147d 100644 --- a/warp/src/blink/mod.rs +++ b/warp/src/blink/mod.rs @@ -133,18 +133,11 @@ pub enum BlinkEventKind { ParticipantSpeaking { peer_id: DID }, #[display(fmt = "SelfSpeaking")] SelfSpeaking, - #[display(fmt = "ParticipantMuted")] - ParticipantMuted { peer_id: DID }, - #[display(fmt = "ParticipantUnmuted")] - ParticipantUnmuted { peer_id: DID }, - #[display(fmt = "ParticipantDeafened")] - ParticipantDeafened { peer_id: DID }, - #[display(fmt = "ParticipantUndeafened")] - ParticipantUndeafened { peer_id: DID }, - #[display(fmt = "ParticipantRecording")] - ParticipantRecording { peer_id: DID }, - #[display(fmt = "ParticipantNotRecording")] - ParticipantNotRecording { peer_id: DID }, + #[display(fmt = "ParticipantStateChanged")] + ParticipantStateChanged { + peer_id: DID, + state: ParticipantState, + }, /// audio packets were dropped for the peer #[display(fmt = "AudioDegradation")] AudioDegradation { peer_id: DID }, From 0037af75a37a1fc7387967cb25e5e88f3a22cb9e Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 16:30:39 -0500 Subject: [PATCH 39/42] fix clippy --- .../src/blink_impl/blink_controller.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index 89bd04495..c59d98b1d 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -16,7 +16,7 @@ use tokio::{ }; use uuid::Uuid; use warp::{ - blink::{BlinkEventKind, CallInfo, CallState, ParticipantState}, + blink::{BlinkEventKind, CallInfo, CallState}, error::Error, }; use webrtc::{ @@ -500,7 +500,7 @@ async fn run( if let Some(data) = call_data_map.get_active_mut() { let call_id = data.info.call_id(); data.state.set_self_muted(true); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = @@ -514,7 +514,7 @@ async fn run( if let Some(data) = call_data_map.get_active_mut() { let call_id = data.info.call_id(); data.state.set_self_muted(false); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = @@ -531,7 +531,7 @@ async fn run( log::error!("{e}"); } data.state.set_deafened(own_id, true); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = @@ -548,7 +548,7 @@ async fn run( log::error!("{e}"); } data.state.set_deafened(own_id, false); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&call_id); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = @@ -581,7 +581,7 @@ async fn run( { Ok(_) => { data.state.set_self_recording(true); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&info.call_id()); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = @@ -607,7 +607,7 @@ async fn run( { Ok(_) => { data.state.set_self_recording(false); - let own_state = data.state.participants_joined.get(&own_id).cloned().unwrap_or_default(); + let own_state = data.state.participants_joined.get(own_id).cloned().unwrap_or_default(); let topic = ipfs_routes::call_signal_route(&data.info.call_id()); let signal = CallSignal::Announce { participant_state: own_state }; if let Err(e) = From c93703a66ba83ed0317ce573033e7eae47933c5b Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 16:32:53 -0500 Subject: [PATCH 40/42] fix clippy --- .../src/blink_impl/blink_controller.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs index c59d98b1d..d146cb6ac 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/blink_controller.rs @@ -5,7 +5,7 @@ use super::signaling::{ self, ipfs_routes, CallSignal, GossipSubSignal, InitiationSignal, PeerSignal, }; -use std::{sync::Arc, time::Duration}; +use std::{cmp, sync::Arc, time::Duration}; use tokio::{ sync::{ broadcast, @@ -288,11 +288,15 @@ async fn run( let peer_str = peer_id.to_string(); let mut should_dial = false; for (l, r) in std::iter::zip(peer_str.as_bytes(), own_id_str.as_bytes()) { - if l < r { - should_dial = true; - break; - } else if r > l { - break; + match l.cmp(r) { + cmp::Ordering::Less => { + should_dial = true; + break; + } + cmp::Ordering::Greater => { + break; + } + _ => {} } } if should_dial { From 864cb8959a0d83adec77782364244aabdbf1425b Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 16:38:33 -0500 Subject: [PATCH 41/42] reset announce timer in response to GossipSubCmd::Announce --- extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 31cac9de3..55eb2daa7 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -296,6 +296,7 @@ async fn run( log::error!("failed to publish aes message: {e}"); } to_announce.replace(GossipSubCmd::Announce { group_key, signal, topic }); + announce_timer.reset(); }, GossipSubCmd::DecodeEcdh { src, data, rsp } => { let r = || { From 74221dfb3fef44b1ad3ab2d0d4c13ae139a1a88e Mon Sep 17 00:00:00 2001 From: Stuart Woodbury Date: Thu, 9 Nov 2023 16:39:13 -0500 Subject: [PATCH 42/42] fix fmt --- .../src/blink_impl/gossipsub_sender.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs index 55eb2daa7..adf66fdf6 100644 --- a/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs +++ b/extensions/warp-blink-wrtc/src/blink_impl/gossipsub_sender.rs @@ -272,17 +272,17 @@ async fn run( }, GossipSubCmd::SendEcdh { dest, signal, topic } => { // only add to the queue if sending fails - let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { - Ok(r) => r, - Err(e) => { - log::error!("failed to encrypt ecdh message: {e}"); - continue; - } - }; - if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { - let queue = ecdh_queue.entry(dest.clone()).or_default(); - queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); + let encrypted = match ecdh_encrypt(&own_id, &dest, signal.clone()) { + Ok(r) => r, + Err(e) => { + log::error!("failed to encrypt ecdh message: {e}"); + continue; } + }; + if ipfs.pubsub_publish(topic.clone(), encrypted).await.is_err() { + let queue = ecdh_queue.entry(dest.clone()).or_default(); + queue.push_back(GossipSubCmd::SendEcdh { dest, signal, topic }); + } } GossipSubCmd::Announce { group_key, signal, topic } => { let encrypted = match Cipher::direct_encrypt(&signal, &group_key) {