From 6f5e08d60ddb0d2fd6c6fff297f6e1488e2efdda Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Wed, 27 Nov 2024 10:41:23 +0000 Subject: [PATCH 1/2] More cleanup of unused code --- Cargo.lock | 2 - .../libraries/chat_events/src/chat_events.rs | 7 ---- .../chat_events/src/chat_events_list.rs | 41 +++++++------------ .../libraries/chat_events/src/hybrid_map.rs | 8 ---- backend/libraries/group_chat_core/Cargo.toml | 2 - backend/libraries/group_chat_core/src/lib.rs | 6 +-- .../libraries/group_chat_core/src/members.rs | 20 +++------ 7 files changed, 21 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05c1ae655f..999b27f7c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2881,7 +2881,6 @@ dependencies = [ "chat_events", "event_store_producer", "group_community_common", - "ic-cdk 0.16.0", "itertools 0.13.0", "lazy_static", "msgpack", @@ -2891,7 +2890,6 @@ dependencies = [ "serde", "serde_repr", "test-strategy", - "tracing", "types", "utils 0.1.0", ] diff --git a/backend/libraries/chat_events/src/chat_events.rs b/backend/libraries/chat_events/src/chat_events.rs index edb9e2807b..1f61486aa1 100644 --- a/backend/libraries/chat_events/src/chat_events.rs +++ b/backend/libraries/chat_events/src/chat_events.rs @@ -77,13 +77,6 @@ impl ChatEvents { false } - pub fn init_maps(&mut self) { - self.main.init_hybrid_map(self.chat, None); - for (message_index, thread) in self.threads.iter_mut() { - thread.init_hybrid_map(self.chat, Some(*message_index)); - } - } - pub fn import_events(chat: Chat, events: Vec<(EventContext, ByteBuf)>) { stable_storage::write_events_as_bytes(chat, events); } diff --git a/backend/libraries/chat_events/src/chat_events_list.rs b/backend/libraries/chat_events/src/chat_events_list.rs index d7fa06ef68..cbd688729f 100644 --- a/backend/libraries/chat_events/src/chat_events_list.rs +++ b/backend/libraries/chat_events/src/chat_events_list.rs @@ -2,7 +2,6 @@ use crate::hybrid_map::HybridMap; use crate::last_updated_timestamps::LastUpdatedTimestamps; use crate::stable_storage::ChatEventsStableStorage; use crate::{ChatEventInternal, EventKey, EventOrExpiredRangeInternal, EventsMap, MessageInternal}; -use candid::Principal; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::hash_map::Entry::Vacant; @@ -15,41 +14,29 @@ use types::{ #[derive(Serialize, Deserialize)] pub struct ChatEventsList { - #[serde(skip_deserializing, default = "default_stable_events_map")] - stable_events_map: HybridMap, + events_map: HybridMap, message_id_map: HashMap, message_event_indexes: Vec, latest_event_index: Option, latest_event_timestamp: Option, } -fn default_stable_events_map() -> HybridMap { - HybridMap::new(Chat::Group(Principal::anonymous().into()), None) -} - impl ChatEventsList { - pub fn init_hybrid_map(&mut self, chat: Chat, thread_root_message_index: Option) { - self.stable_events_map = HybridMap::new(chat, thread_root_message_index); - self.stable_events_map - .populate_fast_map(self.latest_event_index.unwrap_or_default()); - } - - pub fn update_event_in_stable_memory(&mut self, event_key: EventKey) { + pub fn update_event_in_memory(&mut self, event_key: EventKey) { if let Some(event_index) = self.event_index(event_key) { - if let Some(event) = self.stable_events_map.get(event_index) { - self.stable_events_map.insert(event); + if let Some(event) = self.events_map.get(event_index) { + self.events_map.insert(event); } } } pub fn set_stable_memory_prefix(&mut self, chat: Chat, thread_root_message_index: Option) { - self.stable_events_map - .set_stable_memory_prefix(chat, thread_root_message_index); + self.events_map.set_stable_memory_prefix(chat, thread_root_message_index); } pub fn new(chat: Chat, thread_root_message_index: Option) -> Self { ChatEventsList { - stable_events_map: HybridMap::new(chat, thread_root_message_index), + events_map: HybridMap::new(chat, thread_root_message_index), message_id_map: HashMap::new(), message_event_indexes: Vec::new(), latest_event_index: None, @@ -81,7 +68,7 @@ impl ChatEventsList { expires_at, event, }; - self.stable_events_map.insert(event_wrapper); + self.events_map.insert(event_wrapper); self.latest_event_index = Some(event_index); self.latest_event_timestamp = Some(now); @@ -121,7 +108,7 @@ impl ChatEventsList { if let Some(mut event) = self.get_event(event_key, EventIndex::default()) { update_event_fn(&mut event).map(|result| { let event_index = event.index; - self.stable_events_map.insert(event); + self.events_map.insert(event); (result, event_index) }) } else { @@ -153,7 +140,7 @@ impl ChatEventsList { (min_visible_event_index, self.latest_event_index.unwrap_or_default()) }; - let iter = self.stable_events_map.range(min..=max); + let iter = self.events_map.range(min..=max); if ascending { Box::new(ChatEventsListIterator { @@ -206,7 +193,7 @@ impl ChatEventsList { } pub(crate) fn event_count_since bool>(&self, since: TimestampMillis, filter: &F) -> usize { - self.stable_events_map + self.events_map .iter() .rev() .take_while(|e| e.timestamp > since) @@ -215,7 +202,7 @@ impl ChatEventsList { } pub fn remove(&mut self, event_index: EventIndex) -> Option> { - self.stable_events_map.remove(event_index) + self.events_map.remove(event_index) } pub fn latest_event_index(&self) -> Option { @@ -243,7 +230,7 @@ impl ChatEventsList { } pub fn last(&self) -> Option> { - self.stable_events_map.iter().next_back() + self.events_map.iter().next_back() } pub fn contains_message_id(&self, message_id: MessageId) -> bool { @@ -262,12 +249,12 @@ impl ChatEventsList { &self, event_index: EventIndex, ) -> Result, (Option, Option)> { - let next_key = match self.stable_events_map.range(event_index..).next() { + let next_key = match self.events_map.range(event_index..).next() { Some(v) if v.index == event_index => return Ok(v), Some(v) => Some(v.index), None => None, }; - let previous_key = self.stable_events_map.range(..event_index).next_back().map(|e| e.index); + let previous_key = self.events_map.range(..event_index).next_back().map(|e| e.index); Err((previous_key, next_key)) } diff --git a/backend/libraries/chat_events/src/hybrid_map.rs b/backend/libraries/chat_events/src/hybrid_map.rs index cefd008693..179e64857d 100644 --- a/backend/libraries/chat_events/src/hybrid_map.rs +++ b/backend/libraries/chat_events/src/hybrid_map.rs @@ -26,14 +26,6 @@ pub struct HybridMap { } impl HybridMap { - // TODO: Remove this once everything is migrated over - pub fn populate_fast_map(&mut self, latest_event_index: EventIndex) { - for event in self.slow.iter().rev().take(self.max_events_in_fast_map as usize) { - self.fast.insert(event.index, event); - } - self.latest_event_index = latest_event_index; - } - fn fast_enabled(&self) -> bool { self.max_events_in_fast_map > 0 } diff --git a/backend/libraries/group_chat_core/Cargo.toml b/backend/libraries/group_chat_core/Cargo.toml index 23faddf416..a7cbb31d29 100644 --- a/backend/libraries/group_chat_core/Cargo.toml +++ b/backend/libraries/group_chat_core/Cargo.toml @@ -10,14 +10,12 @@ candid = { workspace = true } chat_events = { path = "../chat_events" } event_store_producer = { workspace = true } group_community_common = { path = "../group_community_common" } -ic-cdk = { workspace = true } itertools = { workspace = true } lazy_static = { workspace = true } regex-lite = { workspace = true } search = { path = "../search" } serde = { workspace = true } serde_repr = { workspace = true } -tracing = { workspace = true } types = { path = "../types" } utils = { path = "../utils" } diff --git a/backend/libraries/group_chat_core/src/lib.rs b/backend/libraries/group_chat_core/src/lib.rs index f00d07f301..fed6d84513 100644 --- a/backend/libraries/group_chat_core/src/lib.rs +++ b/backend/libraries/group_chat_core/src/lib.rs @@ -10,7 +10,7 @@ use regex_lite::Regex; use search::Query; use serde::{Deserialize, Serialize}; use std::cmp::{max, min, Reverse}; -use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use types::{ AccessGate, AccessGateConfig, AccessGateConfigInternal, AvatarChanged, ContentValidationError, CustomPermission, Document, EventIndex, EventOrExpiredRange, EventWrapper, EventsResponse, ExternalUrlUpdated, FieldTooLongResult, FieldTooShortResult, @@ -61,10 +61,7 @@ pub struct GroupChatCore { pub invited_users: InvitedUsers, pub min_visible_indexes_for_new_members: Option<(EventIndex, MessageIndex)>, pub external_url: Timestamped>, - #[serde(default)] at_everyone_mentions: BTreeMap, - #[serde(default)] - pub dedupe_at_everyone_mentions_queue: VecDeque, } #[allow(clippy::too_many_arguments)] @@ -121,7 +118,6 @@ impl GroupChatCore { min_visible_indexes_for_new_members: None, external_url: Timestamped::new(external_url, now), at_everyone_mentions: BTreeMap::new(), - dedupe_at_everyone_mentions_queue: VecDeque::new(), } } diff --git a/backend/libraries/group_chat_core/src/members.rs b/backend/libraries/group_chat_core/src/members.rs index bf1cb7df91..8506b0dc83 100644 --- a/backend/libraries/group_chat_core/src/members.rs +++ b/backend/libraries/group_chat_core/src/members.rs @@ -7,7 +7,7 @@ use serde::ser::SerializeSeq; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_repr::{Deserialize_repr, Serialize_repr}; use std::cmp::max; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Formatter; use types::{ is_default, EventIndex, GroupMember, GroupPermissions, MessageIndex, TimestampMillis, Timestamped, UserId, UserType, @@ -23,7 +23,7 @@ const MAX_MEMBERS_PER_GROUP: u32 = 100_000; #[derive(Serialize, Deserialize, Default)] pub struct GroupMembers { #[serde(serialize_with = "serialize_members", deserialize_with = "deserialize_members")] - members: HashMap, + members: BTreeMap, member_ids: BTreeSet, owners: BTreeSet, admins: BTreeSet, @@ -185,14 +185,6 @@ impl GroupMembers { &self.member_ids } - pub fn iter(&self) -> impl Iterator { - self.members.values() - } - - pub fn iter_mut(&mut self) -> impl Iterator { - self.members.values_mut() - } - pub fn get(&self, user_id: &UserId) -> Option<&GroupMemberInternal> { self.members.get(user_id) } @@ -662,7 +654,7 @@ fn mentions_are_empty(value: &Mentions) -> bool { value.is_empty() } -fn serialize_members(value: &HashMap, serializer: S) -> Result { +fn serialize_members(value: &BTreeMap, serializer: S) -> Result { let mut seq = serializer.serialize_seq(Some(value.len()))?; for member in value.values() { seq.serialize_element(member)?; @@ -670,14 +662,14 @@ fn serialize_members(value: &HashMap seq.end() } -fn deserialize_members<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { +fn deserialize_members<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { deserializer.deserialize_seq(GroupMembersMapVisitor) } struct GroupMembersMapVisitor; impl<'de> Visitor<'de> for GroupMembersMapVisitor { - type Value = HashMap; + type Value = BTreeMap; fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { formatter.write_str("a sequence") @@ -687,7 +679,7 @@ impl<'de> Visitor<'de> for GroupMembersMapVisitor { where A: SeqAccess<'de>, { - let mut map = seq.size_hint().map_or_else(HashMap::new, HashMap::with_capacity); + let mut map = BTreeMap::new(); while let Some(next) = seq.next_element::()? { map.insert(next.user_id, next); } From c74dbceb301180d8b34729f6371d96926adb5807 Mon Sep 17 00:00:00 2001 From: Hamish Peebles Date: Wed, 27 Nov 2024 10:45:11 +0000 Subject: [PATCH 2/2] Rename --- backend/libraries/chat_events/src/chat_events.rs | 8 ++++---- backend/libraries/chat_events/src/chat_events_list.rs | 2 +- backend/libraries/chat_events/src/hybrid_map.rs | 2 +- backend/libraries/chat_events/src/lib.rs | 4 ++-- .../src/{stable_storage => stable_memory}/key.rs | 0 .../src/{stable_storage => stable_memory}/mod.rs | 2 +- .../src/{stable_storage => stable_memory}/tests.rs | 4 ++-- .../tests/test_values.rs | 0 8 files changed, 11 insertions(+), 11 deletions(-) rename backend/libraries/chat_events/src/{stable_storage => stable_memory}/key.rs (100%) rename backend/libraries/chat_events/src/{stable_storage => stable_memory}/mod.rs (99%) rename backend/libraries/chat_events/src/{stable_storage => stable_memory}/tests.rs (99%) rename backend/libraries/chat_events/src/{stable_storage => stable_memory}/tests/test_values.rs (100%) diff --git a/backend/libraries/chat_events/src/chat_events.rs b/backend/libraries/chat_events/src/chat_events.rs index 1f61486aa1..0ba88c717c 100644 --- a/backend/libraries/chat_events/src/chat_events.rs +++ b/backend/libraries/chat_events/src/chat_events.rs @@ -3,7 +3,7 @@ use crate::expiring_events::ExpiringEvents; use crate::last_updated_timestamps::LastUpdatedTimestamps; use crate::metrics::{ChatMetricsInternal, MetricKey}; use crate::search_index::SearchIndex; -use crate::stable_storage::key::KeyPrefix; +use crate::stable_memory::key::KeyPrefix; use crate::*; use event_store_producer::{EventBuilder, EventStoreClient, Runtime}; use rand::rngs::StdRng; @@ -78,11 +78,11 @@ impl ChatEvents { } pub fn import_events(chat: Chat, events: Vec<(EventContext, ByteBuf)>) { - stable_storage::write_events_as_bytes(chat, events); + stable_memory::write_events_as_bytes(chat, events); } pub fn garbage_collect_stable_memory(prefix: KeyPrefix) -> Result { - stable_storage::garbage_collect(prefix) + stable_memory::garbage_collect(prefix) } pub fn set_stable_memory_key_prefixes(&mut self) { @@ -167,7 +167,7 @@ impl ChatEvents { } pub fn read_events_as_bytes_from_stable_memory(&self, after: Option) -> Vec<(EventContext, ByteBuf)> { - stable_storage::read_events_as_bytes(self.chat, after, 1_000_000) + stable_memory::read_events_as_bytes(self.chat, after, 1_000_000) } pub fn iter_recently_updated_events( diff --git a/backend/libraries/chat_events/src/chat_events_list.rs b/backend/libraries/chat_events/src/chat_events_list.rs index cbd688729f..3debaa0416 100644 --- a/backend/libraries/chat_events/src/chat_events_list.rs +++ b/backend/libraries/chat_events/src/chat_events_list.rs @@ -1,6 +1,6 @@ use crate::hybrid_map::HybridMap; use crate::last_updated_timestamps::LastUpdatedTimestamps; -use crate::stable_storage::ChatEventsStableStorage; +use crate::stable_memory::ChatEventsStableStorage; use crate::{ChatEventInternal, EventKey, EventOrExpiredRangeInternal, EventsMap, MessageInternal}; use itertools::Itertools; use serde::{Deserialize, Serialize}; diff --git a/backend/libraries/chat_events/src/hybrid_map.rs b/backend/libraries/chat_events/src/hybrid_map.rs index 179e64857d..ee1b1aa29c 100644 --- a/backend/libraries/chat_events/src/hybrid_map.rs +++ b/backend/libraries/chat_events/src/hybrid_map.rs @@ -1,4 +1,4 @@ -use crate::stable_storage::ChatEventsStableStorage; +use crate::stable_memory::ChatEventsStableStorage; use crate::{ChatEventInternal, EventsMap}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/backend/libraries/chat_events/src/lib.rs b/backend/libraries/chat_events/src/lib.rs index ba54654a81..a4cf2b48ca 100644 --- a/backend/libraries/chat_events/src/lib.rs +++ b/backend/libraries/chat_events/src/lib.rs @@ -9,7 +9,7 @@ mod last_updated_timestamps; mod message_content_internal; mod metrics; mod search_index; -mod stable_storage; +mod stable_memory; pub use crate::chat_event_internal::*; pub use crate::chat_events::*; @@ -17,4 +17,4 @@ pub use crate::chat_events_list::*; pub use crate::events_map::*; pub use crate::message_content_internal::*; pub use crate::metrics::*; -pub use crate::stable_storage::key::*; +pub use crate::stable_memory::key::*; diff --git a/backend/libraries/chat_events/src/stable_storage/key.rs b/backend/libraries/chat_events/src/stable_memory/key.rs similarity index 100% rename from backend/libraries/chat_events/src/stable_storage/key.rs rename to backend/libraries/chat_events/src/stable_memory/key.rs diff --git a/backend/libraries/chat_events/src/stable_storage/mod.rs b/backend/libraries/chat_events/src/stable_memory/mod.rs similarity index 99% rename from backend/libraries/chat_events/src/stable_storage/mod.rs rename to backend/libraries/chat_events/src/stable_memory/mod.rs index 82d69999f2..70fd1bc698 100644 --- a/backend/libraries/chat_events/src/stable_storage/mod.rs +++ b/backend/libraries/chat_events/src/stable_memory/mod.rs @@ -1,4 +1,4 @@ -use crate::stable_storage::key::{Key, KeyPrefix}; +use crate::stable_memory::key::{Key, KeyPrefix}; use crate::{ChatEventInternal, EventsMap}; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; diff --git a/backend/libraries/chat_events/src/stable_storage/tests.rs b/backend/libraries/chat_events/src/stable_memory/tests.rs similarity index 99% rename from backend/libraries/chat_events/src/stable_storage/tests.rs rename to backend/libraries/chat_events/src/stable_memory/tests.rs index 01c129263a..54dbaabdf7 100644 --- a/backend/libraries/chat_events/src/stable_storage/tests.rs +++ b/backend/libraries/chat_events/src/stable_memory/tests.rs @@ -1,9 +1,9 @@ use crate::message_content_internal::icrc1::AccountInternal; -use crate::stable_storage::tests::test_values::{ +use crate::stable_memory::tests::test_values::{ AUDIO1, CRYPTO1, CUSTOM1, DELETED1, FILE1, GIPHY1, GOVERNANCE_PROPOSAL1, IMAGE1, MESSAGE_REMINDER1, MESSAGE_REMINDER_CREATED1, P2P_SWAP1, POLL1, PRIZE1, PRIZE_WINNER1, REPORTED_MESSAGE1, TEXT1, VIDEO1, VIDEO_CALL1, }; -use crate::stable_storage::{bytes_to_event, event_to_bytes}; +use crate::stable_memory::{bytes_to_event, event_to_bytes}; use crate::{ AudioContentInternal, BlobReferenceInternal, CallParticipantInternal, ChatEventInternal, ChatInternal, CompletedCryptoTransactionInternal, CryptoContentInternal, CustomContentInternal, DeletedByInternal, FileContentInternal, diff --git a/backend/libraries/chat_events/src/stable_storage/tests/test_values.rs b/backend/libraries/chat_events/src/stable_memory/tests/test_values.rs similarity index 100% rename from backend/libraries/chat_events/src/stable_storage/tests/test_values.rs rename to backend/libraries/chat_events/src/stable_memory/tests/test_values.rs