Reduce size of chat members when serialized (#6925)

backend/canisters/community/CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Remove channel links from being stored on each community member ([#6923](https://github.com/open-chat-labs/open-chat/pull/6923))
 - Remove entries from expiring members queue when channel deleted ([#6924](https://github.com/open-chat-labs/open-chat/pull/6924))
+- Reduce size of chat members when serialized ([#6925](https://github.com/open-chat-labs/open-chat/pull/6925))
 
 ## [[2.0.1479](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1479-community)] - 2024-11-28
 

backend/canisters/community/impl/src/lifecycle/post_upgrade.rs

@@ -18,9 +18,13 @@ fn post_upgrade(args: Args) {
     let memory = get_upgrades_memory();
     let reader = get_reader(&memory);
 
-    let (data, errors, logs, traces): (Data, Vec<LogEntry>, Vec<LogEntry>, Vec<LogEntry>) =
+    let (mut data, errors, logs, traces): (Data, Vec<LogEntry>, Vec<LogEntry>, Vec<LogEntry>) =
         msgpack::deserialize(reader).unwrap();
 
+    for channel in data.channels.iter_mut() {
+        channel.chat.members.set_member_default_timestamps();
+    }
+
     canister_logger::init_with_logs(data.test_mode, errors, logs, traces);
 
     let env = init_env(data.rng_seed);

backend/canisters/community/impl/src/model/members.rs

@@ -740,6 +740,7 @@ impl From<&CommunityMemberInternal> for CommunityMember {
 mod tests {
     use super::*;
     use test_case::test_case;
+    use types::CanisterId;
 
     #[test_case(true)]
     #[test_case(false)]
@@ -775,4 +776,39 @@ mod tests {
         assert!(members.channels_for_member(user_id2).next().is_none());
         assert!(members.channels_removed_for_member(user_id2).next().is_none());
     }
+
+    #[test]
+    fn serialize_member_with_max_defaults() {
+        #[derive(Serialize, Deserialize, Clone)]
+        pub struct CommunityMemberInternal2 {
+            #[serde(rename = "u")]
+            pub user_id: UserId,
+            #[serde(rename = "d")]
+            pub date_added: TimestampMillis,
+        }
+
+        let member1 = CommunityMemberInternal {
+            user_id: CanisterId::from_text("4bkt6-4aaaa-aaaaf-aaaiq-cai").unwrap().into(),
+            date_added: 1732874138000,
+            role: CommunityRole::Member,
+            rules_accepted: None,
+            user_type: UserType::User,
+            display_name: Timestamped::default(),
+            referred_by: None,
+            referrals: BTreeSet::new(),
+            lapsed: Timestamped::default(),
+            suspended: Timestamped::default(),
+        };
+
+        let member2 = CommunityMemberInternal2 {
+            user_id: member1.user_id,
+            date_added: member1.date_added,
+        };
+
+        let bytes1 = msgpack::serialize_then_unwrap(&member1);
+        let bytes2 = msgpack::serialize_then_unwrap(&member2);
+
+        assert_eq!(bytes1, bytes2);
+        assert_eq!(bytes1.len(), 26);
+    }
 }

backend/canisters/group/CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
 
+### Changed
+
+- Reduce size of chat members when serialized ([#6925](https://github.com/open-chat-labs/open-chat/pull/6925))
+
 ## [[2.0.1480](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1480-group)] - 2024-11-28
 
 ### Changed

backend/canisters/group/impl/src/lifecycle/post_upgrade.rs

@@ -17,9 +17,11 @@ fn post_upgrade(args: Args) {
     let memory = get_upgrades_memory();
     let reader = get_reader(&memory);
 
-    let (data, errors, logs, traces): (Data, Vec<LogEntry>, Vec<LogEntry>, Vec<LogEntry>) =
+    let (mut data, errors, logs, traces): (Data, Vec<LogEntry>, Vec<LogEntry>, Vec<LogEntry>) =
         msgpack::deserialize(reader).unwrap();
 
+    data.chat.members.set_member_default_timestamps();
+
     canister_logger::init_with_logs(data.test_mode, errors, logs, traces);
 
     let env = init_env(data.rng_seed);

backend/libraries/group_chat_core/src/members.rs

@@ -35,12 +35,17 @@ pub struct GroupMembers {
     blocked: BTreeSet<UserId>,
     suspended: BTreeSet<UserId>,
     updates: BTreeSet<(TimestampMillis, UserId, MemberUpdate)>,
-    #[serde(default)]
     latest_update_removed: TimestampMillis,
 }
 
 #[allow(clippy::too_many_arguments)]
 impl GroupMembers {
+    pub fn set_member_default_timestamps(&mut self) {
+        for member in self.members.values_mut() {
+            member.set_default_timestamps();
+        }
+    }
+
     pub fn prune_proposal_votes(&mut self, now: TimestampMillis) -> u32 {
         let mut count = 0;
         for member in self.members.values_mut() {
@@ -105,19 +110,19 @@ impl GroupMembers {
             let member = GroupMemberInternal {
                 user_id,
                 date_added: now,
-                role: Timestamped::new(GroupRoleInternal::Member, now),
+                role: Timestamped::new(GroupRoleInternal::Member, 0),
                 min_visible_event_index,
                 min_visible_message_index,
-                notifications_muted: Timestamped::new(notifications_muted, now),
+                notifications_muted: Timestamped::new(notifications_muted, 0),
                 mentions: Mentions::default(),
-                followed_threads: TimestampedSet::new(),
-                unfollowed_threads: TimestampedSet::new(),
+                followed_threads: TimestampedSet::default(),
+                unfollowed_threads: TimestampedSet::default(),
                 proposal_votes: BTreeSet::default(),
                 latest_proposal_vote_removed: 0,
                 suspended: Timestamped::default(),
                 rules_accepted: None,
                 user_type,
-                lapsed: Timestamped::new(false, now),
+                lapsed: Timestamped::default(),
             };
             self.members.insert(user_id, member.clone());
             if user_type.is_bot() {
@@ -547,16 +552,19 @@ pub struct GroupMemberInternal {
     date_added: TimestampMillis,
     #[serde(rename = "r", default, skip_serializing_if = "is_default")]
     role: Timestamped<GroupRoleInternal>,
-    #[serde(rename = "n")]
+    #[serde(
+        rename = "n",
+        default = "default_notifications_muted",
+        skip_serializing_if = "is_default_notifications_muted"
+    )]
     notifications_muted: Timestamped<bool>,
-    #[serde(rename = "m", default, skip_serializing_if = "mentions_are_empty")]
+    #[serde(rename = "m", default, skip_serializing_if = "Mentions::is_empty")]
     pub mentions: Mentions,
     #[serde(rename = "tf", default, skip_serializing_if = "TimestampedSet::is_empty")]
     pub followed_threads: TimestampedSet<MessageIndex>,
     #[serde(rename = "tu", default, skip_serializing_if = "TimestampedSet::is_empty")]
     pub unfollowed_threads: TimestampedSet<MessageIndex>,
     #[serde(rename = "p", default, skip_serializing_if = "BTreeSet::is_empty")]
-    #[serde(deserialize_with = "deserialize_proposal_votes")]
     proposal_votes: BTreeSet<(TimestampMillis, MessageIndex)>,
     #[serde(rename = "pr", default, skip_serializing_if = "is_default")]
     latest_proposal_vote_removed: TimestampMillis,
@@ -574,31 +582,19 @@ pub struct GroupMemberInternal {
     lapsed: Timestamped<bool>,
 }
 
-fn deserialize_proposal_votes<'de, D: Deserializer<'de>>(d: D) -> Result<BTreeSet<(TimestampMillis, MessageIndex)>, D::Error> {
-    let votes = ProposalVotesCombined::deserialize(d)?;
-
-    Ok(match votes {
-        ProposalVotesCombined::Old(map) => {
-            let mut set = BTreeSet::new();
-            for (ts, message_indexes) in map {
-                for message_index in message_indexes {
-                    set.insert((ts, message_index));
-                }
-            }
-            set
+impl GroupMemberInternal {
+    pub fn set_default_timestamps(&mut self) {
+        if self.role.timestamp <= self.date_added {
+            self.role.timestamp = 0;
         }
-        ProposalVotesCombined::New(set) => set,
-    })
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-enum ProposalVotesCombined {
-    Old(BTreeMap<TimestampMillis, Vec<MessageIndex>>),
-    New(BTreeSet<(TimestampMillis, MessageIndex)>),
-}
+        if self.notifications_muted.timestamp <= self.date_added {
+            self.notifications_muted.timestamp = 0;
+        }
+        if self.lapsed.timestamp <= self.date_added {
+            self.lapsed.timestamp = 0;
+        }
+    }
 
-impl GroupMemberInternal {
     pub fn user_id(&self) -> UserId {
         self.user_id
     }
@@ -738,10 +734,6 @@ impl From<&GroupMemberInternal> for GroupMember {
     }
 }
 
-fn mentions_are_empty(value: &Mentions) -> bool {
-    value.is_empty()
-}
-
 fn serialize_members<S: Serializer>(value: &BTreeMap<UserId, GroupMemberInternal>, serializer: S) -> Result<S::Ok, S::Error> {
     let mut seq = serializer.serialize_seq(Some(value.len()))?;
     for member in value.values() {
@@ -775,18 +767,34 @@ impl<'de> Visitor<'de> for GroupMembersMapVisitor {
     }
 }
 
+fn default_notifications_muted() -> Timestamped<bool> {
+    Timestamped::new(true, 0)
+}
+
+fn is_default_notifications_muted(value: &Timestamped<bool>) -> bool {
+    value.value && value.timestamp == 0
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use candid::Principal;
+    use types::CanisterId;
 
     #[test]
     fn serialize_with_max_defaults() {
-        let member = GroupMemberInternal {
-            user_id: Principal::from_text("4bkt6-4aaaa-aaaaf-aaaiq-cai").unwrap().into(),
-            date_added: 1,
-            role: Timestamped::new(GroupRoleInternal::Member, 0),
-            notifications_muted: Timestamped::new(true, 1),
+        #[derive(Serialize)]
+        pub struct GroupMemberInternal2 {
+            #[serde(rename = "u")]
+            user_id: UserId,
+            #[serde(rename = "d")]
+            date_added: TimestampMillis,
+        }
+
+        let member1 = GroupMemberInternal {
+            user_id: CanisterId::from_text("4bkt6-4aaaa-aaaaf-aaaiq-cai").unwrap().into(),
+            date_added: 1732874138000,
+            role: Timestamped::default(),
+            notifications_muted: default_notifications_muted(),
             mentions: Mentions::default(),
             followed_threads: TimestampedSet::default(),
             unfollowed_threads: TimestampedSet::default(),
@@ -795,17 +803,23 @@ mod tests {
             suspended: Timestamped::default(),
             min_visible_event_index: 0.into(),
             min_visible_message_index: 0.into(),
-            rules_accepted: Some(Timestamped::new(Version::zero(), 1)),
+            rules_accepted: None,
             user_type: UserType::User,
             lapsed: Timestamped::default(),
         };
 
-        let member_bytes = msgpack::serialize_then_unwrap(&member);
-        let member_bytes_len = member_bytes.len();
+        let member2 = GroupMemberInternal2 {
+            user_id: member1.user_id,
+            date_added: member1.date_added,
+        };
 
-        assert_eq!(member_bytes_len, 37);
+        let member1_bytes = msgpack::serialize_then_unwrap(&member1);
+        let member2_bytes = msgpack::serialize_then_unwrap(&member2);
 
-        let _deserialized: GroupMemberInternal = msgpack::deserialize_then_unwrap(&member_bytes);
+        assert_eq!(member1_bytes, member2_bytes);
+        assert_eq!(member1_bytes.len(), 26);
+
+        let _deserialized: GroupMemberInternal = msgpack::deserialize_then_unwrap(&member1_bytes);
     }
 
     #[test]
@@ -814,8 +828,8 @@ mod tests {
         mentions.add(Some(1.into()), 1.into(), 1.into(), 1);
 
         let member = GroupMemberInternal {
-            user_id: Principal::from_text("4bkt6-4aaaa-aaaaf-aaaiq-cai").unwrap().into(),
-            date_added: 1,
+            user_id: CanisterId::from_text("4bkt6-4aaaa-aaaaf-aaaiq-cai").unwrap().into(),
+            date_added: 1732874138000,
             role: Timestamped::new(GroupRoleInternal::Owner, 1),
             notifications_muted: Timestamped::new(true, 1),
             mentions,
@@ -834,7 +848,7 @@ mod tests {
         let member_bytes = msgpack::serialize_then_unwrap(&member);
         let member_bytes_len = member_bytes.len();
 
-        assert_eq!(member_bytes_len, 163);
+        assert_eq!(member_bytes_len, 171);
 
         let _deserialized: GroupMemberInternal = msgpack::deserialize_then_unwrap(&member_bytes);
     }