Remove code to migrate events to stable memory (#6837)

backend/canisters/community/api/src/updates/mod.rs

@@ -11,7 +11,6 @@ pub mod c2c_invite_users_to_channel;
 pub mod c2c_join_channel;
 pub mod c2c_join_community;
 pub mod c2c_leave_community;
-pub mod c2c_migrate_events_to_stable_memory;
 pub mod c2c_send_message;
 pub mod c2c_set_user_suspended;
 pub mod c2c_tip_message;

backend/canisters/group/CHANGELOG.md

@@ -15,6 +15,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Add cycles balance check to more timer jobs ([#6822](https://github.com/open-chat-labs/open-chat/pull/6822))
 - Add the `BotCommand` access token type ([#6830](https://github.com/open-chat-labs/open-chat/pull/6830))
 
+### Removed
+
+- Remove code to migrate events to stable memory ([#6837](https://github.com/open-chat-labs/open-chat/pull/6837))
+
 ## [[2.0.1453](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1453-group)] - 2024-11-14
 
 ### Changed

backend/canisters/group/api/src/updates/mod.rs

@@ -8,7 +8,6 @@ pub mod c2c_freeze_group;
 pub mod c2c_invite_users;
 pub mod c2c_join_group;
 pub mod c2c_leave_group;
-pub mod c2c_migrate_events_to_stable_memory;
 pub mod c2c_report_message_v2;
 pub mod c2c_send_message;
 pub mod c2c_set_user_suspended;

backend/canisters/group/c2c_client/src/lib.rs

@@ -24,7 +24,6 @@ generate_c2c_call!(c2c_freeze_group);
 generate_c2c_call!(c2c_invite_users);
 generate_c2c_call!(c2c_join_group);
 generate_c2c_call!(c2c_leave_group);
-generate_c2c_call!(c2c_migrate_events_to_stable_memory);
 generate_c2c_call!(c2c_report_message_v2);
 generate_c2c_call!(c2c_send_message);
 generate_c2c_call!(c2c_set_user_suspended);

backend/canisters/group/impl/src/regular_jobs.rs

@@ -1,4 +1,3 @@
-use crate::updates::c2c_migrate_events_to_stable_memory::migrate_events_to_stable_memory_impl;
 use crate::Data;
 use utils::env::Environment;
 use utils::regular_jobs::{RegularJob, RegularJobs};
@@ -7,13 +6,8 @@ use utils::time::MINUTE_IN_MS;
 pub(crate) fn build() -> RegularJobs<Data> {
     let check_cycles_balance = RegularJob::new("Check cycles balance", check_cycles_balance, 5 * MINUTE_IN_MS);
     let retry_deleting_files = RegularJob::new("Retry deleting files", retry_deleting_files, MINUTE_IN_MS);
-    let migrate_chat_events_to_stable_memory = RegularJob::new("Migrate chat events", migrate_chat_events_to_stable_memory, 0);
 
-    RegularJobs::new(vec![
-        check_cycles_balance,
-        retry_deleting_files,
-        migrate_chat_events_to_stable_memory,
-    ])
+    RegularJobs::new(vec![check_cycles_balance, retry_deleting_files])
 }
 
 fn check_cycles_balance(_: &dyn Environment, data: &mut Data) {
@@ -23,7 +17,3 @@ fn check_cycles_balance(_: &dyn Environment, data: &mut Data) {
 fn retry_deleting_files(_: &dyn Environment, _: &mut Data) {
     storage_bucket_client::retry_failed();
 }
-
-fn migrate_chat_events_to_stable_memory(_: &dyn Environment, data: &mut Data) {
-    migrate_events_to_stable_memory_impl(data, true);
-}

backend/canisters/group/impl/src/updates/mod.rs

@@ -7,7 +7,6 @@ pub mod c2c_freeze_group;
 pub mod c2c_invite_users;
 pub mod c2c_join_group;
 pub mod c2c_leave_group;
-pub mod c2c_migrate_events_to_stable_memory;
 pub mod c2c_notify_p2p_swap_status_change;
 pub mod c2c_report_message_v2;
 pub mod c2c_set_user_suspended;

backend/canisters/user/CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Avoid extra key lookup when iterating over events ([#6680](https://github.com/open-chat-labs/open-chat/pull/6680))
 - Read events in batches when performing stable memory garbage collection ([#6682](https://github.com/open-chat-labs/open-chat/pull/6682))
 - Read events from stable memory once migration is complete ([#6722](https://github.com/open-chat-labs/open-chat/pull/6722))
+- Increase the minimum cycles balance ([#6725](https://github.com/open-chat-labs/open-chat/pull/6725))
 - Perform cycles check when migrating events to stable memory ([#6757](https://github.com/open-chat-labs/open-chat/pull/6757))
 - Store events in `HybridMap` which caches latest events on the heap ([#6762](https://github.com/open-chat-labs/open-chat/pull/6762))
 - Reduce size of metrics in memory ([#6765](https://github.com/open-chat-labs/open-chat/pull/6765))
@@ -27,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Remove events from being stored on the heap ([#6758](https://github.com/open-chat-labs/open-chat/pull/6758))
 - Removed a bunch of unwanted achievements ([#6794](https://github.com/open-chat-labs/open-chat/pull/6794))
+- Remove code to migrate events to stable memory ([#6837](https://github.com/open-chat-labs/open-chat/pull/6837))
 
 ## [[2.0.1412](https://github.com/open-chat-labs/open-chat/releases/tag/v2.0.1414-user)] - 2024-10-24
 

backend/canisters/user/impl/src/regular_jobs.rs

@@ -1,4 +1,3 @@
-use crate::updates::c2c_migrate_events_to_stable_memory::migrate_events_to_stable_memory_impl;
 use crate::Data;
 use utils::env::Environment;
 use utils::regular_jobs::{RegularJob, RegularJobs};
@@ -12,13 +11,11 @@ pub(crate) fn build() -> RegularJobs<Data> {
         5 * MINUTE_IN_MS,
     );
     let retry_deleting_files = RegularJob::new("Retry deleting files", retry_deleting_files, MINUTE_IN_MS);
-    let migrate_chat_events_to_stable_memory = RegularJob::new("Migrate chat events", migrate_chat_events_to_stable_memory, 0);
 
     RegularJobs::new(vec![
         check_cycles_balance,
         aggregate_direct_chat_metrics,
         retry_deleting_files,
-        migrate_chat_events_to_stable_memory,
     ])
 }
 
@@ -33,7 +30,3 @@ fn aggregate_direct_chat_metrics(_: &dyn Environment, data: &mut Data) {
 fn retry_deleting_files(_: &dyn Environment, _: &mut Data) {
     storage_bucket_client::retry_failed();
 }
-
-fn migrate_chat_events_to_stable_memory(_: &dyn Environment, data: &mut Data) {
-    migrate_events_to_stable_memory_impl(data, true);
-}

backend/canisters/user/impl/src/updates/mod.rs

@@ -10,7 +10,6 @@ pub mod c2c_charge_user_account;
 pub mod c2c_grant_super_admin;
 pub mod c2c_mark_community_updated_for_user;
 pub mod c2c_mark_group_updated_for_user;
-pub mod c2c_migrate_events_to_stable_memory;
 pub mod c2c_notify_achievement;
 pub mod c2c_notify_community_canister_events;
 pub mod c2c_notify_community_deleted;

backend/libraries/chat_events/src/chat_events.rs

@@ -14,12 +14,12 @@ use search::{Document, Query};
 use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
 use sha2::{Digest, Sha256};
-use std::cmp::{max, min};
+use std::cmp::max;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::mem;
 use std::ops::DerefMut;
-use tracing::{error, info};
+use tracing::error;
 use types::{
     AcceptP2PSwapResult, CallParticipant, CancelP2PSwapResult, CanisterId, Chat, ChatType, CompleteP2PSwapResult,
     CompletedCryptoTransaction, Cryptocurrency, DirectChatCreated, EventContext, EventIndex, EventWrapper,
@@ -49,25 +49,9 @@ pub struct ChatEvents {
     video_call_in_progress: Timestamped<Option<VideoCall>>,
     anonymized_id: String,
     search_index: SearchIndex,
-    #[serde(default = "default_next_event_to_migrate_to_stable_memory")]
-    next_event_to_migrate_to_stable_memory: Option<EventContext>,
-    #[serde(default)]
-    thread_messages_to_update_in_stable_memory: Vec<MessageIndex>,
-}
-
-fn default_next_event_to_migrate_to_stable_memory() -> Option<EventContext> {
-    Some(EventContext::default())
 }
 
 impl ChatEvents {
-    pub fn thread_messages_to_update_in_stable_memory_len(&self) -> usize {
-        self.thread_messages_to_update_in_stable_memory.len()
-    }
-
-    pub fn update_event_in_stable_memory(&mut self, event_key: EventKey) {
-        self.main.update_event_in_stable_memory(event_key);
-    }
-
     pub fn init_stable_storage(memory: Memory) {
         stable_storage::init(memory)
     }
@@ -94,26 +78,6 @@ impl ChatEvents {
         }
     }
 
-    pub fn migrate_next_batch_of_events_to_stable_storage(&mut self) -> bool {
-        while !self.thread_messages_to_update_in_stable_memory.is_empty() {
-            if ic_cdk::api::instruction_counter() > 1_000_000_000 {
-                return false;
-            }
-
-            let batch: Vec<_> = self
-                .thread_messages_to_update_in_stable_memory
-                .drain(..min(100, self.thread_messages_to_update_in_stable_memory.len()))
-                .collect();
-
-            let count = batch.len();
-            for message_index in batch {
-                self.update_event_in_stable_memory(message_index.into());
-            }
-            info!(chat = ?self.chat, count, "Updated threads in stable memory");
-        }
-        self.next_event_to_migrate_to_stable_memory.is_none()
-    }
-
     pub fn new_direct_chat(
         them: UserId,
         events_ttl: Option<Milliseconds>,
@@ -134,8 +98,6 @@ impl ChatEvents {
             video_call_in_progress: Timestamped::default(),
             anonymized_id: hex::encode(anonymized_id.to_be_bytes()),
             search_index: SearchIndex::default(),
-            next_event_to_migrate_to_stable_memory: None,
-            thread_messages_to_update_in_stable_memory: Vec::new(),
         };
 
         events.push_event(None, ChatEventInternal::DirectChatCreated(DirectChatCreated {}), 0, now);
@@ -166,8 +128,6 @@ impl ChatEvents {
             video_call_in_progress: Timestamped::default(),
             anonymized_id: hex::encode(anonymized_id.to_be_bytes()),
             search_index: SearchIndex::default(),
-            next_event_to_migrate_to_stable_memory: None,
-            thread_messages_to_update_in_stable_memory: Vec::new(),
         };
 
         events.push_event(

backend/libraries/chat_events/src/chat_events_list.rs

@@ -7,7 +7,7 @@ use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use std::collections::hash_map::Entry::Vacant;
 use std::collections::{HashMap, HashSet};
-use std::ops::{Deref, RangeBounds};
+use std::ops::Deref;
 use types::{
     Chat, ChatEvent, EventIndex, EventOrExpiredRange, EventWrapper, EventWrapperInternal, HydratedMention, Mention, Message,
     MessageId, MessageIndex, TimestampMillis, UserId,
@@ -21,7 +21,6 @@ pub struct ChatEventsList {
     message_event_indexes: Vec<EventIndex>,
     latest_event_index: Option<EventIndex>,
     latest_event_timestamp: Option<TimestampMillis>,
-    read_events_from_stable_memory: bool,
 }
 
 fn default_stable_events_map() -> HybridMap<ChatEventsStableStorage> {
@@ -48,18 +47,13 @@ impl ChatEventsList {
             .set_stable_memory_prefix(chat, thread_root_message_index);
     }
 
-    pub fn set_read_events_from_stable_memory(&mut self, value: bool) {
-        self.read_events_from_stable_memory = value;
-    }
-
     pub fn new(chat: Chat, thread_root_message_index: Option<MessageIndex>) -> Self {
         ChatEventsList {
             stable_events_map: HybridMap::new(chat, thread_root_message_index),
             message_id_map: HashMap::new(),
             message_event_indexes: Vec::new(),
             latest_event_index: None,
             latest_event_timestamp: None,
-            read_events_from_stable_memory: false,
         }
     }
 
@@ -159,7 +153,7 @@ impl ChatEventsList {
             (min_visible_event_index, self.latest_event_index.unwrap_or_default())
         };
 
-        let iter = self.range_internal(min..=max);
+        let iter = self.stable_events_map.range(min..=max);
 
         if ascending {
             Box::new(ChatEventsListIterator {
@@ -212,7 +206,8 @@ impl ChatEventsList {
     }
 
     pub(crate) fn event_count_since<F: Fn(&ChatEventInternal) -> bool>(&self, since: TimestampMillis, filter: &F) -> usize {
-        self.iter_internal()
+        self.stable_events_map
+            .iter()
             .rev()
             .take_while(|e| e.timestamp > since)
             .filter(|e| filter(&e.event))
@@ -248,7 +243,7 @@ impl ChatEventsList {
     }
 
     pub fn last(&self) -> Option<EventWrapperInternal<ChatEventInternal>> {
-        self.iter_internal().next_back()
+        self.stable_events_map.iter().next_back()
     }
 
     pub fn contains_message_id(&self, message_id: MessageId) -> bool {
@@ -267,26 +262,15 @@ impl ChatEventsList {
         &self,
         event_index: EventIndex,
     ) -> Result<EventWrapperInternal<ChatEventInternal>, (Option<EventIndex>, Option<EventIndex>)> {
-        let next_key = match self.range_internal(event_index..).next() {
+        let next_key = match self.stable_events_map.range(event_index..).next() {
             Some(v) if v.index == event_index => return Ok(v),
             Some(v) => Some(v.index),
             None => None,
         };
-        let previous_key = self.range_internal(..event_index).next_back().map(|e| e.index);
+        let previous_key = self.stable_events_map.range(..event_index).next_back().map(|e| e.index);
 
         Err((previous_key, next_key))
     }
-
-    fn iter_internal(&self) -> Box<dyn DoubleEndedIterator<Item = EventWrapperInternal<ChatEventInternal>> + '_> {
-        self.stable_events_map.iter()
-    }
-
-    fn range_internal<R: RangeBounds<EventIndex>>(
-        &self,
-        range: R,
-    ) -> Box<dyn DoubleEndedIterator<Item = EventWrapperInternal<ChatEventInternal>> + '_> {
-        self.stable_events_map.range(range)
-    }
 }
 
 pub enum UpdateEventError<E = ()> {