Significantly improve performance of database observers (#3260)

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Reset channel members and watchers state when fetching the initial state of the channel [#3245](https://github.com/GetStream/stream-chat-swift/pull/3245)
 - Fix inconsistent message text when extremely quickly updating it [#3242](https://github.com/GetStream/stream-chat-swift/pull/3242)
 - Fix message attachments returning nil sometimes in push notification extensions [#3261](https://github.com/GetStream/stream-chat-swift/pull/3261)
+- Significantly improve performance of database observers [#3260](https://github.com/GetStream/stream-chat-swift/pull/3260)
 ### 🔄 Changed
 - Enable background mapping by default, which improves performance overall [#3250](https://github.com/GetStream/stream-chat-swift/pull/3250)
 

Sources/StreamChat/Config/StreamRuntimeCheck.swift

@@ -38,4 +38,9 @@ public enum StreamRuntimeCheck {
     ///
     ///  Enables using our legacy web socket connection.
     public static var _useLegacyWebSocketConnection = false
+
+    /// For *internal use* only
+    ///
+    /// Enables reusing unchanged converted items in database observers.
+    public static var _isDatabaseObserverItemReusingEnabled = true
 }

Sources/StreamChat/Controllers/ChannelController/ChannelController.swift

@@ -1550,7 +1550,8 @@ private extension ChatChannelController {
                     deletedMessagesVisibility: deletedMessageVisibility ?? .visibleForCurrentUser,
                     shouldShowShadowedMessages: shouldShowShadowedMessages ?? false
                 ),
-                itemCreator: { try $0.asModel() as ChatMessage }
+                itemCreator: { try $0.asModel() as ChatMessage },
+                itemReuseKeyPaths: (\ChatMessage.id, \MessageDTO.id)
             )
             observer.onDidChange = { [weak self] changes in
                 self?.delegateCallback {

Sources/StreamChat/Controllers/ChannelListController/ChannelListController.swift

@@ -279,7 +279,14 @@ extension ChatChannelListController {
             _ sorting: [SortValue<ChatChannel>]
         )
             -> ListDatabaseObserverWrapper<ChatChannel, ChannelDTO> = {
-                ListDatabaseObserverWrapper(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3, sorting: $4)
+                ListDatabaseObserverWrapper(
+                    isBackground: $0,
+                    database: $1,
+                    fetchRequest: $2,
+                    itemCreator: $3,
+                    itemReuseKeyPaths: (\ChatChannel.cid.rawValue, \ChannelDTO.cid),
+                    sorting: $4
+                )
             }
     }
 }

Sources/StreamChat/Controllers/ChannelWatcherListController/ChatChannelWatcherListController.swift

@@ -157,6 +157,7 @@ extension ChatChannelWatcherListController {
                 database: $1,
                 fetchRequest: $2,
                 itemCreator: $3,
+                itemReuseKeyPaths: (\ChatUser.id, \UserDTO.id),
                 fetchedResultsControllerType: $4
             )
         }

Sources/StreamChat/Controllers/DatabaseObserver/BackgroundDatabaseObserver.swift

@@ -16,6 +16,7 @@ class BackgroundDatabaseObserver<Item, DTO: NSManagedObject> {
 
     /// Used to convert the `DTO`s to the resulting `Item`s.
     private let itemCreator: (DTO) throws -> Item
+    private let itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)?
     private let sorting: [SortValue<Item>]
 
     /// Used to observe the changes in the DB.
@@ -64,17 +65,20 @@ class BackgroundDatabaseObserver<Item, DTO: NSManagedObject> {
     ///   - fetchRequest: The `NSFetchRequest` that specifies the elements of the list.
     ///   - context: The `NSManagedObjectContext` the observer observes.
     ///   - itemCreator: A closure the observer uses to convert DTO objects into Model objects.
+    ///   - itemReuseKeyPaths: A pair of keypaths used for reusing existing items if they have not changed
     ///   - sorting: An array of SortValue that define the order of the elements in the list.
     ///   - fetchedResultsControllerType: The `NSFetchedResultsController` subclass the observer uses to create its FRC. You can
     ///    inject your custom subclass if needed, i.e. when testing.
     init(
         context: NSManagedObjectContext,
         fetchRequest: NSFetchRequest<DTO>,
         itemCreator: @escaping (DTO) throws -> Item,
+        itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)? = nil,
         sorting: [SortValue<Item>],
         fetchedResultsControllerType: NSFetchedResultsController<DTO>.Type
     ) {
         self.itemCreator = itemCreator
+        self.itemReuseKeyPaths = itemReuseKeyPaths
         self.sorting = sorting
         changeAggregator = ListChangeAggregator<DTO, Item>(itemCreator: itemCreator)
         frc = fetchedResultsControllerType.init(
@@ -166,7 +170,7 @@ class BackgroundDatabaseObserver<Item, DTO: NSManagedObject> {
     /// When the process is done, it also updates the `_items`, which is the locally cached list of mapped items
     /// This method will be called through an operation on `processingQueue`, which will serialize the execution until `onCompletion` is called.
     private func processItems(_ changes: [ListChange<Item>]?, onCompletion: @escaping () -> Void) {
-        mapItems { [weak self] items in
+        mapItems(changes) { [weak self] items in
             guard let self = self else {
                 onCompletion()
                 return
@@ -186,22 +190,17 @@ class BackgroundDatabaseObserver<Item, DTO: NSManagedObject> {
     /// This method is intended to be called from the `managedObjectContext` that is publishing the changes (The one linked to the `NSFetchedResultsController`
     /// in this case).
     /// Once the objects are mapped, those are sorted based on `sorting`
-    private func mapItems(completion: @escaping ([Item]) -> Void) {
+    private func mapItems(_ changes: [ListChange<Item>]?, completion: @escaping ([Item]) -> Void) {
         let objects = frc.fetchedObjects ?? []
-
-        var items: [Item?] = []
-        items = objects.map { [weak self] in
-            guard self?.isDeletingDatabase == false else { return nil }
-            return try? self?.itemCreator($0)
-        }
-
-        let sorting = self.sorting
-        queue.async {
-            var result = items.compactMap { $0 }
-            if !sorting.isEmpty {
-                result = result.sort(using: sorting)
-            }
-            completion(result)
-        }
+        let items = DatabaseItemConverter.convert(
+            dtos: objects,
+            existing: rawItems,
+            changes: changes,
+            itemCreator: itemCreator,
+            itemReuseKeyPaths: itemReuseKeyPaths,
+            sorting: sorting,
+            checkCancellation: { [weak self] in self?.isDeletingDatabase == true }
+        )
+        completion(items)
     }
 }

Sources/StreamChat/Controllers/DatabaseObserver/BackgroundListDatabaseObserver.swift

@@ -61,6 +61,7 @@ class ListDatabaseObserverWrapper<Item, DTO: NSManagedObject> {
         database: DatabaseContainer,
         fetchRequest: NSFetchRequest<DTO>,
         itemCreator: @escaping (DTO) throws -> Item,
+        itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)?,
         sorting: [SortValue<Item>] = [],
         fetchedResultsControllerType: NSFetchedResultsController<DTO>.Type = NSFetchedResultsController<DTO>.self
     ) {
@@ -70,6 +71,7 @@ class ListDatabaseObserverWrapper<Item, DTO: NSManagedObject> {
                 context: database.backgroundReadOnlyContext,
                 fetchRequest: fetchRequest,
                 itemCreator: itemCreator,
+                itemReuseKeyPaths: itemReuseKeyPaths,
                 sorting: sorting,
                 fetchedResultsControllerType: fetchedResultsControllerType
             )
@@ -78,6 +80,7 @@ class ListDatabaseObserverWrapper<Item, DTO: NSManagedObject> {
                 context: database.viewContext,
                 fetchRequest: fetchRequest,
                 itemCreator: itemCreator,
+                itemReuseKeyPaths: itemReuseKeyPaths,
                 sorting: sorting,
                 fetchedResultsControllerType: fetchedResultsControllerType
             )
@@ -104,13 +107,15 @@ class BackgroundListDatabaseObserver<Item, DTO: NSManagedObject>: BackgroundData
         context: NSManagedObjectContext,
         fetchRequest: NSFetchRequest<DTO>,
         itemCreator: @escaping (DTO) throws -> Item,
+        itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)? = nil,
         sorting: [SortValue<Item>],
         fetchedResultsControllerType: NSFetchedResultsController<DTO>.Type = NSFetchedResultsController<DTO>.self
     ) {
         super.init(
             context: context,
             fetchRequest: fetchRequest,
             itemCreator: itemCreator,
+            itemReuseKeyPaths: itemReuseKeyPaths,
             sorting: sorting,
             fetchedResultsControllerType: fetchedResultsControllerType
         )

Sources/StreamChat/Controllers/DatabaseObserver/ListDatabaseObserver.swift

@@ -182,6 +182,7 @@ class ListDatabaseObserver<Item, DTO: NSManagedObject> {
     private(set) var frc: NSFetchedResultsController<DTO>!
 
     let itemCreator: (DTO) throws -> Item
+    let itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)?
     let sorting: [SortValue<Item>]
     let request: NSFetchRequest<DTO>
     let context: NSManagedObjectContext
@@ -206,12 +207,14 @@ class ListDatabaseObserver<Item, DTO: NSManagedObject> {
         context: NSManagedObjectContext,
         fetchRequest: NSFetchRequest<DTO>,
         itemCreator: @escaping (DTO) throws -> Item,
+        itemReuseKeyPaths: (item: KeyPath<Item, String>, dto: KeyPath<DTO, String>)? = nil,
         sorting: [SortValue<Item>],
         fetchedResultsControllerType: NSFetchedResultsController<DTO>.Type = NSFetchedResultsController<DTO>.self
     ) {
         self.context = context
         request = fetchRequest
         self.itemCreator = itemCreator
+        self.itemReuseKeyPaths = itemReuseKeyPaths
         self.sorting = sorting
         frc = fetchedResultsControllerType.init(
             fetchRequest: request,

Sources/StreamChat/Controllers/MemberListController/MemberListController.swift

@@ -169,7 +169,14 @@ extension ChatChannelMemberListController {
             _ itemCreator: @escaping (MemberDTO) throws -> ChatChannelMember,
             _ controllerType: NSFetchedResultsController<MemberDTO>.Type
         ) -> ListDatabaseObserverWrapper<ChatChannelMember, MemberDTO> = {
-            .init(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3, fetchedResultsControllerType: $4)
+            .init(
+                isBackground: $0,
+                database: $1,
+                fetchRequest: $2,
+                itemCreator: $3,
+                itemReuseKeyPaths: (\ChatChannelMember.id, \MemberDTO.id),
+                fetchedResultsControllerType: $4
+            )
         }
     }
 }

Sources/StreamChat/Controllers/MessageController/MessageController.swift

@@ -757,7 +757,14 @@ extension ChatMessageController {
             _ itemCreator: @escaping (MessageDTO) throws -> ChatMessage,
             _ fetchedResultsControllerType: NSFetchedResultsController<MessageDTO>.Type
         ) -> ListDatabaseObserverWrapper<ChatMessage, MessageDTO> = {
-            .init(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3, fetchedResultsControllerType: $4)
+            .init(
+                isBackground: $0,
+                database: $1,
+                fetchRequest: $2,
+                itemCreator: $3,
+                itemReuseKeyPaths: (\ChatMessage.id, \MessageDTO.id),
+                fetchedResultsControllerType: $4
+            )
         }
 
         var messageUpdaterBuilder: (

Sources/StreamChat/Controllers/PollController/PollController.swift

@@ -313,7 +313,8 @@ extension PollController {
                 isBackground: $0,
                 database: $1,
                 fetchRequest: $2,
-                itemCreator: $3
+                itemCreator: $3,
+                itemReuseKeyPaths: (\PollVote.id, \PollVoteDTO.id)
             )
         }
     }

Sources/StreamChat/Controllers/PollController/PollVoteListController.swift

@@ -194,7 +194,13 @@ extension PollVoteListController {
             _ itemCreator: @escaping (PollVoteDTO) throws -> PollVote
         )
             -> ListDatabaseObserverWrapper<PollVote, PollVoteDTO> = {
-                ListDatabaseObserverWrapper(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3)
+                ListDatabaseObserverWrapper(
+                    isBackground: $0,
+                    database: $1,
+                    fetchRequest: $2,
+                    itemCreator: $3,
+                    itemReuseKeyPaths: (\PollVote.id, \PollVoteDTO.id)
+                )
             }
     }
 }

Sources/StreamChat/Controllers/ReactionListController/ReactionListController.swift

@@ -172,7 +172,13 @@ extension ChatReactionListController {
             _ itemCreator: @escaping (MessageReactionDTO) throws -> ChatMessageReaction
         )
             -> ListDatabaseObserverWrapper<ChatMessageReaction, MessageReactionDTO> = {
-                ListDatabaseObserverWrapper(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3)
+                ListDatabaseObserverWrapper(
+                    isBackground: $0,
+                    database: $1,
+                    fetchRequest: $2,
+                    itemCreator: $3,
+                    itemReuseKeyPaths: (\ChatMessageReaction.id, \MessageReactionDTO.id)
+                )
             }
     }
 }

Sources/StreamChat/Controllers/SearchControllers/MessageSearchController/MessageSearchController.swift

@@ -94,7 +94,8 @@ public class ChatMessageSearchController: DataController, DelegateCallable, Data
             fetchRequest: MessageDTO.messagesFetchRequest(
                 for: lastQuery ?? query
             ),
-            itemCreator: { try $0.asModel() as ChatMessage }
+            itemCreator: { try $0.asModel() as ChatMessage },
+            itemReuseKeyPaths: (\ChatMessage.id, \MessageDTO.id)
         )
         observer.onDidChange = { [weak self] changes in
             self?.delegateCallback { [weak self] in

Sources/StreamChat/Controllers/ThreadListController/ThreadListController.swift

@@ -204,7 +204,11 @@ extension ChatThreadListController {
         )
             -> ListDatabaseObserverWrapper<ChatThread, ThreadDTO> = {
                 ListDatabaseObserverWrapper(
-                    isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3
+                    isBackground: $0,
+                    database: $1,
+                    fetchRequest: $2,
+                    itemCreator: $3,
+                    itemReuseKeyPaths: (\ChatThread.reuseId, \ThreadDTO.reuseId)
                 )
             }
     }

Sources/StreamChat/Controllers/UserListController/UserListController.swift

@@ -167,7 +167,13 @@ extension ChatUserListController {
             _ itemCreator: @escaping (UserDTO) throws -> ChatUser
         )
             -> ListDatabaseObserverWrapper<ChatUser, UserDTO> = {
-                ListDatabaseObserverWrapper(isBackground: $0, database: $1, fetchRequest: $2, itemCreator: $3)
+                ListDatabaseObserverWrapper(
+                    isBackground: $0,
+                    database: $1,
+                    fetchRequest: $2,
+                    itemCreator: $3,
+                    itemReuseKeyPaths: (\ChatUser.id, \UserDTO.id)
+                )
             }
     }
 }

Sources/StreamChat/Database/DTOs/MessageReactionDTO.swift

@@ -210,6 +210,7 @@ extension MessageReactionDTO {
         }
 
         return try .init(
+            id: id,
             type: .init(rawValue: type),
             score: Int(score),
             createdAt: createdAt?.bridgeDate ?? .init(),

Sources/StreamChat/Database/DTOs/ThreadDTO.swift

@@ -234,3 +234,15 @@ extension NSManagedObjectContext {
         results.forEach { delete($0) }
     }
 }
+
+extension ThreadDTO {
+    var reuseId: String {
+        channel.cid + parentMessageId
+    }
+}
+
+extension ChatThread {
+    var reuseId: String {
+        channel.cid.rawValue + parentMessageId
+    }
+}

Sources/StreamChat/Models/MessageReaction.swift

@@ -7,6 +7,9 @@ import Foundation
 /// A type representing a message reaction. `ChatMessageReaction` is an immutable snapshot
 /// of a message reaction entity at the given time.
 public struct ChatMessageReaction: Hashable {
+    /// The id of the reaction.
+    let id: String
+
     /// The reaction type.
     public let type: MessageReactionType
 

Sources/StreamChat/StateLayer/ChannelListState+Observer.swift

@@ -38,6 +38,7 @@ extension ChannelListState {
                     chatClientConfig: clientConfig
                 ),
                 itemCreator: { try $0.asModel() },
+                itemReuseKeyPaths: (\ChatChannel.cid.rawValue, \ChannelDTO.cid),
                 sorting: query.sort.runtimeSorting
             )
             channelListLinker = ChannelListLinker(

Sources/StreamChat/StateLayer/ChatState+Observer.swift

@@ -43,12 +43,14 @@ extension ChatState {
                     shouldShowShadowedMessages: clientConfig.shouldShowShadowedMessages
                 ),
                 itemCreator: { try $0.asModel() },
+                itemReuseKeyPaths: (\ChatMessage.id, \MessageDTO.id),
                 sorting: []
             )
             watchersObserver = StateLayerDatabaseObserver(
                 databaseContainer: database,
                 fetchRequest: UserDTO.watcherFetchRequest(cid: cid),
                 itemCreator: { try $0.asModel() },
+                itemReuseKeyPaths: (\ChatUser.id, \UserDTO.id),
                 sorting: []
             )
             self.eventNotificationCenter = eventNotificationCenter