From 153f082de9c85fb1713a7bb2f8e8a219a5ff1a26 Mon Sep 17 00:00:00 2001 From: Terry Yiu <963907+tyiu@users.noreply.github.com> Date: Sun, 4 Aug 2024 01:21:32 -0400 Subject: [PATCH] Aggressively optimize queries to relays --- Comingle.xcodeproj/project.pbxproj | 4 + Comingle/ComingleApp.swift | 2 +- Comingle/Controllers/AppState.swift | 258 ++++++++++++++---- .../Models/RelaySubscriptionMetadata.swift | 20 ++ Comingle/Views/EventListView.swift | 8 + Comingle/Views/EventView.swift | 62 ++--- 6 files changed, 265 insertions(+), 89 deletions(-) create mode 100644 Comingle/Models/RelaySubscriptionMetadata.swift diff --git a/Comingle.xcodeproj/project.pbxproj b/Comingle.xcodeproj/project.pbxproj index 76830dc..6647036 100644 --- a/Comingle.xcodeproj/project.pbxproj +++ b/Comingle.xcodeproj/project.pbxproj @@ -64,6 +64,7 @@ 3A6451852C44C879000DC75B /* RelaySettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3A6451842C44C879000DC75B /* RelaySettings.swift */; }; 3A7D25DE2C5D12AE001C1714 /* Launch Screen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3A7D25DD2C5D12AE001C1714 /* Launch Screen.storyboard */; }; 3ABD0C0D2C5EF6A9004A15C7 /* String+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3ABD0C0C2C5EF6A9004A15C7 /* String+Extensions.swift */; }; + 3ABD0C152C5F2508004A15C7 /* RelaySubscriptionMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -142,6 +143,7 @@ 3A7D25D12C5D0CAA001C1714 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 3A7D25DD2C5D12AE001C1714 /* Launch Screen.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = "Launch Screen.storyboard"; sourceTree = ""; }; 3ABD0C0C2C5EF6A9004A15C7 /* String+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+Extensions.swift"; sourceTree = ""; }; + 3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelaySubscriptionMetadata.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -272,6 +274,7 @@ 3A3419492C50A36B0039620B /* PersistentNostrEvent.swift */, 3A464FFB2C3A312200AD4B09 /* Profile.swift */, 3A317C182C59C130004DA56B /* PublicKeySortComparator.swift */, + 3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */, 3A317C1B2C59C5EA004DA56B /* RSVPSortComparator.swift */, 3A317C122C594AFE004DA56B /* SearchViewModel.swift */, 3A4FCB532C365C62007AE9A4 /* TimeBasedCalendarEventSortComparator.swift */, @@ -530,6 +533,7 @@ 3A317C242C5A71E9004DA56B /* MKAutocompleteManager.swift in Sources */, 3A34AAAA2B491DE900C5F910 /* SwiftUI+LocalizedStringResource.swift in Sources */, 3A317C172C59BFAE004DA56B /* CalendarEventParticipantSortComparator.swift in Sources */, + 3ABD0C152C5F2508004A15C7 /* RelaySubscriptionMetadata.swift in Sources */, 3A4A898A2C3C240C00238043 /* ProfilePictureAndNameView.swift in Sources */, 3A317C272C5B4260004DA56B /* LocationSearchView.swift in Sources */, 3A4A89942C3CE40A00238043 /* GuestProfilePictureView.swift in Sources */, diff --git a/Comingle/ComingleApp.swift b/Comingle/ComingleApp.swift index 199ba9c..b45417d 100644 --- a/Comingle/ComingleApp.swift +++ b/Comingle/ComingleApp.swift @@ -18,7 +18,7 @@ struct ComingleApp: App { init() { NostrEventValueTransformer.register() do { - container = try ModelContainer(for: AppSettings.self, PersistentNostrEvent.self) + container = try ModelContainer(for: RelaySubscriptionMetadata.self, AppSettings.self, PersistentNostrEvent.self) appState = AppState(modelContext: container.mainContext) } catch { fatalError("Failed to create ModelContainer for AppSettings and PersistentNostrEvent.") diff --git a/Comingle/Controllers/AppState.swift b/Comingle/Controllers/AppState.swift index 6c3b958..ae04c8d 100644 --- a/Comingle/Controllers/AppState.swift +++ b/Comingle/Controllers/AppState.swift @@ -45,6 +45,14 @@ class AppState: ObservableObject, Hashable { @Published var metadataTrie = Trie() + // Keep track of relay pool active subscriptions and the until filter so that we can limit the scope of how much we query from the relay pools. + var metadataSubscriptionCounts = [String: Int]() + var metadataSubscriptionDates = [String: Date]() + var bootstrapSubscriptionCounts = [String: Int]() + var bootstrapSubscriptionDates = [String: Date]() + var timeBasedCalendarEventSubscriptionCounts = [String: Int]() + var timeBasedCalendarEventSubscriptionDates = [String: Date]() + init(modelContext: ModelContext) { self.modelContext = modelContext } @@ -222,6 +230,23 @@ class AppState: ObservableObject, Hashable { ) return try? modelContext.fetch(descriptor).first } + + var relaySubscriptionMetadata: RelaySubscriptionMetadata? { + let descriptor = FetchDescriptor() + + if let result = try? modelContext.fetch(descriptor).first { + return result + } else { + let result = RelaySubscriptionMetadata() + modelContext.insert(result) + do { + try modelContext.save() + } catch { + print("Unable to save initial RelaySubscriptionMetadata object.") + } + return result + } + } } extension AppState: EventVerifying, RelayDelegate { @@ -233,80 +258,158 @@ extension AppState: EventVerifying, RelayDelegate { } func pullMissingMetadata(_ pubkeys: [String]) { - let pubkeysToFetchMetadata = Set(pubkeys).filter { self.metadataEvents[$0] == nil } + // There has to be at least one connected relay to be able to pull metadata. + guard !relayReadPool.relays.isEmpty && relayReadPool.relays.contains(where: { $0.state == .connected }) else { + return + } + + let relaySubscriptionMetadata = relaySubscriptionMetadata + + let since: Int? + if let lastPulledMetadataEvents = relaySubscriptionMetadata?.lastPulledMetadataEvents { + since = Int(lastPulledMetadataEvents.timeIntervalSince1970) + 1 + } else { + since = nil + } + let until = Date.now + + let allPubkeysSet = Set(pubkeys) + let pubkeysToFetchMetadata = allPubkeysSet.filter { self.metadataEvents[$0] == nil } if !pubkeysToFetchMetadata.isEmpty { - guard let metadataFilter = Filter( + guard let missingMetadataFilter = Filter( authors: Array(pubkeysToFetchMetadata), kinds: [EventKind.metadata.rawValue] ) else { - print("Unable to create metadata filter for \(pubkeysToFetchMetadata).") + print("Unable to create missing metadata filter for \(pubkeysToFetchMetadata).") return } - _ = relayReadPool.subscribe(with: metadataFilter) + _ = relayReadPool.subscribe(with: missingMetadataFilter) + } + + if !metadataSubscriptionCounts.isEmpty { + // Do not refresh metadata if one is already in progress. + return + } + + let pubkeysToRefresh = allPubkeysSet.subtracting(pubkeysToFetchMetadata) + guard let metadataRefreshFilter = Filter( + authors: Array(pubkeysToRefresh), + kinds: [EventKind.metadata.rawValue], + since: since + ) else { + print("Unable to create refresh metadata filter for \(pubkeysToRefresh).") + return + } + + relaySubscriptionMetadata?.lastPulledMetadataEvents = until + _ = relayReadPool.subscribe(with: metadataRefreshFilter) + + } + + /// Subscribe with filter to relay if provided, or use relay read pool if not. + func subscribe(filter: Filter, relay: Relay? = nil) throws -> String? { + if let relay { + do { + return try relay.subscribe(with: filter) + } catch { + print("Could not subscribe to relay with filter.") + return nil + } + } else { + return relayReadPool.subscribe(with: filter) } } func refresh(relay: Relay? = nil) { - guard (relay == nil && !relayReadPool.relays.isEmpty) || relay?.state == .connected else { + guard (relay == nil && !relayReadPool.relays.isEmpty && relayReadPool.relays.contains(where: { $0.state == .connected })) || relay?.state == .connected else { return } - let authors = profiles.compactMap({ $0.publicKeyHex }) - if !authors.isEmpty { - guard let bootstrapFilter = Filter( - authors: authors, - kinds: [EventKind.metadata.rawValue, EventKind.followList.rawValue, EventKind.timeBasedCalendarEvent.rawValue, EventKind.calendarEventRSVP.rawValue, EventKind.deletion.rawValue] - ) else { - print("Unable to create the boostrap filter.") - return - } + let relaySubscriptionMetadata = relaySubscriptionMetadata + let until = Date.now + + if bootstrapSubscriptionCounts.isEmpty { + let authors = profiles.compactMap({ $0.publicKeyHex }) + if !authors.isEmpty { + let since: Int? + if let lastBootstrapped = relaySubscriptionMetadata?.lastBootstrapped { + since = Int(lastBootstrapped.timeIntervalSince1970) + 1 + } else { + since = nil + } + + guard let bootstrapFilter = Filter( + authors: authors, + kinds: [EventKind.metadata.rawValue, EventKind.followList.rawValue, EventKind.timeBasedCalendarEvent.rawValue, EventKind.calendarEventRSVP.rawValue, EventKind.deletion.rawValue], + since: since + ) else { + print("Unable to create the boostrap filter.") + return + } - if let relay { do { - try relay.subscribe(with: bootstrapFilter) + if let bootstrapSubscriptionId = try subscribe(filter: bootstrapFilter, relay: relay), relay == nil { + if let bootstrapSubscriptionCount = bootstrapSubscriptionCounts[bootstrapSubscriptionId] { + bootstrapSubscriptionCounts[bootstrapSubscriptionId] = bootstrapSubscriptionCount + 1 + } else { + bootstrapSubscriptionCounts[bootstrapSubscriptionId] = 1 + } + } } catch { print("Could not subscribe to relay with the boostrap filter.") } - } else { - _ = relayReadPool.subscribe(with: bootstrapFilter) } } - guard let timeBasedCalendarEventFilter = Filter( - kinds: [EventKind.timeBasedCalendarEvent.rawValue] - ) else { - print("Unable to create the time-based calendar event filter.") - return - } + if timeBasedCalendarEventSubscriptionCounts.isEmpty { + let since: Int? + if let lastPulledAllTimeBasedCalendarEvents = relaySubscriptionMetadata?.lastPulledAllTimeBasedCalendarEvents { + since = Int(lastPulledAllTimeBasedCalendarEvents.timeIntervalSince1970) + 1 + } else { + since = nil + } + + guard let timeBasedCalendarEventFilter = Filter( + kinds: [EventKind.timeBasedCalendarEvent.rawValue], + since: since + ) else { + print("Unable to create the time-based calendar event filter.") + return + } - if let relay { do { - try relay.subscribe(with: timeBasedCalendarEventFilter) + if let timeBasedCalendarEventSubscriptionId = try subscribe(filter: timeBasedCalendarEventFilter, relay: relay), relay == nil { + if let timeBasedCalendarEventSubscriptionCount = timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] { + timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] = timeBasedCalendarEventSubscriptionCount + 1 + } else { + timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] = 1 + } + } } catch { print("Could not subscribe to relay with the time-based calendar event filter.") } - } else { - _ = relayReadPool.subscribe(with: timeBasedCalendarEventFilter) } } private func didReceiveFollowListEvent(_ followListEvent: FollowListEvent, shouldPullMissingMetadata: Bool = false) { if let existingFollowList = self.followListEvents[followListEvent.pubkey] { if existingFollowList.createdAt < followListEvent.createdAt { - cache(followListEvent) + cache(followListEvent, shouldPullMissingMetadata: shouldPullMissingMetadata) } } else { - cache(followListEvent) + cache(followListEvent, shouldPullMissingMetadata: shouldPullMissingMetadata) } + } + + private func cache(_ followListEvent: FollowListEvent, shouldPullMissingMetadata: Bool) { + self.followListEvents[followListEvent.pubkey] = followListEvent if shouldPullMissingMetadata { pullMissingMetadata(followListEvent.followedPubkeys) } - } - private func cache(_ followListEvent: FollowListEvent) { - self.followListEvents[followListEvent.pubkey] = followListEvent + // TODO Here or elsewhere. Query for calendar events that follows who have RSVP'd. } private func didReceiveMetadataEvent(_ metadataEvent: MetadataEvent) { @@ -336,7 +439,7 @@ extension AppState: EventVerifying, RelayDelegate { } } - private func didReceiveTimeBasedCalendarEvent(_ timeBasedCalendarEvent: TimeBasedCalendarEvent, shouldPullMissingMetadata: Bool = false) { + private func didReceiveTimeBasedCalendarEvent(_ timeBasedCalendarEvent: TimeBasedCalendarEvent) { guard let eventCoordinates = timeBasedCalendarEvent.replaceableEventCoordinates()?.tag.value, let startTimestamp = timeBasedCalendarEvent.startTimestamp, startTimestamp <= timeBasedCalendarEvent.endTimestamp ?? startTimestamp, @@ -351,27 +454,6 @@ extension AppState: EventVerifying, RelayDelegate { } else { timeBasedCalendarEvents[eventCoordinates] = timeBasedCalendarEvent } - - if shouldPullMissingMetadata { - pullMissingMetadata([timeBasedCalendarEvent.pubkey]) - } - - guard let replaceableEventCoordinates = timeBasedCalendarEvent.replaceableEventCoordinates() else { - print("Unable to get replaceable event coordinates for time-based calendar event.") - return - } - - let replaceableEventCoordinatesTag = replaceableEventCoordinates.tag - - guard let rsvpFilter = Filter( - kinds: [EventKind.calendarEventRSVP.rawValue], - tags: ["a": [replaceableEventCoordinatesTag.value]]) - else { - print("Unable to create calendar event RSVP filter.") - return - } - - _ = relayReadPool.subscribe(with: rsvpFilter) } func updateCalendarEventRSVP(_ rsvp: CalendarEventRSVP, rsvpEventCoordinates: String) { @@ -511,7 +593,7 @@ extension AppState: EventVerifying, RelayDelegate { case let metadataEvent as MetadataEvent: self.didReceiveMetadataEvent(metadataEvent) case let timeBasedCalendarEvent as TimeBasedCalendarEvent: - self.didReceiveTimeBasedCalendarEvent(timeBasedCalendarEvent, shouldPullMissingMetadata: true) + self.didReceiveTimeBasedCalendarEvent(timeBasedCalendarEvent) case let rsvpEvent as CalendarEventRSVP: self.didReceiveCalendarEventRSVP(rsvpEvent) case let deletionEvent as DeletionEvent: @@ -551,6 +633,10 @@ extension AppState: EventVerifying, RelayDelegate { // Live new events are not strictly needed for this app for now. // In the future, we could keep subscriptions open for updates. try? relay.closeSubscription(with: subscriptionId) + updateRelaySubscriptionMetadataTimestamps(with: subscriptionId) + updateRelaySubscriptionCounts(closedSubscriptionId: subscriptionId) + case let .closed(subscriptionId, _): + updateRelaySubscriptionCounts(closedSubscriptionId: subscriptionId) case let .ok(eventId, success, message): if success { if let persistentNostrEvent = persistentNostrEvent(eventId), !persistentNostrEvent.relays.contains(relay.url) { @@ -564,6 +650,66 @@ extension AppState: EventVerifying, RelayDelegate { } } + func updateRelaySubscriptionCounts(closedSubscriptionId: String) { + if let metadataSubscriptionCount = metadataSubscriptionCounts[closedSubscriptionId] { + if metadataSubscriptionCount <= 1 { + metadataSubscriptionCounts.removeValue(forKey: closedSubscriptionId) + } else { + metadataSubscriptionCounts[closedSubscriptionId] = metadataSubscriptionCount - 1 + } + } + + if let bootstrapSubscriptionCount = bootstrapSubscriptionCounts[closedSubscriptionId] { + if bootstrapSubscriptionCount <= 1 { + bootstrapSubscriptionCounts.removeValue(forKey: closedSubscriptionId) + } else { + bootstrapSubscriptionCounts[closedSubscriptionId] = bootstrapSubscriptionCount - 1 + } + } + + if let timeBasedCalendarEventSubscriptionCount = timeBasedCalendarEventSubscriptionCounts[closedSubscriptionId] { + if timeBasedCalendarEventSubscriptionCount <= 1 { + timeBasedCalendarEventSubscriptionCounts.removeValue(forKey: closedSubscriptionId) + + // Wait until we have fetched all the time-based calendar events before fetching metadata in bulk. + pullMissingMetadata(timeBasedCalendarEvents.values.map { $0.pubkey }) + } else { + timeBasedCalendarEventSubscriptionCounts[closedSubscriptionId] = timeBasedCalendarEventSubscriptionCount - 1 + } + } + } + + func updateRelaySubscriptionMetadataTimestamps(with subscriptionId: String) { + let relaySubscriptionMetadata = relaySubscriptionMetadata + + if let relaySubscriptionMetadata, + let lastPulledMetadataEvents = relaySubscriptionMetadata.lastPulledMetadataEvents, + let metadataSubscriptionDate = metadataSubscriptionDates[subscriptionId] { + if lastPulledMetadataEvents < metadataSubscriptionDate { + relaySubscriptionMetadata.lastPulledMetadataEvents = metadataSubscriptionDate + } + metadataSubscriptionDates.removeValue(forKey: subscriptionId) + } + + if let relaySubscriptionMetadata, + let lastBootstrapped = relaySubscriptionMetadata.lastBootstrapped, + let bootstrapSubscriptionDate = bootstrapSubscriptionDates[subscriptionId] { + if lastBootstrapped < bootstrapSubscriptionDate { + relaySubscriptionMetadata.lastPulledMetadataEvents = bootstrapSubscriptionDate + } + bootstrapSubscriptionDates.removeValue(forKey: subscriptionId) + } + + if let relaySubscriptionMetadata, + let lastPulledAllTimeBasedCalendarEvents = relaySubscriptionMetadata.lastPulledAllTimeBasedCalendarEvents, + let timeBasedCalendarEventSubscriptionDate = timeBasedCalendarEventSubscriptionDates[subscriptionId] { + if lastPulledAllTimeBasedCalendarEvents < timeBasedCalendarEventSubscriptionDate { + relaySubscriptionMetadata.lastPulledAllTimeBasedCalendarEvents = timeBasedCalendarEventSubscriptionDate + } + timeBasedCalendarEventSubscriptionDates.removeValue(forKey: subscriptionId) + } + } + } enum HomeTabs { diff --git a/Comingle/Models/RelaySubscriptionMetadata.swift b/Comingle/Models/RelaySubscriptionMetadata.swift new file mode 100644 index 0000000..bffbf0c --- /dev/null +++ b/Comingle/Models/RelaySubscriptionMetadata.swift @@ -0,0 +1,20 @@ +// +// RelaySubscriptionMetadata.swift +// Comingle +// +// Created by Terry Yiu on 8/3/24. +// + +import Foundation +import SwiftData + +@Model +final class RelaySubscriptionMetadata { + var lastBootstrapped: Date? + var lastPulledAllTimeBasedCalendarEvents: Date? + var lastPulledMetadataEvents: Date? + + init() { + + } +} diff --git a/Comingle/Views/EventListView.swift b/Comingle/Views/EventListView.swift index a2eef68..d9f6b34 100644 --- a/Comingle/Views/EventListView.swift +++ b/Comingle/Views/EventListView.swift @@ -85,6 +85,14 @@ struct EventListView: View { } ) .padding(.vertical, 10) + .onAppear { + var pubkeysToPullMetadata = [event.pubkey] + event.participants.compactMap { $0.pubkey?.hex } + if let calendarEventCoordinates = event.replaceableEventCoordinates()?.tag.value, + let rsvps = appState.calendarEventsToRsvps[calendarEventCoordinates] { + pubkeysToPullMetadata += rsvps.map { $0.pubkey } + } + appState.pullMissingMetadata(pubkeysToPullMetadata) + } } } } diff --git a/Comingle/Views/EventView.swift b/Comingle/Views/EventView.swift index 7fe4e50..f4329a8 100644 --- a/Comingle/Views/EventView.swift +++ b/Comingle/Views/EventView.swift @@ -681,44 +681,42 @@ struct EventView: View, EventCreating { .presentationDragIndicator(.visible) } .task { - var pubkeysToPullMetadata = event?.participants.compactMap { $0.pubkey?.hex } ?? [] + refresh() + } + .refreshable { + refresh() + } + } - if let rsvps = appState.calendarEventsToRsvps[eventCoordinates.tag.value] { - pubkeysToPullMetadata += rsvps.map { $0.pubkey } + func refresh() { + if let event { + let calendarEventCoordinates = eventCoordinates.tag.value + guard let eventFilter = Filter( + authors: [event.pubkey], + kinds: [EventKind.timeBasedCalendarEvent.rawValue], + tags: ["d": [calendarEventCoordinates]], + since: Int(event.createdAt) + ) else { + print("Unable to create time-based calendar event filter.") + return } + _ = appState.relayReadPool.subscribe(with: eventFilter) + var pubkeysToPullMetadata = [event.pubkey] + event.participants.compactMap { $0.pubkey?.hex } + if let calendarEventCoordinates = event.replaceableEventCoordinates()?.tag.value, + let rsvps = appState.calendarEventsToRsvps[calendarEventCoordinates] { + pubkeysToPullMetadata += rsvps.map { $0.pubkey } + } appState.pullMissingMetadata(pubkeysToPullMetadata) - } - .refreshable { - if let event = event { - let calendarEventCoordinates = eventCoordinates.tag.value - guard let eventFilter = Filter( - authors: [event.pubkey], - kinds: [EventKind.timeBasedCalendarEvent.rawValue], - tags: ["d": [calendarEventCoordinates]], - since: Int(event.createdAt) - ) else { - print("Unable to create time-based calendar event filter.") - return - } - _ = appState.relayReadPool.subscribe(with: eventFilter) - var pubkeysToPullMetadata = [event.pubkey] + event.participants.compactMap { $0.pubkey?.hex } - if let calendarEventCoordinates = event.replaceableEventCoordinates()?.tag.value, - let rsvps = appState.calendarEventsToRsvps[calendarEventCoordinates] { - pubkeysToPullMetadata += rsvps.map { $0.pubkey } - } - appState.pullMissingMetadata(pubkeysToPullMetadata) - - guard let rsvpFilter = Filter( - kinds: [EventKind.calendarEventRSVP.rawValue], - tags: ["a": [calendarEventCoordinates]]) - else { - print("Unable to create calendar event RSVP filter.") - return - } - _ = appState.relayReadPool.subscribe(with: rsvpFilter) + guard let rsvpFilter = Filter( + kinds: [EventKind.calendarEventRSVP.rawValue], + tags: ["a": [calendarEventCoordinates]]) + else { + print("Unable to create calendar event RSVP filter.") + return } + _ = appState.relayReadPool.subscribe(with: rsvpFilter) } } }