Skip to content

Commit

Permalink
Aggressively optimize queries to relays
Browse files Browse the repository at this point in the history
  • Loading branch information
tyiu committed Aug 4, 2024
1 parent a1467c8 commit 153f082
Show file tree
Hide file tree
Showing 6 changed files with 265 additions and 89 deletions.
4 changes: 4 additions & 0 deletions Comingle.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
3A6451852C44C879000DC75B /* RelaySettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3A6451842C44C879000DC75B /* RelaySettings.swift */; };
3A7D25DE2C5D12AE001C1714 /* Launch Screen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3A7D25DD2C5D12AE001C1714 /* Launch Screen.storyboard */; };
3ABD0C0D2C5EF6A9004A15C7 /* String+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3ABD0C0C2C5EF6A9004A15C7 /* String+Extensions.swift */; };
3ABD0C152C5F2508004A15C7 /* RelaySubscriptionMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand Down Expand Up @@ -142,6 +143,7 @@
3A7D25D12C5D0CAA001C1714 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
3A7D25DD2C5D12AE001C1714 /* Launch Screen.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = "Launch Screen.storyboard"; sourceTree = "<group>"; };
3ABD0C0C2C5EF6A9004A15C7 /* String+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+Extensions.swift"; sourceTree = "<group>"; };
3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelaySubscriptionMetadata.swift; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
Expand Down Expand Up @@ -272,6 +274,7 @@
3A3419492C50A36B0039620B /* PersistentNostrEvent.swift */,
3A464FFB2C3A312200AD4B09 /* Profile.swift */,
3A317C182C59C130004DA56B /* PublicKeySortComparator.swift */,
3ABD0C142C5F2508004A15C7 /* RelaySubscriptionMetadata.swift */,
3A317C1B2C59C5EA004DA56B /* RSVPSortComparator.swift */,
3A317C122C594AFE004DA56B /* SearchViewModel.swift */,
3A4FCB532C365C62007AE9A4 /* TimeBasedCalendarEventSortComparator.swift */,
Expand Down Expand Up @@ -530,6 +533,7 @@
3A317C242C5A71E9004DA56B /* MKAutocompleteManager.swift in Sources */,
3A34AAAA2B491DE900C5F910 /* SwiftUI+LocalizedStringResource.swift in Sources */,
3A317C172C59BFAE004DA56B /* CalendarEventParticipantSortComparator.swift in Sources */,
3ABD0C152C5F2508004A15C7 /* RelaySubscriptionMetadata.swift in Sources */,
3A4A898A2C3C240C00238043 /* ProfilePictureAndNameView.swift in Sources */,
3A317C272C5B4260004DA56B /* LocationSearchView.swift in Sources */,
3A4A89942C3CE40A00238043 /* GuestProfilePictureView.swift in Sources */,
Expand Down
2 changes: 1 addition & 1 deletion Comingle/ComingleApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ struct ComingleApp: App {
init() {
NostrEventValueTransformer.register()
do {
container = try ModelContainer(for: AppSettings.self, PersistentNostrEvent.self)
container = try ModelContainer(for: RelaySubscriptionMetadata.self, AppSettings.self, PersistentNostrEvent.self)
appState = AppState(modelContext: container.mainContext)
} catch {
fatalError("Failed to create ModelContainer for AppSettings and PersistentNostrEvent.")
Expand Down
258 changes: 202 additions & 56 deletions Comingle/Controllers/AppState.swift
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ class AppState: ObservableObject, Hashable {

@Published var metadataTrie = Trie<MetadataEvent>()

// Keep track of relay pool active subscriptions and the until filter so that we can limit the scope of how much we query from the relay pools.
var metadataSubscriptionCounts = [String: Int]()
var metadataSubscriptionDates = [String: Date]()
var bootstrapSubscriptionCounts = [String: Int]()
var bootstrapSubscriptionDates = [String: Date]()
var timeBasedCalendarEventSubscriptionCounts = [String: Int]()
var timeBasedCalendarEventSubscriptionDates = [String: Date]()

init(modelContext: ModelContext) {
self.modelContext = modelContext
}
Expand Down Expand Up @@ -222,6 +230,23 @@ class AppState: ObservableObject, Hashable {
)
return try? modelContext.fetch(descriptor).first
}

var relaySubscriptionMetadata: RelaySubscriptionMetadata? {
let descriptor = FetchDescriptor<RelaySubscriptionMetadata>()

if let result = try? modelContext.fetch(descriptor).first {
return result
} else {
let result = RelaySubscriptionMetadata()
modelContext.insert(result)
do {
try modelContext.save()
} catch {
print("Unable to save initial RelaySubscriptionMetadata object.")
}
return result
}
}
}

extension AppState: EventVerifying, RelayDelegate {
Expand All @@ -233,80 +258,158 @@ extension AppState: EventVerifying, RelayDelegate {
}

func pullMissingMetadata(_ pubkeys: [String]) {
let pubkeysToFetchMetadata = Set(pubkeys).filter { self.metadataEvents[$0] == nil }
// There has to be at least one connected relay to be able to pull metadata.
guard !relayReadPool.relays.isEmpty && relayReadPool.relays.contains(where: { $0.state == .connected }) else {
return
}

let relaySubscriptionMetadata = relaySubscriptionMetadata

let since: Int?
if let lastPulledMetadataEvents = relaySubscriptionMetadata?.lastPulledMetadataEvents {
since = Int(lastPulledMetadataEvents.timeIntervalSince1970) + 1
} else {
since = nil
}
let until = Date.now

let allPubkeysSet = Set(pubkeys)
let pubkeysToFetchMetadata = allPubkeysSet.filter { self.metadataEvents[$0] == nil }
if !pubkeysToFetchMetadata.isEmpty {
guard let metadataFilter = Filter(
guard let missingMetadataFilter = Filter(
authors: Array(pubkeysToFetchMetadata),
kinds: [EventKind.metadata.rawValue]
) else {
print("Unable to create metadata filter for \(pubkeysToFetchMetadata).")
print("Unable to create missing metadata filter for \(pubkeysToFetchMetadata).")
return
}

_ = relayReadPool.subscribe(with: metadataFilter)
_ = relayReadPool.subscribe(with: missingMetadataFilter)
}

if !metadataSubscriptionCounts.isEmpty {
// Do not refresh metadata if one is already in progress.
return
}

let pubkeysToRefresh = allPubkeysSet.subtracting(pubkeysToFetchMetadata)
guard let metadataRefreshFilter = Filter(
authors: Array(pubkeysToRefresh),
kinds: [EventKind.metadata.rawValue],
since: since
) else {
print("Unable to create refresh metadata filter for \(pubkeysToRefresh).")
return
}

relaySubscriptionMetadata?.lastPulledMetadataEvents = until
_ = relayReadPool.subscribe(with: metadataRefreshFilter)

}

/// Subscribe with filter to relay if provided, or use relay read pool if not.
func subscribe(filter: Filter, relay: Relay? = nil) throws -> String? {
if let relay {
do {
return try relay.subscribe(with: filter)
} catch {
print("Could not subscribe to relay with filter.")
return nil
}
} else {
return relayReadPool.subscribe(with: filter)
}
}

func refresh(relay: Relay? = nil) {
guard (relay == nil && !relayReadPool.relays.isEmpty) || relay?.state == .connected else {
guard (relay == nil && !relayReadPool.relays.isEmpty && relayReadPool.relays.contains(where: { $0.state == .connected })) || relay?.state == .connected else {
return
}

let authors = profiles.compactMap({ $0.publicKeyHex })
if !authors.isEmpty {
guard let bootstrapFilter = Filter(
authors: authors,
kinds: [EventKind.metadata.rawValue, EventKind.followList.rawValue, EventKind.timeBasedCalendarEvent.rawValue, EventKind.calendarEventRSVP.rawValue, EventKind.deletion.rawValue]
) else {
print("Unable to create the boostrap filter.")
return
}
let relaySubscriptionMetadata = relaySubscriptionMetadata
let until = Date.now

if bootstrapSubscriptionCounts.isEmpty {
let authors = profiles.compactMap({ $0.publicKeyHex })
if !authors.isEmpty {
let since: Int?
if let lastBootstrapped = relaySubscriptionMetadata?.lastBootstrapped {
since = Int(lastBootstrapped.timeIntervalSince1970) + 1
} else {
since = nil
}

guard let bootstrapFilter = Filter(
authors: authors,
kinds: [EventKind.metadata.rawValue, EventKind.followList.rawValue, EventKind.timeBasedCalendarEvent.rawValue, EventKind.calendarEventRSVP.rawValue, EventKind.deletion.rawValue],
since: since
) else {
print("Unable to create the boostrap filter.")
return
}

if let relay {
do {
try relay.subscribe(with: bootstrapFilter)
if let bootstrapSubscriptionId = try subscribe(filter: bootstrapFilter, relay: relay), relay == nil {
if let bootstrapSubscriptionCount = bootstrapSubscriptionCounts[bootstrapSubscriptionId] {
bootstrapSubscriptionCounts[bootstrapSubscriptionId] = bootstrapSubscriptionCount + 1
} else {
bootstrapSubscriptionCounts[bootstrapSubscriptionId] = 1
}
}
} catch {
print("Could not subscribe to relay with the boostrap filter.")
}
} else {
_ = relayReadPool.subscribe(with: bootstrapFilter)
}
}

guard let timeBasedCalendarEventFilter = Filter(
kinds: [EventKind.timeBasedCalendarEvent.rawValue]
) else {
print("Unable to create the time-based calendar event filter.")
return
}
if timeBasedCalendarEventSubscriptionCounts.isEmpty {
let since: Int?
if let lastPulledAllTimeBasedCalendarEvents = relaySubscriptionMetadata?.lastPulledAllTimeBasedCalendarEvents {
since = Int(lastPulledAllTimeBasedCalendarEvents.timeIntervalSince1970) + 1
} else {
since = nil
}

guard let timeBasedCalendarEventFilter = Filter(
kinds: [EventKind.timeBasedCalendarEvent.rawValue],
since: since
) else {
print("Unable to create the time-based calendar event filter.")
return
}

if let relay {
do {
try relay.subscribe(with: timeBasedCalendarEventFilter)
if let timeBasedCalendarEventSubscriptionId = try subscribe(filter: timeBasedCalendarEventFilter, relay: relay), relay == nil {
if let timeBasedCalendarEventSubscriptionCount = timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] {
timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] = timeBasedCalendarEventSubscriptionCount + 1
} else {
timeBasedCalendarEventSubscriptionCounts[timeBasedCalendarEventSubscriptionId] = 1
}
}
} catch {
print("Could not subscribe to relay with the time-based calendar event filter.")
}
} else {
_ = relayReadPool.subscribe(with: timeBasedCalendarEventFilter)
}
}

private func didReceiveFollowListEvent(_ followListEvent: FollowListEvent, shouldPullMissingMetadata: Bool = false) {
if let existingFollowList = self.followListEvents[followListEvent.pubkey] {
if existingFollowList.createdAt < followListEvent.createdAt {
cache(followListEvent)
cache(followListEvent, shouldPullMissingMetadata: shouldPullMissingMetadata)
}
} else {
cache(followListEvent)
cache(followListEvent, shouldPullMissingMetadata: shouldPullMissingMetadata)
}
}

private func cache(_ followListEvent: FollowListEvent, shouldPullMissingMetadata: Bool) {
self.followListEvents[followListEvent.pubkey] = followListEvent

if shouldPullMissingMetadata {
pullMissingMetadata(followListEvent.followedPubkeys)
}
}

private func cache(_ followListEvent: FollowListEvent) {
self.followListEvents[followListEvent.pubkey] = followListEvent
// TODO Here or elsewhere. Query for calendar events that follows who have RSVP'd.
}

private func didReceiveMetadataEvent(_ metadataEvent: MetadataEvent) {
Expand Down Expand Up @@ -336,7 +439,7 @@ extension AppState: EventVerifying, RelayDelegate {
}
}

private func didReceiveTimeBasedCalendarEvent(_ timeBasedCalendarEvent: TimeBasedCalendarEvent, shouldPullMissingMetadata: Bool = false) {
private func didReceiveTimeBasedCalendarEvent(_ timeBasedCalendarEvent: TimeBasedCalendarEvent) {
guard let eventCoordinates = timeBasedCalendarEvent.replaceableEventCoordinates()?.tag.value,
let startTimestamp = timeBasedCalendarEvent.startTimestamp,
startTimestamp <= timeBasedCalendarEvent.endTimestamp ?? startTimestamp,
Expand All @@ -351,27 +454,6 @@ extension AppState: EventVerifying, RelayDelegate {
} else {
timeBasedCalendarEvents[eventCoordinates] = timeBasedCalendarEvent
}

if shouldPullMissingMetadata {
pullMissingMetadata([timeBasedCalendarEvent.pubkey])
}

guard let replaceableEventCoordinates = timeBasedCalendarEvent.replaceableEventCoordinates() else {
print("Unable to get replaceable event coordinates for time-based calendar event.")
return
}

let replaceableEventCoordinatesTag = replaceableEventCoordinates.tag

guard let rsvpFilter = Filter(
kinds: [EventKind.calendarEventRSVP.rawValue],
tags: ["a": [replaceableEventCoordinatesTag.value]])
else {
print("Unable to create calendar event RSVP filter.")
return
}

_ = relayReadPool.subscribe(with: rsvpFilter)
}

func updateCalendarEventRSVP(_ rsvp: CalendarEventRSVP, rsvpEventCoordinates: String) {
Expand Down Expand Up @@ -511,7 +593,7 @@ extension AppState: EventVerifying, RelayDelegate {
case let metadataEvent as MetadataEvent:
self.didReceiveMetadataEvent(metadataEvent)
case let timeBasedCalendarEvent as TimeBasedCalendarEvent:
self.didReceiveTimeBasedCalendarEvent(timeBasedCalendarEvent, shouldPullMissingMetadata: true)
self.didReceiveTimeBasedCalendarEvent(timeBasedCalendarEvent)
case let rsvpEvent as CalendarEventRSVP:
self.didReceiveCalendarEventRSVP(rsvpEvent)
case let deletionEvent as DeletionEvent:
Expand Down Expand Up @@ -551,6 +633,10 @@ extension AppState: EventVerifying, RelayDelegate {
// Live new events are not strictly needed for this app for now.
// In the future, we could keep subscriptions open for updates.
try? relay.closeSubscription(with: subscriptionId)
updateRelaySubscriptionMetadataTimestamps(with: subscriptionId)
updateRelaySubscriptionCounts(closedSubscriptionId: subscriptionId)
case let .closed(subscriptionId, _):
updateRelaySubscriptionCounts(closedSubscriptionId: subscriptionId)
case let .ok(eventId, success, message):
if success {
if let persistentNostrEvent = persistentNostrEvent(eventId), !persistentNostrEvent.relays.contains(relay.url) {
Expand All @@ -564,6 +650,66 @@ extension AppState: EventVerifying, RelayDelegate {
}
}

func updateRelaySubscriptionCounts(closedSubscriptionId: String) {
if let metadataSubscriptionCount = metadataSubscriptionCounts[closedSubscriptionId] {
if metadataSubscriptionCount <= 1 {
metadataSubscriptionCounts.removeValue(forKey: closedSubscriptionId)
} else {
metadataSubscriptionCounts[closedSubscriptionId] = metadataSubscriptionCount - 1
}
}

if let bootstrapSubscriptionCount = bootstrapSubscriptionCounts[closedSubscriptionId] {
if bootstrapSubscriptionCount <= 1 {
bootstrapSubscriptionCounts.removeValue(forKey: closedSubscriptionId)
} else {
bootstrapSubscriptionCounts[closedSubscriptionId] = bootstrapSubscriptionCount - 1
}
}

if let timeBasedCalendarEventSubscriptionCount = timeBasedCalendarEventSubscriptionCounts[closedSubscriptionId] {
if timeBasedCalendarEventSubscriptionCount <= 1 {
timeBasedCalendarEventSubscriptionCounts.removeValue(forKey: closedSubscriptionId)

// Wait until we have fetched all the time-based calendar events before fetching metadata in bulk.
pullMissingMetadata(timeBasedCalendarEvents.values.map { $0.pubkey })
} else {
timeBasedCalendarEventSubscriptionCounts[closedSubscriptionId] = timeBasedCalendarEventSubscriptionCount - 1
}
}
}

func updateRelaySubscriptionMetadataTimestamps(with subscriptionId: String) {
let relaySubscriptionMetadata = relaySubscriptionMetadata

if let relaySubscriptionMetadata,
let lastPulledMetadataEvents = relaySubscriptionMetadata.lastPulledMetadataEvents,
let metadataSubscriptionDate = metadataSubscriptionDates[subscriptionId] {
if lastPulledMetadataEvents < metadataSubscriptionDate {
relaySubscriptionMetadata.lastPulledMetadataEvents = metadataSubscriptionDate
}
metadataSubscriptionDates.removeValue(forKey: subscriptionId)
}

if let relaySubscriptionMetadata,
let lastBootstrapped = relaySubscriptionMetadata.lastBootstrapped,
let bootstrapSubscriptionDate = bootstrapSubscriptionDates[subscriptionId] {
if lastBootstrapped < bootstrapSubscriptionDate {
relaySubscriptionMetadata.lastPulledMetadataEvents = bootstrapSubscriptionDate
}
bootstrapSubscriptionDates.removeValue(forKey: subscriptionId)
}

if let relaySubscriptionMetadata,
let lastPulledAllTimeBasedCalendarEvents = relaySubscriptionMetadata.lastPulledAllTimeBasedCalendarEvents,
let timeBasedCalendarEventSubscriptionDate = timeBasedCalendarEventSubscriptionDates[subscriptionId] {
if lastPulledAllTimeBasedCalendarEvents < timeBasedCalendarEventSubscriptionDate {
relaySubscriptionMetadata.lastPulledAllTimeBasedCalendarEvents = timeBasedCalendarEventSubscriptionDate
}
timeBasedCalendarEventSubscriptionDates.removeValue(forKey: subscriptionId)
}
}

}

enum HomeTabs {
Expand Down
Loading

0 comments on commit 153f082

Please sign in to comment.