From e353c1c06f969bbcda48c77fecd0febf8a1eaa66 Mon Sep 17 00:00:00 2001
From: Pablo Fernandez
Date: Tue, 25 Jun 2024 11:53:13 +0100
Subject: [PATCH] improve url normalization and be more careful on connection
management
---
.../src/lib/event/content/EventContent.svelte | 4 +-
ndk/src/events/fetch-tagged-event.ts | 4 +-
ndk/src/events/index.ts | 14 +--
ndk/src/events/kinds/NDKRelayList.ts | 8 +-
ndk/src/index.ts | 2 +-
ndk/src/ndk/fetch-event-from-tag.test.ts | 21 +++++
ndk/src/ndk/fetch-event-from-tag.ts | 85 +++++++++++++++++--
ndk/src/ndk/index.ts | 19 ++---
ndk/src/outbox/tracker.ts | 8 +-
ndk/src/relay/connectivity.ts | 24 +++---
ndk/src/relay/index.ts | 6 +-
ndk/src/relay/pool/index.ts | 19 ++++-
ndk/src/utils/get-users-relay-list.ts | 16 ++--
ndk/src/utils/normalize-url.ts | 8 ++
ndk/src/zap/index.test.ts | 57 ++++++++++---
ndk/src/zap/index.ts | 12 +--
16 files changed, 231 insertions(+), 76 deletions(-)
create mode 100644 ndk/src/ndk/fetch-event-from-tag.test.ts
diff --git a/ndk-svelte-components/src/lib/event/content/EventContent.svelte b/ndk-svelte-components/src/lib/event/content/EventContent.svelte
index d37757a2..fae012ba 100644
--- a/ndk-svelte-components/src/lib/event/content/EventContent.svelte
+++ b/ndk-svelte-components/src/lib/event/content/EventContent.svelte
@@ -32,6 +32,8 @@
* Optional content to use instead of the one from the event
*/
export let content = event?.content;
+
+ const markdownKinds = [ NDKKind.Article, 30818, 30041 ]
{#if event}
@@ -49,7 +51,7 @@
{:else if event.kind === 30001}
- {:else if event.kind === 30023 || event.kind === 30818}
+ {:else if markdownKinds.includes(event.kind)}
tag[0] === tagName);
-
- if (marker !== undefined) return t;
+ const t = this.tags.filter((tag) => tag[0] === tagName);
+
+ if (marker === undefined) return t;
return t.filter((tag) => tag[3] === marker);
}
@@ -420,7 +421,10 @@ export class NDKEvent extends EventEmitter {
}
}
- if ((this.ndk?.clientName || this.ndk?.clientNip89)) {
+ if (
+ (this.ndk?.clientName || this.ndk?.clientNip89) &&
+ skipClientTagOnKinds.includes(this.kind!)
+ ) {
if (!this.tags.some((tag) => tag[0] === "client")) {
const clientTag: NDKTag = ["client", this.ndk.clientName ?? ""];
if (this.ndk.clientNip89) clientTag.push(this.ndk.clientNip89);
diff --git a/ndk/src/events/kinds/NDKRelayList.ts b/ndk/src/events/kinds/NDKRelayList.ts
index de53a326..cb564ae2 100644
--- a/ndk/src/events/kinds/NDKRelayList.ts
+++ b/ndk/src/events/kinds/NDKRelayList.ts
@@ -3,7 +3,7 @@ import type { NostrEvent } from "../index.js";
import { NDKEvent } from "../index.js";
import type { NDK } from "../../ndk/index.js";
import { NDKRelaySet } from "../../relay/sets/index.js";
-import { normalizeRelayUrl } from "../../utils/normalize-url.js";
+import { normalizeRelayUrl, tryNormalizeRelayUrl } from "../../utils/normalize-url.js";
const READ_MARKER = "read";
const WRITE_MARKER = "write";
@@ -26,7 +26,8 @@ export class NDKRelayList extends NDKEvent {
return this.tags
.filter((tag) => tag[0] === "r" || tag[0] === "relay")
.filter((tag) => !tag[2] || (tag[2] && tag[2] === READ_MARKER))
- .map((tag) => normalizeRelayUrl(tag[1]));
+ .map((tag) => tryNormalizeRelayUrl(tag[1]))
+ .filter((url) => !!url) as WebSocket["url"][];
}
set readRelayUrls(relays: WebSocket["url"][]) {
@@ -39,7 +40,8 @@ export class NDKRelayList extends NDKEvent {
return this.tags
.filter((tag) => tag[0] === "r" || tag[0] === "relay")
.filter((tag) => !tag[2] || (tag[2] && tag[2] === WRITE_MARKER))
- .map((tag) => normalizeRelayUrl(tag[1]));
+ .map((tag) => tryNormalizeRelayUrl(tag[1]))
+ .filter((url) => !!url) as WebSocket["url"][];
}
set writeRelayUrls(relays: WebSocket["url"][]) {
diff --git a/ndk/src/index.ts b/ndk/src/index.ts
index ce0c43ee..5e5c6e8e 100644
--- a/ndk/src/index.ts
+++ b/ndk/src/index.ts
@@ -46,4 +46,4 @@ export { NDK as default, NDKConstructorParams } from "./ndk/index.js";
export { NDKZapInvoice, zapInvoiceFromEvent } from "./zap/invoice.js";
export * from "./zap/index.js";
export * from "./utils/normalize-url.js";
-export * from './utils/get-users-relay-list.js';
\ No newline at end of file
+export * from "./utils/get-users-relay-list.js";
diff --git a/ndk/src/ndk/fetch-event-from-tag.test.ts b/ndk/src/ndk/fetch-event-from-tag.test.ts
new file mode 100644
index 00000000..40800637
--- /dev/null
+++ b/ndk/src/ndk/fetch-event-from-tag.test.ts
@@ -0,0 +1,21 @@
+import { NDK } from ".";
+import { NDKEvent } from "../events";
+import { NDKSubscriptionCacheUsage, NDKSubscriptionOptions } from "../subscription";
+
+const ndk = new NDK();
+
+describe("fetchEventFromTag", () => {
+ describe("with subOpts specifying only cache", () => {
+ it("does not try to load a relay", async () => {
+ const originalEvent = new NDKEvent();
+ const tag = ["e", "id", "hint"];
+ originalEvent.tags.push(tag);
+ const subOpts: NDKSubscriptionOptions = {
+ cacheUsage: NDKSubscriptionCacheUsage.ONLY_CACHE,
+ };
+ jest.spyOn(ndk.pool, "getRelay");
+ const event = await ndk.fetchEventFromTag(tag, originalEvent, subOpts);
+ expect(ndk.pool.getRelay).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/ndk/src/ndk/fetch-event-from-tag.ts b/ndk/src/ndk/fetch-event-from-tag.ts
index 8bf7311c..3c25dd55 100644
--- a/ndk/src/ndk/fetch-event-from-tag.ts
+++ b/ndk/src/ndk/fetch-event-from-tag.ts
@@ -1,6 +1,8 @@
import { NDK } from ".";
import { NDKEvent, NDKTag } from "../events";
-import type { NDKRelaySet } from "../relay/sets";
+import { getWriteRelaysFor } from "../outbox/read/with-authors";
+import { NDKRelaySet } from "../relay/sets";
+import { calculateRelaySetsFromFilters } from "../relay/sets/calculate";
import { NDKSubscriptionOptions } from "../subscription";
/**
@@ -35,31 +37,98 @@ export type NDKFetchFallbackOptions = {
timeout?: number;
};
+function isValidHint(hint: string | undefined) {
+ if (!hint || hint === "") return false;
+
+ // Check if the hint is a valid URL
+ try {
+ new URL(hint);
+ return true;
+ } catch (e) {
+ return false;
+ }
+}
+
+function isRelayHintConnected(ndk: NDK, hint: string | undefined) {
+ if (!isValidHint(hint)) return false;
+
+ return ndk.pool.isRelayConnected(hint!);
+}
+
+/**
+ * @ignore
+ */
export async function fetchEventFromTag(
this: NDK,
tag: NDKTag,
+ originalEvent: NDKEvent,
subOpts?: NDKSubscriptionOptions,
fallback: NDKFetchFallbackOptions = {
type: "timeout",
}
) {
const d = this.debug.extend("fetch-event-from-tag");
- const [tagType, id, hint] = tag;
+ const [_, id, hint] = tag;
+
+ // If we are supposed to stick to the cache, just go with that
+ // if (subOpts?.cacheUsage === NDKSubscriptionCacheUsage.ONLY_CACHE) {
+ // return this.fetchEvent(id, subOpts);
+ // }
+
+ // XXXXX
+ subOpts = {};
+ if (!isValidHint(hint)) return;
+
+ console;
+ d("fetching event from tag", tag, subOpts, fallback);
+
+ // If we are connected to the relay hint, try exclusively from that relay
+ // if (isRelayHintConnected(this, hint)) {
+ // d("fetching event from connected relay hint (%s)", normalizeRelayUrl(hint));
+ // let event = await this.fetchEvent(id, subOpts, this.pool.getRelay(hint));
+ // if (event) return event;
+ // }
+
+ // Check if we have a relay list for the author of the original event
+ // and prefer to use those relays
+ const authorRelays = getWriteRelaysFor(this, originalEvent.pubkey);
+ if (authorRelays && authorRelays.size > 0) {
+ d("fetching event from author relays %o", Array.from(authorRelays));
+ const relaySet = NDKRelaySet.fromRelayUrls(Array.from(authorRelays), this);
+ let event = await this.fetchEvent(id, subOpts, relaySet);
+ if (event) return event;
+ } else {
+ d("no author relays found for %s", originalEvent.pubkey, originalEvent);
+ }
+
+ // Attempt without relay hint on whatever NDK calculates
+ const relaySet = calculateRelaySetsFromFilters(this, [{ ids: [id] }], this.pool);
+ d("fetching event without relay hint", relaySet);
+ let event = await this.fetchEvent(id, subOpts);
+ if (event) return event;
+
+ // If we didn't get the event, try to fetch in the relay hint
+ if (hint && hint !== "") {
+ let event = await this.fetchEvent(
+ id,
+ subOpts,
+ this.pool.getRelay(hint, true, true, [{ ids: [id] }])
+ );
+ if (event) return event;
+ }
let result: NDKEvent | null | undefined = undefined;
- let relay =
- hint && hint !== "" ? this.pool.getRelay(hint, true, true, [{ ids: [id] }]) : undefined;
+ let relay = isValidHint(hint)
+ ? this.pool.getRelay(hint, false, true, [{ ids: [id] }])
+ : undefined;
- /**
- * Fetch with (maybe) a relay hint.
- */
const fetchMaybeWithRelayHint = new Promise((resolve) => {
this.fetchEvent(id, subOpts, relay).then(resolve);
});
// if we don't have a relay hint we don't need to setup a fallback
- if (hint === "" || !hint || fallback.type === "none") {
+ if (!isValidHint(hint) || fallback.type === "none") {
return fetchMaybeWithRelayHint;
}
diff --git a/ndk/src/ndk/index.ts b/ndk/src/ndk/index.ts
index c44c940d..590716b0 100644
--- a/ndk/src/ndk/index.ts
+++ b/ndk/src/ndk/index.ts
@@ -139,7 +139,7 @@ export interface GetUserParams extends NDKUserParams {
hexpubkey?: string;
}
-export const DEFAULT_OUTBOX_RELAYS = ["wss://purplepag.es/", "wss://profiles.nos.social/"];
+export const DEFAULT_OUTBOX_RELAYS = ["wss://purplepag.es/", "wss://nos.lol/"];
/**
* TODO: Move this to a outbox policy
@@ -491,9 +491,12 @@ export class NDK extends EventEmitter<{
}
/**
- * Fetches event following a tag
- * @param tag
- * @param subOpts
+ * Attempts to fetch an event from a tag, following relay hints and
+ * other best practices.
+ * @param tag Tag to fetch the event from
+ * @param originalEvent Event where the tag came from
+ * @param subOpts Subscription options to use when fetching the event
+ * @param fallback Fallback options to use when the hint relay doesn't respond
* @returns
*/
public fetchEventFromTag = fetchEventFromTag.bind(this);
@@ -701,12 +704,6 @@ export class NDK extends EventEmitter<{
const zap = new NDKZap(zapOpts);
- return zap.createZapRequest(
- amount,
- comment,
- extraTags,
- undefined,
- signer
- );
+ return zap.createZapRequest(amount, comment, extraTags, undefined, signer);
}
}
diff --git a/ndk/src/outbox/tracker.ts b/ndk/src/outbox/tracker.ts
index a046ee55..35af8869 100644
--- a/ndk/src/outbox/tracker.ts
+++ b/ndk/src/outbox/tracker.ts
@@ -65,7 +65,7 @@ export class OutboxTracker extends EventEmitter {
});
}
- public trackUsers(items: NDKUser[] | Hexpubkey[]) {
+ public trackUsers(items: NDKUser[] | Hexpubkey[], skipCache = false) {
for (let i = 0; i < items.length; i += 400) {
const slice = items.slice(i, i + 400);
let pubkeys = slice
@@ -80,7 +80,7 @@ export class OutboxTracker extends EventEmitter {
this.data.set(pubkey, new OutboxItem("user"));
}
- getRelayListForUsers(pubkeys, this.ndk).then(
+ getRelayListForUsers(pubkeys, this.ndk, skipCache).then(
(relayLists: Map) => {
for (const [pubkey, relayList] of relayLists) {
const outboxItem = this.data.get(pubkey)!;
@@ -112,7 +112,7 @@ export class OutboxTracker extends EventEmitter {
this.data.set(pubkey, outboxItem);
// this.debug(
- // `Adding ${outboxItem.readRelays.size} read relays and ${outboxItem.writeRelays.size} write relays for ${user.pubkey}`
+ // `Adding ${outboxItem.readRelays.size} read relays and ${outboxItem.writeRelays.size} write relays for ${pubkey}, %o`, relayList?.rawEvent()
// );
}
}
@@ -126,7 +126,7 @@ export class OutboxTracker extends EventEmitter {
* @param key
* @param score
*/
- public track(item: NDKUser | Hexpubkey, type?: OutboxItemType): OutboxItem {
+ public track(item: NDKUser | Hexpubkey, type?: OutboxItemType, skipCache = true): OutboxItem {
const key = getKeyFromItem(item);
type ??= getTypeFromItem(item);
let outboxItem = this.data.get(key);
diff --git a/ndk/src/relay/connectivity.ts b/ndk/src/relay/connectivity.ts
index 8e4f11fa..b732c854 100644
--- a/ndk/src/relay/connectivity.ts
+++ b/ndk/src/relay/connectivity.ts
@@ -62,7 +62,7 @@ export class NDKRelayConnectivity {
const authHandler = async (challenge: string) => {
const authPolicy = this.ndkRelay.authPolicy ?? this.ndk?.relayAuthDefaultPolicy;
-
+
this.debug("Relay requested authentication", {
havePolicy: !!authPolicy,
});
@@ -79,15 +79,19 @@ export class NDKRelayConnectivity {
});
}
- if (this._status === NDKRelayStatus.AUTHENTICATING) {
- this.debug("Authentication policy finished");
- this.relay.auth(async (evt: EventTemplate): Promise => {
- const event = new NDKEvent(this.ndk, evt as NostrEvent);
- await event.sign();
- return event.rawEvent() as VerifiedEvent;
- });
- this._status = NDKRelayStatus.CONNECTED;
- this.ndkRelay.emit("authed");
+ if (res === true) {
+ if (!this.ndk?.signer) {
+ throw new Error("No signer available for authentication");
+ } else if (this._status === NDKRelayStatus.AUTHENTICATING) {
+ this.debug("Authentication policy finished");
+ this.relay.auth(async (evt: EventTemplate): Promise => {
+ const event = new NDKEvent(this.ndk, evt as NostrEvent);
+ await event.sign();
+ return event.rawEvent() as VerifiedEvent;
+ });
+ this._status = NDKRelayStatus.CONNECTED;
+ this.ndkRelay.emit("authed");
+ }
}
}
} else {
diff --git a/ndk/src/relay/index.ts b/ndk/src/relay/index.ts
index e980523d..3f0e5c8a 100644
--- a/ndk/src/relay/index.ts
+++ b/ndk/src/relay/index.ts
@@ -110,11 +110,7 @@ export class NDKRelay extends EventEmitter<{
public complaining = false;
readonly debug: debug.Debugger;
- public constructor(
- url: WebSocket["url"],
- authPolicy?: NDKAuthPolicy,
- ndk?: NDK,
- ) {
+ public constructor(url: WebSocket["url"], authPolicy?: NDKAuthPolicy, ndk?: NDK) {
super();
this.url = normalizeRelayUrl(url);
this.scores = new Map();
diff --git a/ndk/src/relay/pool/index.ts b/ndk/src/relay/pool/index.ts
index 153c281b..92ef3a01 100644
--- a/ndk/src/relay/pool/index.ts
+++ b/ndk/src/relay/pool/index.ts
@@ -4,6 +4,7 @@ import { EventEmitter } from "tseep";
import type { NDK } from "../../ndk/index.js";
import { NDKRelay, NDKRelayStatus } from "../index.js";
import { NDKFilter } from "../../subscription/index.js";
+import { normalizeRelayUrl } from "../../utils/normalize-url.js";
export type NDKPoolStats = {
total: number;
@@ -77,11 +78,12 @@ export class NDKPool extends EventEmitter<{
* @param relay - The relay to add to the pool.
* @param removeIfUnusedAfter - The time in milliseconds to wait before removing the relay from the pool after it is no longer used.
*/
- public useTemporaryRelay(relay: NDKRelay, removeIfUnusedAfter = 30000) {
+ public useTemporaryRelay(relay: NDKRelay, removeIfUnusedAfter = 30000, filters?: NDKFilter[]) {
const relayAlreadyInPool = this.relays.has(relay.url);
// check if the relay is already in the pool
if (!relayAlreadyInPool) {
+ // console.trace("adding relay to pool", relay.url, filters);
this.addRelay(relay);
}
@@ -100,7 +102,7 @@ export class NDKPool extends EventEmitter<{
// check if this relay is in the explicit relays list, if it is, it was connected temporary first
// and then made explicit, so we shouldn't disconnect
if (this.ndk.explicitRelayUrls?.includes(relay.url)) return;
-
+
this.removeRelay(relay.url);
}, removeIfUnusedAfter) as unknown as NodeJS.Timeout;
@@ -218,6 +220,17 @@ export class NDKPool extends EventEmitter<{
return false;
}
+ /**
+ * Checks whether a relay is already connected in the pool.
+ */
+ public isRelayConnected(url: WebSocket["url"]) {
+ const normalizedUrl = normalizeRelayUrl(url);
+ const relay = this.relays.get(normalizedUrl);
+ if (!relay) return false;
+
+ return relay.status === NDKRelayStatus.CONNECTED;
+ }
+
/**
* Fetches a relay from the pool, or creates a new one if it does not exist.
*
@@ -234,7 +247,7 @@ export class NDKPool extends EventEmitter<{
if (!relay) {
relay = new NDKRelay(url, undefined, this.ndk);
if (temporary) {
- this.useTemporaryRelay(relay);
+ this.useTemporaryRelay(relay, 30000, filters);
} else {
this.addRelay(relay, connect);
}
diff --git a/ndk/src/utils/get-users-relay-list.ts b/ndk/src/utils/get-users-relay-list.ts
index 62e077a4..c08bca2f 100644
--- a/ndk/src/utils/get-users-relay-list.ts
+++ b/ndk/src/utils/get-users-relay-list.ts
@@ -14,11 +14,15 @@ export async function getRelayListForUser(pubkey: Hexpubkey, ndk: NDK): Promise<
/**
* Fetches a map of relay lists for a number of users
- * @param pubkeys
- * @param ndk
- * @returns
+ * @param pubkeys
+ * @param ndk
+ * @returns
*/
-export async function getRelayListForUsers(pubkeys: Hexpubkey[], ndk: NDK): Promise