From eb5762e2ba084b2dd868d25d99493c9e784733c4 Mon Sep 17 00:00:00 2001 From: Jack Ellis Date: Wed, 29 May 2024 15:26:21 +0100 Subject: [PATCH] feat: add requiredBlock to querySubgraph you can now pass a requiredBlock to querySubgraph, if provided, it will check the block number of the subgraph response if the subgraph is behind the required block, it will wait and make 2 more attempts to get the latest block if the subgraph is still behind, an error will be thrown ths requiredBlock parameter is optional and if omitted, it will work exactly as before currently not being used within nftx.js methods internally but this provision means it can be added in the future --- .../src/__tests__/querySubgraph.test.ts | 90 +++++++- packages/subgraph/src/querySubgraph.ts | 199 ++++++++++++++---- 2 files changed, 236 insertions(+), 53 deletions(-) diff --git a/packages/subgraph/src/__tests__/querySubgraph.test.ts b/packages/subgraph/src/__tests__/querySubgraph.test.ts index 70438c4d..a3f9e687 100644 --- a/packages/subgraph/src/__tests__/querySubgraph.test.ts +++ b/packages/subgraph/src/__tests__/querySubgraph.test.ts @@ -1,6 +1,10 @@ import { UnknownError } from '@nftx/errors'; import { gql, querySubgraph } from '..'; +const ignoreWs = (str: string) => str.replace(/ /g, '').replace(/\n/g, ''); + +jest.setTimeout(30000); + // We're absolutely not bothered about a fully-working api // We just want to make sure that our method is sending the right payload // So I think it's reasonable to just stub the fetch api entirely instead @@ -8,7 +12,7 @@ import { gql, querySubgraph } from '..'; let fetch: jest.Mock; let response: { ok: boolean; - text: () => Promise; + text: jest.Mock; json: () => Record; headers: { get: (key: string) => string } & Record; }; @@ -18,7 +22,7 @@ beforeEach(() => { data = { vault: { id: '0x' } }; response = { ok: true, - text: async () => JSON.stringify({ data }), + text: jest.fn(async () => JSON.stringify({ data })), json: async () => ({}), headers: { 'Content-Type': 'application/json', @@ -74,6 +78,68 @@ it('injects variables into the query', async () => { expect(body).toEqual({ query: expected }); }); +it('enforces block accuracy', async () => { + data = { ...data, _meta: { block: { number: '100' } } }; + const tempData = { ...data, _meta: { block: { number: '99' } } }; + response.text.mockResolvedValueOnce(JSON.stringify({ data: tempData })); + + const query = gql` + { + vault(id: "0x") { + id + } + } + `; + + const result = await querySubgraph({ + url: 'https://nftx.io', + query, + fetch, + requiredBlock: 100, + }); + + const body = JSON.parse(fetch.mock.calls[0][1].body); + expect(ignoreWs(body.query)).toEqual( + ignoreWs(` + { + _meta { + block { + number + } + } + vault(id: "0x") { + id + } + } + `) + ); + expect(fetch).toBeCalledTimes(2); + expect(result).toEqual({ vault: { id: '0x' } }); +}); + +describe('when subgraph never catches up', () => { + it('throws an error', async () => { + data = { ...data, _meta: { block: { number: '99' } } }; + + const query = gql` + { + vault(id: "0x") { + id + } + } + `; + + const promise = querySubgraph({ + url: 'https://nftx.io', + query, + fetch, + requiredBlock: 100, + }); + + await expect(promise).rejects.toThrowError(); + }); +}); + it('returns the response data', async () => { const query = gql` { @@ -114,7 +180,7 @@ describe('error handling', () => { describe('when the response fails with an error message', () => { beforeEach(() => { response.ok = false; - response.text = async () => 'Failed'; + response.text.mockResolvedValue('Failed'); }); it('throws a specific error message', async () => { @@ -139,7 +205,9 @@ describe('error handling', () => { describe('when the response fails with an error object', () => { beforeEach(() => { response.ok = false; - response.text = async () => JSON.stringify({ error: 'Failed' }); + response.text.mockImplementation(async () => + JSON.stringify({ error: 'Failed' }) + ); }); it('throws a specific error message', async () => { @@ -162,7 +230,9 @@ describe('error handling', () => { describe('when response is not valid json', () => { beforeEach(() => { - response.text = async () => '

This is an error page

'; + response.text.mockImplementation( + async () => '

This is an error page

' + ); }); it('throws an error', async () => { @@ -182,8 +252,9 @@ describe('error handling', () => { describe('when response contains an error object', () => { beforeEach(() => { - response.text = async () => - JSON.stringify({ errors: { message: 'An error happened' } }); + response.text.mockImplementation(async () => + JSON.stringify({ errors: { message: 'An error happened' } }) + ); }); it('throws an error', async () => { @@ -203,10 +274,11 @@ describe('error handling', () => { describe('when response contains an errors object', () => { beforeEach(() => { - response.text = async () => + response.text.mockImplementation(async () => JSON.stringify({ errors: [{ message: 'Invalid subgraph syntax' }], - }); + }) + ); }); it('throws an error', async () => { diff --git a/packages/subgraph/src/querySubgraph.ts b/packages/subgraph/src/querySubgraph.ts index b636c6ac..6f575e0b 100644 --- a/packages/subgraph/src/querySubgraph.ts +++ b/packages/subgraph/src/querySubgraph.ts @@ -6,6 +6,146 @@ import type { QueryBase } from './createQuery'; type Fetch = typeof fetch; const globalFetch = typeof fetch === 'undefined' ? undefined : fetch; +const formatQuery = ({ + query, + requiredBlock, + variables, +}: { + query: any; + variables: Record | undefined; + requiredBlock: number | undefined; +}) => { + if (typeof query !== 'string') { + query = query.toString(); + } + if (variables) { + query = interpolateQuery(query, variables); + } + if (requiredBlock) { + query = query.replace(/\{/, `{\n _meta { block { number } }`); + } + return query; +}; + +const handleErrors = (errors: any) => { + // If there was an error with the query, we'll receive an array of errors + if (errors?.[0]?.message) { + throw new UnknownError(errors[0].message); + } + // Potentially a more generic error (like the endpoint was down) + if (errors?.message) { + throw new UnknownError(errors.message); + } +}; + +const doSubgraphQuery = async ({ + query, + url, + fetch, +}: { + url: string; + query: string; + fetch: Fetch | undefined; +}) => { + const { data, errors } = await sendQuery<{ + errors: { message: string }[] & { message: string }; + data: any; + }>({ + url, + cache: 'no-cache', + fetch, + headers: { 'Content-Type': 'application/json' }, + query: { query }, + method: 'POST', + }); + + handleErrors(errors); + + return data; +}; + +const queryWhileSubgraphBehind = async ({ + query, + requiredBlock, + url, + fetch, +}: { + url: string; + query: string; + requiredBlock: number | undefined; + fetch: Fetch | undefined; +}) => { + // If we're given a required block, we want to add it to the query string, and check it + // if the subgraph is behind, we'll wait and try again (up to 3 times) + let blockChecks = 0; + + do { + const data = await doSubgraphQuery({ query, url, fetch }); + + // If we're not checking for a specific block, we can stop here + if (!requiredBlock) { + return data; + } + // If the subgraph is up to date, we can stop here + if (Number(data?._meta?.block?.number) >= requiredBlock) { + delete data._meta; + return data; + } + // If we've tried 3 times and the subgraph is still behind, throw an error + if (blockChecks >= 2) { + throw new Error( + `Subgraph at ${url} is not up to date. Expected block ${requiredBlock}, got ${data?._meta?.block?.number}` + ); + } + // Wait 5s and try again + blockChecks += 1; + await new Promise((res) => setTimeout(res, 5000)); + } while (blockChecks < 3); + + throw new Error(`Subgraph at ${url} is not up to date`); +}; + +const queryUrls = async ({ + baseUrl, + query, + requiredBlock, + fetch, +}: { + baseUrl: string | string[]; + query: string; + requiredBlock: number | undefined; + fetch: Fetch | undefined; +}) => { + // We can be passed a single url or an array of urls + // If we have an array, we'll try them in order until we get a successful response + const urls = [baseUrl].flat(); + + while (urls.length) { + try { + const url = urls.shift(); + // Ignore empty urls (baseUrl could be undefined, or an array could've been built with missing content) + if (url == null) { + continue; + } + + const data = await queryWhileSubgraphBehind({ + query, + requiredBlock, + url, + fetch, + }); + + return data; + } catch (e) { + // If there's been an error, we'll try the next url + // if we've exhausted all urls, throw the most recent error + if (!urls.length) { + throw e; + } + } + } +}; + /** Sends a request to the subgraph * Uses the fetch api under the hood so if running in node you'll need to polyfill global.fetch */ @@ -21,6 +161,9 @@ async function querySubgraph>(args: { variables?: Q['__v']; /** The fetch api to use, if you are using a ponyfill, you can manually pass it in here */ fetch?: Fetch; + /** A block number that the subgraph must be up to in order to be considered up to date. + * If the subgraph block is less than this number, it will wait and re-attempt 3 times */ + requiredBlock?: number; }): Promise; async function querySubgraph>(args: { /** The subgraph url */ @@ -29,6 +172,9 @@ async function querySubgraph>(args: { query: Q; /** The fetch api to use, if you are using a ponyfill, you can manually pass it in here */ fetch?: Fetch; + /** A block number that the subgraph must be up to in order to be considered up to date. + * If the subgraph block is less than this number, it will wait and re-attempt 3 times */ + requiredBlock?: number; }): Promise; async function querySubgraph(args: { /** The subgraph url */ @@ -36,62 +182,27 @@ async function querySubgraph(args: { query: string; variables?: Record; fetch?: Fetch; + requiredBlock?: number; }): Promise; async function querySubgraph({ url: baseUrl, query, variables, fetch = globalFetch, + requiredBlock, }: { url: string | string[]; query: any; variables?: Record; fetch?: Fetch; + requiredBlock?: number; }) { - if (typeof query !== 'string') { - query = query.toString(); - } - if (variables) { - query = interpolateQuery(query, variables); - } - - // Override the default api key with a custom one if set - const urls = [baseUrl].flat(); - - while (urls.length) { - try { - const url = urls.shift(); - if (url == null) { - continue; - } - - const { data, errors } = await sendQuery<{ - errors: { message: string }[] & { message: string }; - data: any; - }>({ - url, - cache: 'no-cache', - fetch, - headers: { 'Content-Type': 'application/json' }, - query: { query }, - method: 'POST', - }); - - // If there was an error with the query, we'll receive an array of errors - if (errors?.[0]?.message) { - throw new UnknownError(errors[0].message); - } - if (errors?.message) { - throw new UnknownError(errors.message); - } - - return data; - } catch (e) { - if (!urls.length) { - throw e; - } - } - } + return queryUrls({ + baseUrl, + query: formatQuery({ query, requiredBlock, variables }), + requiredBlock, + fetch, + }); } export default querySubgraph;