Skip to content

Commit

Permalink
feat: add requiredBlock to querySubgraph
Browse files Browse the repository at this point in the history 
you can now pass a requiredBlock to querySubgraph, if provided, it will check the block number of the subgraph response
if the subgraph is behind the required block, it will wait and make 2 more attempts to get the latest block
if the subgraph is still behind, an error will be thrown
ths requiredBlock parameter is optional and if omitted, it will work exactly as before
currently not being used within nftx.js methods internally but this provision means it can be added in the future
  • Loading branch information
@jackmellis
jackmellis committed May 30, 2024
1 parent a1b550f commit eb5762e
Show file tree
Hide file tree
Showing 2 changed files with 236 additions and 53 deletions.
90 changes: 81 additions & 9 deletions packages/subgraph/src/__tests__/querySubgraph.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
import { UnknownError } from '@nftx/errors';
import { gql, querySubgraph } from '..';

const ignoreWs = (str: string) => str.replace(/ /g, '').replace(/\n/g, '');

jest.setTimeout(30000);

// We're absolutely not bothered about a fully-working api
// We just want to make sure that our method is sending the right payload
// So I think it's reasonable to just stub the fetch api entirely instead
// of trying to construct a fully-working backend server
let fetch: jest.Mock;
let response: {
ok: boolean;
text: () => Promise<string>;
text: jest.Mock;
json: () => Record<string, any>;
headers: { get: (key: string) => string } & Record<string, any>;
};
Expand All @@ -18,7 +22,7 @@ beforeEach(() => {
data = { vault: { id: '0x' } };
response = {
ok: true,
text: async () => JSON.stringify({ data }),
text: jest.fn(async () => JSON.stringify({ data })),
json: async () => ({}),
headers: {
'Content-Type': 'application/json',
Expand Down Expand Up @@ -74,6 +78,68 @@ it('injects variables into the query', async () => {
expect(body).toEqual({ query: expected });
});

it('enforces block accuracy', async () => {
data = { ...data, _meta: { block: { number: '100' } } };
const tempData = { ...data, _meta: { block: { number: '99' } } };
response.text.mockResolvedValueOnce(JSON.stringify({ data: tempData }));

const query = gql`
{
vault(id: "0x") {
id
}
}
`;

const result = await querySubgraph({
url: 'https://nftx.io',
query,
fetch,
requiredBlock: 100,
});

const body = JSON.parse(fetch.mock.calls[0][1].body);
expect(ignoreWs(body.query)).toEqual(
ignoreWs(`
{
_meta {
block {
number
}
}
vault(id: "0x") {
id
}
}
`)
);
expect(fetch).toBeCalledTimes(2);
expect(result).toEqual({ vault: { id: '0x' } });
});

describe('when subgraph never catches up', () => {
it('throws an error', async () => {
data = { ...data, _meta: { block: { number: '99' } } };

const query = gql`
{
vault(id: "0x") {
id
}
}
`;

const promise = querySubgraph({
url: 'https://nftx.io',
query,
fetch,
requiredBlock: 100,
});

await expect(promise).rejects.toThrowError();
});
});

it('returns the response data', async () => {
const query = gql`
{
Expand Down Expand Up @@ -114,7 +180,7 @@ describe('error handling', () => {
describe('when the response fails with an error message', () => {
beforeEach(() => {
response.ok = false;
response.text = async () => 'Failed';
response.text.mockResolvedValue('Failed');
});

it('throws a specific error message', async () => {
Expand All @@ -139,7 +205,9 @@ describe('error handling', () => {
describe('when the response fails with an error object', () => {
beforeEach(() => {
response.ok = false;
response.text = async () => JSON.stringify({ error: 'Failed' });
response.text.mockImplementation(async () =>
JSON.stringify({ error: 'Failed' })
);
});

it('throws a specific error message', async () => {
Expand All @@ -162,7 +230,9 @@ describe('error handling', () => {

describe('when response is not valid json', () => {
beforeEach(() => {
response.text = async () => '<body><p>This is an error page</p></body>';
response.text.mockImplementation(
async () => '<body><p>This is an error page</p></body>'
);
});

it('throws an error', async () => {
Expand All @@ -182,8 +252,9 @@ describe('error handling', () => {

describe('when response contains an error object', () => {
beforeEach(() => {
response.text = async () =>
JSON.stringify({ errors: { message: 'An error happened' } });
response.text.mockImplementation(async () =>
JSON.stringify({ errors: { message: 'An error happened' } })
);
});

it('throws an error', async () => {
Expand All @@ -203,10 +274,11 @@ describe('error handling', () => {

describe('when response contains an errors object', () => {
beforeEach(() => {
response.text = async () =>
response.text.mockImplementation(async () =>
JSON.stringify({
errors: [{ message: 'Invalid subgraph syntax' }],
});
})
);
});

it('throws an error', async () => {
Expand Down
199 changes: 155 additions & 44 deletions packages/subgraph/src/querySubgraph.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,146 @@ import type { QueryBase } from './createQuery';
type Fetch = typeof fetch;
const globalFetch = typeof fetch === 'undefined' ? undefined : fetch;

const formatQuery = ({
query,
requiredBlock,
variables,
}: {
query: any;
variables: Record<string, any> | undefined;
requiredBlock: number | undefined;
}) => {
if (typeof query !== 'string') {
query = query.toString();
}
if (variables) {
query = interpolateQuery(query, variables);
}
if (requiredBlock) {
query = query.replace(/\{/, `{\n _meta { block { number } }`);
}
return query;
};

const handleErrors = (errors: any) => {
// If there was an error with the query, we'll receive an array of errors
if (errors?.[0]?.message) {
throw new UnknownError(errors[0].message);
}
// Potentially a more generic error (like the endpoint was down)
if (errors?.message) {
throw new UnknownError(errors.message);
}
};

const doSubgraphQuery = async ({
query,
url,
fetch,
}: {
url: string;
query: string;
fetch: Fetch | undefined;
}) => {
const { data, errors } = await sendQuery<{
errors: { message: string }[] & { message: string };
data: any;
}>({
url,
cache: 'no-cache',
fetch,
headers: { 'Content-Type': 'application/json' },
query: { query },
method: 'POST',
});

handleErrors(errors);

return data;
};

const queryWhileSubgraphBehind = async ({
query,
requiredBlock,
url,
fetch,
}: {
url: string;
query: string;
requiredBlock: number | undefined;
fetch: Fetch | undefined;
}) => {
// If we're given a required block, we want to add it to the query string, and check it
// if the subgraph is behind, we'll wait and try again (up to 3 times)
let blockChecks = 0;

do {
const data = await doSubgraphQuery({ query, url, fetch });

// If we're not checking for a specific block, we can stop here
if (!requiredBlock) {
return data;
}
// If the subgraph is up to date, we can stop here
if (Number(data?._meta?.block?.number) >= requiredBlock) {
delete data._meta;
return data;
}
// If we've tried 3 times and the subgraph is still behind, throw an error
if (blockChecks >= 2) {
throw new Error(
`Subgraph at ${url} is not up to date. Expected block ${requiredBlock}, got ${data?._meta?.block?.number}`
);
}
// Wait 5s and try again
blockChecks += 1;
await new Promise((res) => setTimeout(res, 5000));
} while (blockChecks < 3);

throw new Error(`Subgraph at ${url} is not up to date`);
};

const queryUrls = async ({
baseUrl,
query,
requiredBlock,
fetch,
}: {
baseUrl: string | string[];
query: string;
requiredBlock: number | undefined;
fetch: Fetch | undefined;
}) => {
// We can be passed a single url or an array of urls
// If we have an array, we'll try them in order until we get a successful response
const urls = [baseUrl].flat();

while (urls.length) {
try {
const url = urls.shift();
// Ignore empty urls (baseUrl could be undefined, or an array could've been built with missing content)
if (url == null) {
continue;
}

const data = await queryWhileSubgraphBehind({
query,
requiredBlock,
url,
fetch,
});

return data;
} catch (e) {
// If there's been an error, we'll try the next url
// if we've exhausted all urls, throw the most recent error
if (!urls.length) {
throw e;
}
}
}
};

/** Sends a request to the subgraph
* Uses the fetch api under the hood so if running in node you'll need to polyfill global.fetch
*/
Expand All @@ -21,6 +161,9 @@ async function querySubgraph<Q extends GraphQueryString<any, any>>(args: {
variables?: Q['__v'];
/** The fetch api to use, if you are using a ponyfill, you can manually pass it in here */
fetch?: Fetch;
/** A block number that the subgraph must be up to in order to be considered up to date.
* If the subgraph block is less than this number, it will wait and re-attempt 3 times */
requiredBlock?: number;
}): Promise<Q['__r']>;
async function querySubgraph<Q extends QueryBase<any, any>>(args: {
/** The subgraph url */
Expand All @@ -29,69 +172,37 @@ async function querySubgraph<Q extends QueryBase<any, any>>(args: {
query: Q;
/** The fetch api to use, if you are using a ponyfill, you can manually pass it in here */
fetch?: Fetch;
/** A block number that the subgraph must be up to in order to be considered up to date.
* If the subgraph block is less than this number, it will wait and re-attempt 3 times */
requiredBlock?: number;
}): Promise<Q['__r']>;
async function querySubgraph(args: {
/** The subgraph url */
url: string | string[];
query: string;
variables?: Record<string, any>;
fetch?: Fetch;
requiredBlock?: number;
}): Promise<any>;
async function querySubgraph({
url: baseUrl,
query,
variables,
fetch = globalFetch,
requiredBlock,
}: {
url: string | string[];
query: any;
variables?: Record<string, any>;
fetch?: Fetch;
requiredBlock?: number;
}) {
if (typeof query !== 'string') {
query = query.toString();
}
if (variables) {
query = interpolateQuery(query, variables);
}

// Override the default api key with a custom one if set
const urls = [baseUrl].flat();

while (urls.length) {
try {
const url = urls.shift();
if (url == null) {
continue;
}

const { data, errors } = await sendQuery<{
errors: { message: string }[] & { message: string };
data: any;
}>({
url,
cache: 'no-cache',
fetch,
headers: { 'Content-Type': 'application/json' },
query: { query },
method: 'POST',
});

// If there was an error with the query, we'll receive an array of errors
if (errors?.[0]?.message) {
throw new UnknownError(errors[0].message);
}
if (errors?.message) {
throw new UnknownError(errors.message);
}

return data;
} catch (e) {
if (!urls.length) {
throw e;
}
}
}
return queryUrls({
baseUrl,
query: formatQuery({ query, requiredBlock, variables }),
requiredBlock,
fetch,
});
}

export default querySubgraph;

0 comments on commit eb5762e

Please sign in to comment.