From 8b79234638af68531125d78cdf122b453c2f70e6 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Mon, 18 Nov 2024 13:15:31 -0800 Subject: [PATCH] common: add pagination support to allocation queries --- .../__tests__/helpers.test.ts | 2 +- .../resolvers/allocations.ts | 79 ++++++++++++++----- .../resolvers/indexer-status.ts | 63 +++++++++------ 3 files changed, 100 insertions(+), 44 deletions(-) diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index a2358926e..402775edb 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -473,7 +473,7 @@ describe('Monitor: CI', () => { test('Fetch subgraph deployments (unconstrained block)', async () => { const deployments = await networkMonitor.subgraphDeployments() await expect(deployments.length).toBeGreaterThan(500) - }, 30000) + }, 40000) }) describe('Network layer detection', () => { diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index f5ad5da3a..385c9a424 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -66,8 +66,13 @@ interface AllocationInfo { const ALLOCATION_QUERIES = { [AllocationQuery.all]: gql` - query allocations($indexer: String!) { - allocations(where: { indexer: $indexer }, first: 1000) { + query allocations($indexer: String!, $lastId: String!) { + allocations( + where: { indexer: $indexer, id_gt: $lastId } + orderBy: id + orderDirection: asc + first: 1000 + ) { id subgraphDeployment { id @@ -87,8 +92,13 @@ const ALLOCATION_QUERIES = { } `, [AllocationQuery.active]: gql` - query allocations($indexer: String!) { - allocations(where: { indexer: $indexer, status: Active }, first: 1000) { + query allocations($indexer: String!, $lastId: String!) { + allocations( + where: { indexer: $indexer, id_gt: $lastId, status: Active } + orderBy: id + orderDirection: asc + first: 1000 + ) { id subgraphDeployment { id @@ -108,8 +118,13 @@ const ALLOCATION_QUERIES = { } `, [AllocationQuery.closed]: gql` - query allocations($indexer: String!) { - allocations(where: { indexer: $indexer, status: Closed }, first: 1000) { + query allocations($indexer: String!, $lastId: String!) { + allocations( + where: { indexer: $indexer, id_gt: $lastId, status: Closed } + orderBy: id + orderDirection: asc + first: 1000 + ) { id subgraphDeployment { id @@ -129,8 +144,13 @@ const ALLOCATION_QUERIES = { } `, [AllocationQuery.allocation]: gql` - query allocations($allocation: String!) { - allocations(where: { id: $allocation }, first: 1000) { + query allocations($allocation: String!, $lastId: String!) { + allocations( + where: { id: $allocation, id_gt: $lastId } + orderBy: id + orderDirection: asc + first: 1000 + ) { id subgraphDeployment { id @@ -203,27 +223,44 @@ async function queryAllocations( ) } - const result = await networkSubgraph.checkedQuery( - ALLOCATION_QUERIES[filterType], - filterVars, - ) + let lastId = '' + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const resultAllocations: any[] = [] + for (;;) { + const pageVars = { + ...filterVars, + lastId, + } + const result = await networkSubgraph.checkedQuery( + ALLOCATION_QUERIES[filterType], + pageVars, + ) + + if (result.error) { + logger.warning('Querying allocations failed', { + error: result.error, + lastId: lastId, + }) + throw result.error + } - if (result.data.allocations.length == 0) { + if (result.data.allocations.length == 0) { + break + } + // merge results + resultAllocations.push(...result.data.allocations) + lastId = result.data.allocations.slice(-1)[0].id + } + + if (resultAllocations.length == 0) { // TODO: Is 'Claimable' still the correct term here, after Exponential Rebates? logger.info(`No 'Claimable' allocations found`) return [] } - if (result.error) { - logger.warning('Query failed', { - error: result.error, - }) - throw result.error - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any return pMap( - result.data.allocations, + resultAllocations, // eslint-disable-next-line @typescript-eslint/no-explicit-any async (allocation: any): Promise => { const deadlineEpoch = allocation.createdAtEpoch + context.maxAllocationEpochs diff --git a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts index dad36df87..c040113a2 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts @@ -124,33 +124,52 @@ export default { const address = network.specification.indexerOptions.address try { - const result = await network.networkSubgraph.checkedQuery( - gql` - query allocations($indexer: String!) { - allocations( - where: { indexer: $indexer, status: Active } - first: 1000 - orderDirection: desc - ) { - id - allocatedTokens - createdAtEpoch - closedAtEpoch - subgraphDeployment { + let lastId = '' + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const allAllocations: any[] = [] + for (;;) { + const result = await network.networkSubgraph.checkedQuery( + gql` + query allocations($indexer: String!, $lastId: String!) { + allocations( + where: { indexer: $indexer, status: Active, id_gt: $lastId } + first: 1000 + orderBy: id + orderDirection: asc + ) { id - stakedTokens - signalledTokens + allocatedTokens + createdAtEpoch + closedAtEpoch + subgraphDeployment { + id + stakedTokens + signalledTokens + } } } - } - `, - { indexer: address.toLocaleLowerCase() }, - ) - if (result.error) { - throw result.error + `, + { indexer: address.toLocaleLowerCase(), lastId }, + ) + + if (result.error) { + logger.warning('Querying allocations failed', { + error: result.error, + lastId: lastId, + }) + throw result.error + } + + if (result.data.allocations.length === 0) { + break + } + + allAllocations.push(...result.data.allocations) + lastId = result.data.allocations.slice(-1)[0].id } + // eslint-disable-next-line @typescript-eslint/no-explicit-any - return result.data.allocations.map((allocation: any) => ({ + return allAllocations.map((allocation: any) => ({ ...allocation, subgraphDeployment: new SubgraphDeploymentID(allocation.subgraphDeployment.id) .ipfsHash,