common: add pagination support to allocation queries

graphprotocol · Nov 19, 2024 · 8b79234 · 8b79234
1 parent b663c65
commit 8b79234
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 44 deletions.
diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts
@@ -473,7 +473,7 @@ describe('Monitor: CI', () => {
   test('Fetch subgraph deployments (unconstrained block)', async () => {
     const deployments = await networkMonitor.subgraphDeployments()
     await expect(deployments.length).toBeGreaterThan(500)
-  }, 30000)
+  }, 40000)
 })
 
 describe('Network layer detection', () => {

diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts
@@ -66,8 +66,13 @@ interface AllocationInfo {
 
 const ALLOCATION_QUERIES = {
   [AllocationQuery.all]: gql`
-    query allocations($indexer: String!) {
-      allocations(where: { indexer: $indexer }, first: 1000) {
+    query allocations($indexer: String!, $lastId: String!) {
+      allocations(
+        where: { indexer: $indexer, id_gt: $lastId }
+        orderBy: id
+        orderDirection: asc
+        first: 1000
+      ) {
         id
         subgraphDeployment {
           id
@@ -87,8 +92,13 @@ const ALLOCATION_QUERIES = {
     }
   `,
   [AllocationQuery.active]: gql`
-    query allocations($indexer: String!) {
-      allocations(where: { indexer: $indexer, status: Active }, first: 1000) {
+    query allocations($indexer: String!, $lastId: String!) {
+      allocations(
+        where: { indexer: $indexer, id_gt: $lastId, status: Active }
+        orderBy: id
+        orderDirection: asc
+        first: 1000
+      ) {
         id
         subgraphDeployment {
           id
@@ -108,8 +118,13 @@ const ALLOCATION_QUERIES = {
     }
   `,
   [AllocationQuery.closed]: gql`
-    query allocations($indexer: String!) {
-      allocations(where: { indexer: $indexer, status: Closed }, first: 1000) {
+    query allocations($indexer: String!, $lastId: String!) {
+      allocations(
+        where: { indexer: $indexer, id_gt: $lastId, status: Closed }
+        orderBy: id
+        orderDirection: asc
+        first: 1000
+      ) {
         id
         subgraphDeployment {
           id
@@ -129,8 +144,13 @@ const ALLOCATION_QUERIES = {
     }
   `,
   [AllocationQuery.allocation]: gql`
-    query allocations($allocation: String!) {
-      allocations(where: { id: $allocation }, first: 1000) {
+    query allocations($allocation: String!, $lastId: String!) {
+      allocations(
+        where: { id: $allocation, id_gt: $lastId }
+        orderBy: id
+        orderDirection: asc
+        first: 1000
+      ) {
         id
         subgraphDeployment {
           id
@@ -203,27 +223,44 @@ async function queryAllocations(
     )
   }
 
-  const result = await networkSubgraph.checkedQuery(
-    ALLOCATION_QUERIES[filterType],
-    filterVars,
-  )
+  let lastId = ''
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const resultAllocations: any[] = []
+  for (;;) {
+    const pageVars = {
+      ...filterVars,
+      lastId,
+    }
+    const result = await networkSubgraph.checkedQuery(
+      ALLOCATION_QUERIES[filterType],
+      pageVars,
+    )
+
+    if (result.error) {
+      logger.warning('Querying allocations failed', {
+        error: result.error,
+        lastId: lastId,
+      })
+      throw result.error
+    }
 
-  if (result.data.allocations.length == 0) {
+    if (result.data.allocations.length == 0) {
+      break
+    }
+    // merge results
+    resultAllocations.push(...result.data.allocations)
+    lastId = result.data.allocations.slice(-1)[0].id
+  }
+
+  if (resultAllocations.length == 0) {
     // TODO: Is 'Claimable' still the correct term here, after Exponential Rebates?
     logger.info(`No 'Claimable' allocations found`)
     return []
   }
 
-  if (result.error) {
-    logger.warning('Query failed', {
-      error: result.error,
-    })
-    throw result.error
-  }
-
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   return pMap(
-    result.data.allocations,
+    resultAllocations,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     async (allocation: any): Promise<AllocationInfo> => {
       const deadlineEpoch = allocation.createdAtEpoch + context.maxAllocationEpochs

diff --git a/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts b/packages/indexer-common/src/indexer-management/resolvers/indexer-status.ts
@@ -124,33 +124,52 @@ export default {
     const address = network.specification.indexerOptions.address
 
     try {
-      const result = await network.networkSubgraph.checkedQuery(
-        gql`
-          query allocations($indexer: String!) {
-            allocations(
-              where: { indexer: $indexer, status: Active }
-              first: 1000
-              orderDirection: desc
-            ) {
-              id
-              allocatedTokens
-              createdAtEpoch
-              closedAtEpoch
-              subgraphDeployment {
+      let lastId = ''
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const allAllocations: any[] = []
+      for (;;) {
+        const result = await network.networkSubgraph.checkedQuery(
+          gql`
+            query allocations($indexer: String!, $lastId: String!) {
+              allocations(
+                where: { indexer: $indexer, status: Active, id_gt: $lastId }
+                first: 1000
+                orderBy: id
+                orderDirection: asc
+              ) {
                 id
-                stakedTokens
-                signalledTokens
+                allocatedTokens
+                createdAtEpoch
+                closedAtEpoch
+                subgraphDeployment {
+                  id
+                  stakedTokens
+                  signalledTokens
+                }
               }
             }
-          }
-        `,
-        { indexer: address.toLocaleLowerCase() },
-      )
-      if (result.error) {
-        throw result.error
+          `,
+          { indexer: address.toLocaleLowerCase(), lastId },
+        )
+
+        if (result.error) {
+          logger.warning('Querying allocations failed', {
+            error: result.error,
+            lastId: lastId,
+          })
+          throw result.error
+        }
+
+        if (result.data.allocations.length === 0) {
+          break
+        }
+
+        allAllocations.push(...result.data.allocations)
+        lastId = result.data.allocations.slice(-1)[0].id
       }
+
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      return result.data.allocations.map((allocation: any) => ({
+      return allAllocations.map((allocation: any) => ({
         ...allocation,
         subgraphDeployment: new SubgraphDeploymentID(allocation.subgraphDeployment.id)
           .ipfsHash,