From 9d71dedb561ebfb18c67f31322c336a3cf9e81a9 Mon Sep 17 00:00:00 2001 From: Arda TANRIKULU Date: Wed, 11 May 2022 02:13:50 -0400 Subject: [PATCH] Respect other arguments while creating underlying pagination selections and use serial queries if skip limit exceeds (#76) --- .changeset/eleven-pianos-glow.md | 5 + .changeset/smooth-rice-turn.md | 5 + .github/workflows/canary.yml | 67 ++++++ examples/transforms/.graphclientrc.yml | 7 +- .../__tests__/auto-pagination.test.ts | 85 +++++--- packages/auto-pagination/package.json | 6 + packages/auto-pagination/src/index.ts | 193 ++++++++++++------ scripts/canary-release.js | 83 ++++++++ yarn.lock | 5 + 9 files changed, 371 insertions(+), 85 deletions(-) create mode 100644 .changeset/eleven-pianos-glow.md create mode 100644 .changeset/smooth-rice-turn.md create mode 100644 .github/workflows/canary.yml create mode 100644 scripts/canary-release.js diff --git a/.changeset/eleven-pianos-glow.md b/.changeset/eleven-pianos-glow.md new file mode 100644 index 00000000..647e3b03 --- /dev/null +++ b/.changeset/eleven-pianos-glow.md @@ -0,0 +1,5 @@ +--- +'@graphprotocol/client-auto-pagination': patch +--- + +Respect other arguments while creating underlying pagination selections diff --git a/.changeset/smooth-rice-turn.md b/.changeset/smooth-rice-turn.md new file mode 100644 index 00000000..f2060e21 --- /dev/null +++ b/.changeset/smooth-rice-turn.md @@ -0,0 +1,5 @@ +--- +'@graphprotocol/client-auto-pagination': patch +--- + +Use lastID if skip exceeds the limit diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 00000000..d9b6fa0d --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,67 @@ +name: Canary Release + +on: + pull_request: + paths: + - '.changeset/**/*.md' + +jobs: + publish-canary: + name: Publish Canary + runs-on: ubuntu-latest + if: github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Use Node + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + + - name: Install Dependencies using Yarn + run: yarn + + - name: Setup NPM credentials + run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Release Canary + id: canary + uses: 'kamilkisiela/release-canary@master' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + with: + npm-token: ${{ secrets.NPM_TOKEN }} + npm-script: 'yarn release:canary' + changesets: true + + - name: Publish a message + if: steps.canary.outputs.released == 'true' + uses: 'kamilkisiela/pr-comment@master' + with: + commentKey: canary + message: | + The latest changes of this PR are available as canary in npm (based on the declared `changesets`): + + ``` + ${{ steps.canary.outputs.changesetsPublishedPackages}} + ``` + bot-token: ${{ secrets.GITHUB_TOKEN }} + bot: 'github-actions[bot]' + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish a empty message + if: steps.canary.outputs.released == 'false' + uses: 'kamilkisiela/pr-comment@master' + with: + commentKey: canary + message: | + The latest changes of this PR are not available as canary, since there are no linked `changesets` for this PR. + bot-token: ${{ secrets.GITHUB_TOKEN }} + bot: 'github-actions[bot]' + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/examples/transforms/.graphclientrc.yml b/examples/transforms/.graphclientrc.yml index 012db761..a7375413 100644 --- a/examples/transforms/.graphclientrc.yml +++ b/examples/transforms/.graphclientrc.yml @@ -5,7 +5,10 @@ sources: endpoint: https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2 transforms: # Enable Automatic Block Tracking - - autoPagination: - validateSchema: true - blockTracking: validateSchema: true + - autoPagination: + validateSchema: true + +serve: + browser: false diff --git a/packages/auto-pagination/__tests__/auto-pagination.test.ts b/packages/auto-pagination/__tests__/auto-pagination.test.ts index 9469be9e..c4a65d22 100644 --- a/packages/auto-pagination/__tests__/auto-pagination.test.ts +++ b/packages/auto-pagination/__tests__/auto-pagination.test.ts @@ -4,13 +4,13 @@ import { execute, ExecutionResult, parse } from 'graphql' import AutoPaginationTransform from '../src' describe('Auto Pagination', () => { - const users = new Array(20).fill({}).map((_, i) => ({ id: (i + 1).toString(), name: `User ${i + 1}` })) - const LIMIT = 3 + const users = new Array(20000).fill({}).map((_, i) => ({ id: (i + 1).toString(), name: `User ${i + 1}` })) + const usersOdd = users.filter((_, i) => i % 2 === 1) const schema = makeExecutableSchema({ typeDefs: /* GraphQL */ ` type Query { _meta: Meta - users(first: Int = ${LIMIT}, skip: Int = 0): [User!]! + users(first: Int = ${1000}, skip: Int = 0, odd: Boolean, where: WhereInput): [User!]! } type User { id: ID! @@ -22,14 +22,27 @@ describe('Auto Pagination', () => { type Block { number: Int } + input WhereInput { + id_gte: ID + } `, resolvers: { Query: { - users: (_, { first = LIMIT, skip = 0 }) => { - if (first > LIMIT) { - throw new Error(`You cannot request more than ${LIMIT} users; you requested ${first}`) + users: (_, { first = 1000, skip = 0, odd, where }) => { + if (first > 1000) { + throw new Error(`You cannot request more than 1000 users; you requested ${first}`) + } + if (skip > 5000) { + throw new Error(`You cannot skip more than 5000 users; you requested ${skip}`) + } + let usersSlice = users + if (odd) { + usersSlice = usersOdd + } + if (where?.id_gte) { + usersSlice = users.slice(where.id_gte) } - return users.slice(skip, skip + first) + return usersSlice.slice(skip, skip + first) }, _meta: () => ({ block: { @@ -41,18 +54,12 @@ describe('Auto Pagination', () => { }) const wrappedSchema = wrapSchema({ schema, - transforms: [ - new AutoPaginationTransform({ - config: { - limitOfRecords: LIMIT, - }, - }), - ], + transforms: [new AutoPaginationTransform()], }) it('should give correct numbers of results if first arg are higher than given limit', async () => { const query = /* GraphQL */ ` query { - users(first: 10) { + users(first: 2000) { id name } @@ -62,13 +69,13 @@ describe('Auto Pagination', () => { schema: wrappedSchema, document: parse(query), }) - expect(result.data?.users).toHaveLength(10) - expect(result.data?.users).toEqual(users.slice(0, 10)) + expect(result.data?.users).toHaveLength(2000) + expect(result.data?.users).toEqual(users.slice(0, 2000)) }) it('should respect skip argument', async () => { const query = /* GraphQL */ ` query { - users(first: 10, skip: 1) { + users(first: 2000, skip: 1) { id name } @@ -78,8 +85,8 @@ describe('Auto Pagination', () => { schema: wrappedSchema, document: parse(query), }) - expect(result.data?.users).toHaveLength(10) - expect(result.data?.users).toEqual(users.slice(1, 11)) + expect(result.data?.users).toHaveLength(2000) + expect(result.data?.users).toEqual(users.slice(1, 2001)) }) it('should work with the values under the limit', async () => { const query = /* GraphQL */ ` @@ -105,7 +112,7 @@ describe('Auto Pagination', () => { number } } - users(first: 10) { + users(first: 2000) { id name } @@ -116,7 +123,39 @@ describe('Auto Pagination', () => { document: parse(query), }) expect(result.data?._meta?.block?.number).toBeDefined() - expect(result.data?.users).toHaveLength(10) - expect(result.data?.users).toEqual(users.slice(0, 10)) + expect(result.data?.users).toHaveLength(2000) + expect(result.data?.users).toEqual(users.slice(0, 2000)) + }) + it('should respect other arguments', async () => { + const query = /* GraphQL */ ` + query { + users(first: 2000, odd: true) { + id + name + } + } + ` + const result: ExecutionResult = await execute({ + schema: wrappedSchema, + document: parse(query), + }) + expect(result.data?.users).toHaveLength(2000) + expect(result.data?.users).toEqual(usersOdd.slice(0, 2000)) + }) + it('should make queries serially if skip limit reaches the limit', async () => { + const query = /* GraphQL */ ` + query { + users(first: 15000) { + id + name + } + } + ` + const result: ExecutionResult = await execute({ + schema: wrappedSchema, + document: parse(query), + }) + expect(result.data?.users).toHaveLength(15000) + expect(result.data?.users).toEqual(users.slice(0, 15000)) }) }) diff --git a/packages/auto-pagination/package.json b/packages/auto-pagination/package.json index d9a1dcff..ed6a7fa8 100644 --- a/packages/auto-pagination/package.json +++ b/packages/auto-pagination/package.json @@ -39,9 +39,15 @@ "access": "public" }, "dependencies": { + "@graphql-tools/delegate": "8.7.7", + "@graphql-tools/wrap": "8.4.16", "@graphql-tools/utils": "8.6.10", + "lodash": "4.17.21", "tslib": "2.4.0" }, + "devDependencies": { + "@types/lodash": "4.14.182" + }, "peerDependencies": { "graphql": "^15.2.0 || ^16.0.0", "@graphql-mesh/types": "^0.72.0" diff --git a/packages/auto-pagination/src/index.ts b/packages/auto-pagination/src/index.ts index d182524d..8dd92496 100644 --- a/packages/auto-pagination/src/index.ts +++ b/packages/auto-pagination/src/index.ts @@ -1,11 +1,10 @@ import type { MeshTransform } from '@graphql-mesh/types' -import type { DelegationContext } from '@graphql-tools/delegate' +import { delegateToSchema, DelegationContext, SubschemaConfig } from '@graphql-tools/delegate' import type { ExecutionRequest } from '@graphql-tools/utils' import { ArgumentNode, ExecutionResult, GraphQLSchema, - IntValueNode, isListType, isNonNullType, Kind, @@ -14,6 +13,7 @@ import { visit, } from 'graphql' import { memoize1, memoize2 } from '@graphql-tools/utils' +import _ from 'lodash' interface AutoPaginationTransformConfig { if?: boolean @@ -21,6 +21,8 @@ interface AutoPaginationTransformConfig { limitOfRecords?: number firstArgumentName?: string skipArgumentName?: string + lastIdArgumentName?: string + skipArgumentLimit?: number } const DEFAULTS: Required = { @@ -29,6 +31,8 @@ const DEFAULTS: Required = { limitOfRecords: 1000, firstArgumentName: 'first', skipArgumentName: 'skip', + lastIdArgumentName: 'where.id_gte', + skipArgumentLimit: 5000, } const validateSchema = memoize2(function validateSchema( @@ -74,10 +78,63 @@ export default class AutoPaginationTransform implements MeshTransform { } } - transformSchema(schema: GraphQLSchema) { + transformSchema( + schema: GraphQLSchema, + subschemaConfig: SubschemaConfig, + transformedSchema: GraphQLSchema | undefined, + ) { if (this.config.validateSchema) { validateSchema(schema, this.config) } + if (transformedSchema != null) { + const queryType = transformedSchema.getQueryType() + if (queryType != null) { + const queryFields = queryType.getFields() + for (const fieldName in queryFields) { + if (!fieldName.startsWith('_')) { + const field = queryFields[fieldName] + const existingResolver = field.resolve! + field.resolve = async (root, args, context, info) => { + const totalRecords = args[this.config.firstArgumentName] || 1000 + const initialSkipValue = args[this.config.skipArgumentName] || 0 + if (totalRecords >= this.config.skipArgumentLimit * 2) { + let remainingRecords = totalRecords + const records: any[] = [] + while (remainingRecords > 0) { + let skipValue = records.length === 0 ? initialSkipValue : 0 + const lastIdValue = records.length > 0 ? records[records.length - 1].id : null + while (skipValue < this.config.skipArgumentLimit && remainingRecords > 0) { + const newArgs = { + ...args, + } + if (lastIdValue) { + _.set(newArgs, this.config.lastIdArgumentName, lastIdValue) + } + _.set(newArgs, this.config.skipArgumentName, skipValue) + const askedRecords = Math.min(remainingRecords, this.config.skipArgumentLimit) + _.set(newArgs, this.config.firstArgumentName, askedRecords) + const result = await delegateToSchema({ + schema: transformedSchema, + args: newArgs, + context, + info, + }) + if (!Array.isArray(result)) { + return result + } + records.push(...result) + skipValue += askedRecords + remainingRecords -= askedRecords + } + } + return records + } + return existingResolver(root, args, context, info) + } + } + } + } + } return schema } @@ -87,67 +144,83 @@ export default class AutoPaginationTransform implements MeshTransform { leave: (selectionSet) => { const newSelections: SelectionNode[] = [] for (const selectionNode of selectionSet.selections) { - if (selectionNode.kind === Kind.FIELD) { - if ( - !selectionNode.name.value.startsWith('_') && - getQueryFieldNames(delegationContext.transformedSchema).includes(selectionNode.name.value) && - !selectionNode.arguments?.some((argNode) => argNode.name.value === 'id') - ) { - const firstArg = selectionNode.arguments?.find( - (argNode) => argNode.name.value === this.config.firstArgumentName, - ) - const skipArg = selectionNode.arguments?.find( - (argNode) => argNode.name.value === this.config.skipArgumentName, - )?.value as IntValueNode | undefined - if (firstArg != null && firstArg.value.kind === Kind.INT) { - const numberOfTotalRecords = parseInt(firstArg.value.value) - if (numberOfTotalRecords > this.config.limitOfRecords) { - const fieldName = selectionNode.name.value - const aliasName = selectionNode.alias?.value || fieldName - const initialSkip = skipArg?.value ? parseInt(skipArg.value) : 0 - let skip: number - for ( - skip = initialSkip; - numberOfTotalRecords - skip + initialSkip > 0; - skip += this.config.limitOfRecords - ) { - newSelections.push({ - ...selectionNode, - alias: { - kind: Kind.NAME, - value: `splitted_${skip}_${aliasName}`, + if ( + selectionNode.kind === Kind.FIELD && + !selectionNode.name.value.startsWith('_') && + getQueryFieldNames(delegationContext.transformedSchema).includes(selectionNode.name.value) && + !selectionNode.arguments?.some((argNode) => argNode.name.value === 'id') + ) { + const existingArgs: ArgumentNode[] = [] + let firstArg: ArgumentNode | undefined + let skipArg: ArgumentNode | undefined + for (const existingArg of selectionNode.arguments ?? []) { + if (existingArg.name.value === this.config.firstArgumentName) { + firstArg = existingArg + } else if (existingArg.name.value === this.config.skipArgumentName) { + skipArg = existingArg + } else { + existingArgs.push(existingArg) + } + } + if (firstArg != null) { + let numberOfTotalRecords: number | undefined + if (firstArg.value.kind === Kind.INT) { + numberOfTotalRecords = parseInt(firstArg.value.value) + } else if (firstArg.value.kind === Kind.VARIABLE) { + numberOfTotalRecords = executionRequest.variables?.[firstArg.value.name.value] + } + if (numberOfTotalRecords != null && numberOfTotalRecords > this.config.limitOfRecords) { + const fieldName = selectionNode.name.value + const aliasName = selectionNode.alias?.value || fieldName + let initialSkip = 0 + if (skipArg?.value?.kind === Kind.INT) { + initialSkip = parseInt(skipArg.value.value) + } else if (skipArg?.value?.kind === Kind.VARIABLE) { + initialSkip = executionRequest.variables?.[skipArg.value.name.value] + } + let skip: number + for ( + skip = initialSkip; + numberOfTotalRecords - skip + initialSkip > 0; + skip += this.config.limitOfRecords + ) { + newSelections.push({ + ...selectionNode, + alias: { + kind: Kind.NAME, + value: `splitted_${skip}_${aliasName}`, + }, + arguments: [ + ...existingArgs, + { + kind: Kind.ARGUMENT, + name: { + kind: Kind.NAME, + value: this.config.firstArgumentName, + }, + value: { + kind: Kind.INT, + value: Math.min( + numberOfTotalRecords - skip + initialSkip, + this.config.limitOfRecords, + ).toString(), + }, }, - arguments: [ - { - kind: Kind.ARGUMENT, - name: { - kind: Kind.NAME, - value: this.config.firstArgumentName, - }, - value: { - kind: Kind.INT, - value: Math.min( - numberOfTotalRecords - skip + initialSkip, - this.config.limitOfRecords, - ).toString(), - }, + { + kind: Kind.ARGUMENT, + name: { + kind: Kind.NAME, + value: this.config.skipArgumentName, }, - { - kind: Kind.ARGUMENT, - name: { - kind: Kind.NAME, - value: this.config.skipArgumentName, - }, - value: { - kind: Kind.INT, - value: skip.toString(), - }, + value: { + kind: Kind.INT, + value: skip.toString(), }, - ], - }) - } - continue + }, + ], + }) } + continue } } } diff --git a/scripts/canary-release.js b/scripts/canary-release.js new file mode 100644 index 00000000..82cafff1 --- /dev/null +++ b/scripts/canary-release.js @@ -0,0 +1,83 @@ +/* eslint-disable */ +const semver = require('semver') +const cp = require('child_process') +const { basename, join } = require('path') + +const { read: readConfig } = require('@changesets/config') +const readChangesets = require('@changesets/read').default +const assembleReleasePlan = require('@changesets/assemble-release-plan').default +const applyReleasePlan = require('@changesets/apply-release-plan').default +const { getPackages } = require('@manypkg/get-packages') +const { + promises: { unlink }, +} = require('fs') + +function getNewVersion(version, type) { + const gitHash = cp.spawnSync('git', ['rev-parse', '--short', 'HEAD']).stdout.toString().trim() + + return semver.inc(version, `pre${type}`, true, 'canary-' + gitHash) +} + +function getRelevantChangesets(baseBranch) { + const comparePoint = cp + .spawnSync('git', ['merge-base', `origin/${baseBranch}`, 'HEAD']) + .stdout.toString() + .trim() + const listModifiedFiles = cp + .spawnSync('git', ['diff', '--name-only', comparePoint]) + .stdout.toString() + .trim() + .split('\n') + + const items = listModifiedFiles.filter((f) => f.startsWith('.changeset')).map((f) => basename(f, '.md')) + + return items +} + +async function updateVersions() { + const cwd = process.cwd() + // Exit pre mode + // await unlink(join(cwd, '.changeset/pre.json')) + const packages = await getPackages(cwd) + const config = await readConfig(cwd, packages) + const modifiedChangesets = getRelevantChangesets(config.baseBranch) + const changesets = (await readChangesets(cwd)).filter((change) => modifiedChangesets.includes(change.id)) + + if (changesets.length === 0) { + console.warn(`Unable to find any relevant package for canary publishing. Please make sure changesets exists!`) + process.exit(1) + } else { + const releasePlan = assembleReleasePlan(changesets, packages, config, [], false) + + if (releasePlan.releases.length === 0) { + console.warn(`Unable to find any relevant package for canary releasing. Please make sure changesets exists!`) + process.exit(1) + } else { + for (const release of releasePlan.releases) { + if (release.type !== 'none') { + release.newVersion = getNewVersion(release.oldVersion, release.type) + } + } + + await applyReleasePlan( + releasePlan, + packages, + { + ...config, + commit: false, + }, + false, + true, + ) + } + } +} + +updateVersions() + .then(() => { + console.info(`Done!`) + }) + .catch((err) => { + console.error(err) + process.exit(1) + }) diff --git a/yarn.lock b/yarn.lock index a6555bae..22e936c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2548,6 +2548,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/lodash@4.14.182": + version "4.14.182" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.182.tgz#05301a4d5e62963227eaafe0ce04dd77c54ea5c2" + integrity sha512-/THyiqyQAP9AfARo4pF+aCGcyiQ94tX/Is2I7HofNRqoYLgN1PBoOWu2/zTA5zMxzP5EFutMtWtGAFRKUe961Q== + "@types/minimatch@^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"