From 02c4ea8bfa7f9e74f03afbf24fd7d9ddb800ea3d Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Fri, 21 Jun 2024 21:21:34 -0500 Subject: [PATCH 01/11] Tag column migration --- .../14-indexing-rules-add-deployment-tag.ts | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts diff --git a/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts new file mode 100644 index 000000000..403c0af42 --- /dev/null +++ b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts @@ -0,0 +1,57 @@ +import { Logger } from '@graphprotocol/common-ts' +import { DataTypes, QueryInterface } from 'sequelize' + +interface MigrationContext { + queryInterface: QueryInterface + logger: Logger +} + +interface Context { + context: MigrationContext +} + +export async function up({ context }: Context): Promise { + const { queryInterface, logger } = context + + logger.debug(`Checking if indexing rules table exists`) + const tables = await queryInterface.showAllTables() + if (!tables.includes('IndexingRules')) { + logger.info(`Indexing rules table does not exist, migration not necessary`) + return + } + + logger.debug(`Checking if 'IndexingRules' table needs to be migrated`) + const table = await queryInterface.describeTable('IndexingRules') + const subgraphTagColumn = table.tag + if (subgraphTagColumn) { + logger.info( + `'tag' column already exists, migration not necessary`, + ) + return + } + + logger.info(`Add 'tag' column to 'IndexingRules' table`) + await queryInterface.addColumn('IndexingRules', 'tag', { + type: DataTypes.STRING, + primaryKey: false, + defaultValue: 'indexer-agent', + }) + +} + +export async function down({ context }: Context): Promise { + const { queryInterface, logger } = context + + return await queryInterface.sequelize.transaction({}, async transaction => { + const tables = await queryInterface.showAllTables() + + if (tables.includes('IndexingRules')) { + logger.info(`Remove 'tag' column`) + await context.queryInterface.removeColumn( + 'IndexingRules', + 'tag', + { transaction }, + ) + } + }) +} From 5f6c613fdb58b0454eb526da752121586d2b880f Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Fri, 21 Jun 2024 21:47:54 -0500 Subject: [PATCH 02/11] Add tag to IndexingRuleAttributes interface --- .../src/indexer-management/models/indexing-rule.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts index 76ee2b265..ffca19e49 100644 --- a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts +++ b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts @@ -31,6 +31,7 @@ export interface IndexingRuleAttributes { requireSupported: boolean safety: boolean protocolNetwork: string + tag: string } // Unambiguously identify a Indexing Rule in the Database. From 077fa48ae981aebf3f6278503ab185e748d29453 Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Fri, 21 Jun 2024 21:48:43 -0500 Subject: [PATCH 03/11] Add tag to indexer-cli --- packages/indexer-cli/src/rules.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/indexer-cli/src/rules.ts b/packages/indexer-cli/src/rules.ts index e50027849..12526bfc3 100644 --- a/packages/indexer-cli/src/rules.ts +++ b/packages/indexer-cli/src/rules.ts @@ -37,6 +37,7 @@ const INDEXING_RULE_PARSERS: Record requireSupported: x => parseBoolean(x), safety: x => parseBoolean(x), protocolNetwork: x => x, + tag: x => x, } const INDEXING_RULE_FORMATTERS: Record< @@ -61,6 +62,7 @@ const INDEXING_RULE_FORMATTERS: Record< requireSupported: x => x, safety: x => x, protocolNetwork: resolveChainAlias, + tag: x => x, } const INDEXING_RULE_CONVERTERS_FROM_GRAPHQL: Record< @@ -85,6 +87,7 @@ const INDEXING_RULE_CONVERTERS_FROM_GRAPHQL: Record< requireSupported: x => x, safety: x => x, protocolNetwork: x => x, + tag: x => x, } const INDEXING_RULE_CONVERTERS_TO_GRAPHQL: Record< @@ -109,6 +112,7 @@ const INDEXING_RULE_CONVERTERS_TO_GRAPHQL: Record< requireSupported: x => x, safety: x => x, protocolNetwork: x => x, + tag: x => x, } /** @@ -267,6 +271,7 @@ export const indexingRules = async ( decisionBasis requireSupported safety + tag } } `, @@ -307,6 +312,7 @@ export const indexingRule = async ( requireSupported safety protocolNetwork + tag } } `, @@ -350,6 +356,7 @@ export const setIndexingRule = async ( requireSupported safety protocolNetwork + tag } } `, From 0ef024907078f4bdac554f9d5c2433800f6bb9bc Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Fri, 21 Jun 2024 22:29:21 -0500 Subject: [PATCH 04/11] Using deployment tags for deployment name first attempt --- packages/indexer-agent/src/agent.ts | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 517f75e03..c827765cb 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -567,6 +567,34 @@ export class Agent { }, ) + const deploymentTags: Eventual> = join({ + ticker: timer(120_000), + indexingRules, + }).tryMap( + async ({ indexingRules }) => { + logger.trace('Resolving deployment tags') + const deploymentTags: Map = {} + + // Add offchain subgraphs to the deployment list from rules + Object.values(indexingRules) + .flat() + .filter( + rule => rule?.decisionBasis === IndexingDecisionBasis.OFFCHAIN, + ) + .forEach(rule => { + deploymentTags.set(rule.identifier, rule.tag) + }) + return deploymentTags + }, + { + onError: error => + logger.warn( + `Failed to resolve deployment tags, trying again later`, + { error }, + ), + }, + ) + const activeAllocations: Eventual> = timer( 120_000, ).tryMap( @@ -904,6 +932,7 @@ export class Agent { activeDeployments: SubgraphDeploymentID[], targetDeployments: SubgraphDeploymentID[], eligibleAllocations: Allocation[], + deploymentTags: Map, ): Promise { const logger = this.logger.child({ function: 'reconcileDeployments' }) logger.debug('Reconcile deployments') @@ -974,7 +1003,7 @@ export class Agent { // Index all new deployments worth indexing await queue.addAll( deploy.map(deployment => async () => { - const name = `indexer-agent/${deployment.ipfsHash.slice(-10)}` + const name = `${deploymentTags.get(deployment.display) ? deploymentTags.get(deployment.display) : 'indexer-agent'}/${deployment.ipfsHash.slice(-10)}` logger.info(`Index subgraph deployment`, { name, From 8aac036dee43c4e0195848a7ca555980daf5e244 Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 00:35:20 -0500 Subject: [PATCH 05/11] Most of the TS, plus prettier --- packages/indexer-agent/src/agent.ts | 27 +++++++++++++------ .../14-indexing-rules-add-deployment-tag.ts | 13 +++------ .../src/indexer-management/__tests__/util.ts | 2 ++ .../models/indexing-rule.ts | 8 ++++++ packages/indexer-common/src/rules.ts | 1 + 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index c827765cb..5d5e96049 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -573,7 +573,7 @@ export class Agent { }).tryMap( async ({ indexingRules }) => { logger.trace('Resolving deployment tags') - const deploymentTags: Map = {} + const deploymentTags: Map = new Map() // Add offchain subgraphs to the deployment list from rules Object.values(indexingRules) @@ -582,16 +582,18 @@ export class Agent { rule => rule?.decisionBasis === IndexingDecisionBasis.OFFCHAIN, ) .forEach(rule => { - deploymentTags.set(rule.identifier, rule.tag) + deploymentTags.set( + new SubgraphDeploymentID(rule.identifier), + rule.tag, + ) }) - return deploymentTags + return new Map(deploymentTags) }, { onError: error => - logger.warn( - `Failed to resolve deployment tags, trying again later`, - { error }, - ), + logger.warn(`Failed to resolve deployment tags, trying again later`, { + error, + }), }, ) @@ -740,6 +742,7 @@ export class Agent { activeDeployments, targetDeployments, eligibleAllocations, + deploymentTags, ) } catch (err) { logger.warn( @@ -1003,7 +1006,15 @@ export class Agent { // Index all new deployments worth indexing await queue.addAll( deploy.map(deployment => async () => { - const name = `${deploymentTags.get(deployment.display) ? deploymentTags.get(deployment.display) : 'indexer-agent'}/${deployment.ipfsHash.slice(-10)}` + const name = `${ + deploymentTags.get( + new SubgraphDeploymentID(deployment.display.ipfsHash), + ) + ? deploymentTags.get( + new SubgraphDeploymentID(deployment.display.ipfsHash), + ) + : 'indexer-agent' + }/${deployment.ipfsHash.slice(-10)}` logger.info(`Index subgraph deployment`, { name, diff --git a/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts index 403c0af42..87f0da2ad 100644 --- a/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts +++ b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts @@ -24,9 +24,7 @@ export async function up({ context }: Context): Promise { const table = await queryInterface.describeTable('IndexingRules') const subgraphTagColumn = table.tag if (subgraphTagColumn) { - logger.info( - `'tag' column already exists, migration not necessary`, - ) + logger.info(`'tag' column already exists, migration not necessary`) return } @@ -36,7 +34,6 @@ export async function up({ context }: Context): Promise { primaryKey: false, defaultValue: 'indexer-agent', }) - } export async function down({ context }: Context): Promise { @@ -47,11 +44,9 @@ export async function down({ context }: Context): Promise { if (tables.includes('IndexingRules')) { logger.info(`Remove 'tag' column`) - await context.queryInterface.removeColumn( - 'IndexingRules', - 'tag', - { transaction }, - ) + await context.queryInterface.removeColumn('IndexingRules', 'tag', { + transaction, + }) } }) } diff --git a/packages/indexer-common/src/indexer-management/__tests__/util.ts b/packages/indexer-common/src/indexer-management/__tests__/util.ts index 1e08341dd..2a8628119 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/util.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/util.ts @@ -101,6 +101,7 @@ export const createTestManagementClient = async ( requireSupported: true, safety: true, protocolNetwork: 'sepolia', + tag: 'indexer-agent', }, } @@ -137,6 +138,7 @@ export const defaults: IndexerManagementDefaults = { parallelAllocations: 1, requireSupported: true, safety: true, + tag: 'indexer-agent', }, } diff --git a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts index ffca19e49..df48f2523 100644 --- a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts +++ b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts @@ -61,6 +61,7 @@ export interface IndexingRuleCreationAttributes | 'requireSupported' | 'safety' | 'protocolNetwork' + | 'tag' > {} export class IndexingRule @@ -84,6 +85,7 @@ export class IndexingRule public requireSupported!: boolean public safety!: boolean public protocolNetwork!: string + public tag!: string public createdAt!: Date public updatedAt!: Date @@ -268,6 +270,12 @@ export const defineIndexingRuleModels = (sequelize: Sequelize): IndexingRuleMode is: caip2IdRegex, }, }, + tag: { + type: DataTypes.STRING, + primaryKey: true, + allowNull: false, + defaultValue: 'indexer-agent', + }, }, { modelName: 'IndexingRule', diff --git a/packages/indexer-common/src/rules.ts b/packages/indexer-common/src/rules.ts index e03bfd40e..eabe3ec14 100644 --- a/packages/indexer-common/src/rules.ts +++ b/packages/indexer-common/src/rules.ts @@ -36,6 +36,7 @@ const INDEXING_RULE_READABLE_TO_MODEL_PARSERS: Record< requireSupported: (x) => parseBoolean(x), safety: (x) => parseBoolean(x), protocolNetwork: (x: string) => validateNetworkIdentifier(x), + tag: (x: string) => x, } export const parseIndexingRule = ( From bedf5188dfc86c8d5ec0321fb7009a5c32367c54 Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 10:02:07 -0500 Subject: [PATCH 06/11] Resolved deployment tags workaround attempt --- packages/indexer-agent/src/agent.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 5d5e96049..237b139ff 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -738,11 +738,12 @@ export class Agent { switch (this.deploymentManagement) { case DeploymentManagementMode.AUTO: try { + const resolvedDeploymentTags = await deploymentTags.value() await this.reconcileDeployments( activeDeployments, targetDeployments, eligibleAllocations, - deploymentTags, + resolvedDeploymentTags, ) } catch (err) { logger.warn( From 3ad52a0de59224e0cc8f8cecd4e4ff37802451cc Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 11:25:45 -0500 Subject: [PATCH 07/11] Add tag to IndexingRuleInput type --- .../__tests__/resolvers/indexing-rules.test.ts | 1 + packages/indexer-common/src/indexer-management/client.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/indexer-common/src/indexer-management/__tests__/resolvers/indexing-rules.test.ts b/packages/indexer-common/src/indexer-management/__tests__/resolvers/indexing-rules.test.ts index 930bc1523..4023381eb 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/resolvers/indexing-rules.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/resolvers/indexing-rules.test.ts @@ -45,6 +45,7 @@ const SET_INDEXING_RULE_MUTATION = gql` requireSupported safety protocolNetwork + tag } } ` diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index 648dea8cc..5d39b8869 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -263,6 +263,7 @@ const SCHEMA_SDL = gql` requireSupported: Boolean! safety: Boolean! protocolNetwork: String! + tag: String } input IndexingRuleInput { @@ -282,6 +283,7 @@ const SCHEMA_SDL = gql` requireSupported: Boolean safety: Boolean protocolNetwork: String! + tag: String } input IndexingRuleIdentifier { From 78a275c91a9e3004ccbacfc2ec00f6508796073b Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 11:57:58 -0500 Subject: [PATCH 08/11] Add tag to indexing rules query --- packages/indexer-common/src/operator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/indexer-common/src/operator.ts b/packages/indexer-common/src/operator.ts index 1d97904c4..dde4f39a5 100644 --- a/packages/indexer-common/src/operator.ts +++ b/packages/indexer-common/src/operator.ts @@ -109,6 +109,7 @@ export class Operator { decisionBasis requireSupported protocolNetwork + tag } } `, From daeec2cc1e03926791a217e4c57c45852132b29b Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 13:27:51 -0500 Subject: [PATCH 09/11] Changed deploymentTags Map from to for simplicity --- packages/indexer-agent/src/agent.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 237b139ff..7c0c3c3c5 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -567,13 +567,13 @@ export class Agent { }, ) - const deploymentTags: Eventual> = join({ + const deploymentTags: Eventual> = join({ ticker: timer(120_000), indexingRules, }).tryMap( async ({ indexingRules }) => { logger.trace('Resolving deployment tags') - const deploymentTags: Map = new Map() + const deploymentTags: Map = new Map() // Add offchain subgraphs to the deployment list from rules Object.values(indexingRules) @@ -583,7 +583,7 @@ export class Agent { ) .forEach(rule => { deploymentTags.set( - new SubgraphDeploymentID(rule.identifier), + new SubgraphDeploymentID(rule.identifier).toString(), rule.tag, ) }) @@ -936,7 +936,7 @@ export class Agent { activeDeployments: SubgraphDeploymentID[], targetDeployments: SubgraphDeploymentID[], eligibleAllocations: Allocation[], - deploymentTags: Map, + deploymentTags: Map, ): Promise { const logger = this.logger.child({ function: 'reconcileDeployments' }) logger.debug('Reconcile deployments') @@ -1008,12 +1008,8 @@ export class Agent { await queue.addAll( deploy.map(deployment => async () => { const name = `${ - deploymentTags.get( - new SubgraphDeploymentID(deployment.display.ipfsHash), - ) - ? deploymentTags.get( - new SubgraphDeploymentID(deployment.display.ipfsHash), - ) + deploymentTags.get(deployment.toString()) + ? deploymentTags.get(deployment.toString()) : 'indexer-agent' }/${deployment.ipfsHash.slice(-10)}` From 5e1d4f2a82317c7845d1437b2b56b4c51539c260 Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 18:45:11 -0500 Subject: [PATCH 10/11] Filter out global rules and nothing else, simplify deployment name creation, reduce timer --- packages/indexer-agent/src/agent.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 7c0c3c3c5..70f896ad1 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -568,7 +568,7 @@ export class Agent { ) const deploymentTags: Eventual> = join({ - ticker: timer(120_000), + ticker: timer(60_000), indexingRules, }).tryMap( async ({ indexingRules }) => { @@ -578,9 +578,7 @@ export class Agent { // Add offchain subgraphs to the deployment list from rules Object.values(indexingRules) .flat() - .filter( - rule => rule?.decisionBasis === IndexingDecisionBasis.OFFCHAIN, - ) + .filter(rule => rule?.identifier !== 'global') .forEach(rule => { deploymentTags.set( new SubgraphDeploymentID(rule.identifier).toString(), @@ -1008,9 +1006,7 @@ export class Agent { await queue.addAll( deploy.map(deployment => async () => { const name = `${ - deploymentTags.get(deployment.toString()) - ? deploymentTags.get(deployment.toString()) - : 'indexer-agent' + deploymentTags.get(deployment.toString()) || 'indexer-agent' }/${deployment.ipfsHash.slice(-10)}` logger.info(`Index subgraph deployment`, { From 73f81d39e19727ff91922f2e46b69a1a3adbc2bf Mon Sep 17 00:00:00 2001 From: Vincent Taglia Date: Sat, 22 Jun 2024 19:04:24 -0500 Subject: [PATCH 11/11] Getting the DB schema and indexer management schema on the same page --- .../src/db/migrations/14-indexing-rules-add-deployment-tag.ts | 1 + .../src/indexer-management/models/indexing-rule.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts index 87f0da2ad..213fb9e7d 100644 --- a/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts +++ b/packages/indexer-agent/src/db/migrations/14-indexing-rules-add-deployment-tag.ts @@ -32,6 +32,7 @@ export async function up({ context }: Context): Promise { await queryInterface.addColumn('IndexingRules', 'tag', { type: DataTypes.STRING, primaryKey: false, + allowNull: false, defaultValue: 'indexer-agent', }) } diff --git a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts index df48f2523..48cba049b 100644 --- a/packages/indexer-common/src/indexer-management/models/indexing-rule.ts +++ b/packages/indexer-common/src/indexer-management/models/indexing-rule.ts @@ -272,7 +272,7 @@ export const defineIndexingRuleModels = (sequelize: Sequelize): IndexingRuleMode }, tag: { type: DataTypes.STRING, - primaryKey: true, + primaryKey: false, allowNull: false, defaultValue: 'indexer-agent', },