From 26652d76bb8c9f91c35185fc95c63de6a156ab9b Mon Sep 17 00:00:00 2001 From: ilija42 <57732589+ilija42@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:58:26 +0200 Subject: [PATCH] BCFG-2637 Change evmChainID not null migration to use env var to inject goose (#10711) * Change evmChainID not null migration to use env var to inject goose * Update changelog * Add sonar qube no scan to goose migration script * Fix BEGIN-NOSCAN sonarqube comment * Fix sonarqube no scan comment * Move 0195 migration env var to env package * Exclude 0195 migration script from sq scan * Exclude 0195 migration script from sq scan * Fix exclude 0195 migration script from sq scan * Update sonar qube tests artifacts output path --- core/cmd/shell.go | 41 +------------------ core/cmd/shell_local.go | 14 +------ core/config/env/env.go | 2 + core/store/migrate/migrate.go | 13 ++++++ core/store/migrate/migrate_test.go | 36 ++++++++++++++++ ...d_not_null_to_evm_chain_id_in_job_specs.go | 22 ++++++++++ docs/CHANGELOG.md | 2 +- sonar-project.properties | 2 +- 8 files changed, 77 insertions(+), 55 deletions(-) diff --git a/core/cmd/shell.go b/core/cmd/shell.go index b9c031fd8b7..2a771a83587 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -133,8 +133,7 @@ type ChainlinkAppFactory struct{} func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, db *sqlx.DB) (app chainlink.Application, err error) { initGlobals(cfg.Prometheus()) - // TODO TO BE REMOVED IN v2.7.0 - err = evmChainIDMigration(cfg, db.DB, appLggr) + err = migrate.SetMigrationENVVars(cfg) if err != nil { return nil, err } @@ -301,44 +300,6 @@ func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbacku return err } -// evmChainIDMigration TODO TO BE REMOVED IN v2.7.0. This is a helper function for evmChainID 0195 migration in v2.6.0 only, so that we don't have to inject evmChainID into goose. -func evmChainIDMigration(generalConfig chainlink.GeneralConfig, db *sql.DB, lggr logger.Logger) error { - migrationVer, err := migrate.Current(db, lggr) - if err != nil { - return err - } - if migrationVer != 194 { - return nil - } - - if generalConfig.EVMEnabled() { - if generalConfig.EVMConfigs() == nil { - return errors.New("evm configs are missing") - } - if generalConfig.EVMConfigs()[0] == nil { - return errors.New("evm config is nil") - } - updateQueries := []string{ - `UPDATE direct_request_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE flux_monitor_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE ocr_oracle_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE keeper_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE vrf_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE blockhash_store_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - `UPDATE block_header_feeder_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, - } - - chainID := generalConfig.EVMConfigs()[0].ChainID.String() - for i := range updateQueries { - _, err := db.Exec(updateQueries[i], chainID) - if err != nil { - return errors.Wrap(err, "failed to set missing evm chain ids") - } - } - } - return nil -} - // Runner implements the Run method. type Runner interface { Run(context.Context, chainlink.Application) error diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index 26c0b5ed802..051f28d8e7a 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -840,19 +840,7 @@ func (s *Shell) MigrateDatabase(_ *cli.Context) error { return s.errorOut(errDBURLMissing) } - // TODO TO BE REMOVED IN v2.7.0 - db, err := sql.Open(string(dialects.Postgres), parsed.String()) - if err != nil { - return fmt.Errorf("unable to open postgres database for evmChainID helper migration: %+v", err) - } - defer func() { - if cerr := db.Close(); cerr != nil { - err = multierr.Append(err, cerr) - } - }() - - // TODO TO BE REMOVED IN v2.7.0 - err = evmChainIDMigration(s.Config, db, s.Logger) + err := migrate.SetMigrationENVVars(s.Config) if err != nil { return err } diff --git a/core/config/env/env.go b/core/config/env/env.go index 7813df4a8cc..ea84a50b754 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -36,6 +36,8 @@ var ( PyroscopeAuthToken = Secret("CL_PYROSCOPE_AUTH_TOKEN") PrometheusAuthToken = Secret("CL_PROMETHEUS_AUTH_TOKEN") ThresholdKeyShare = Secret("CL_THRESHOLD_KEY_SHARE") + // Migrations env vars + EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195" ) type Var string diff --git a/core/store/migrate/migrate.go b/core/store/migrate/migrate.go index 97714b68bef..69cf9a78247 100644 --- a/core/store/migrate/migrate.go +++ b/core/store/migrate/migrate.go @@ -14,7 +14,9 @@ import ( "github.com/smartcontractkit/sqlx" "gopkg.in/guregu/null.v4" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/pg" "github.com/smartcontractkit/chainlink/v2/core/store/migrate/migrations" // Invoke init() functions within migrations pkg. ) @@ -131,3 +133,14 @@ func Status(db *sql.DB, lggr logger.Logger) error { func Create(db *sql.DB, name, migrationType string) error { return goose.Create(db, "core/store/migrate/migrations", name, migrationType) } + +// SetMigrationENVVars is used to inject values from config to goose migrations via env. +func SetMigrationENVVars(generalConfig chainlink.GeneralConfig) error { + if generalConfig.EVMEnabled() { + err := os.Setenv(env.EVMChainIDNotNullMigration0195, generalConfig.EVMConfigs()[0].ChainID.String()) + if err != nil { + panic(errors.Wrap(err, "failed to set migrations env variables")) + } + } + return nil +} diff --git a/core/store/migrate/migrate_test.go b/core/store/migrate/migrate_test.go index 66764a266f0..d6135ce4529 100644 --- a/core/store/migrate/migrate_test.go +++ b/core/store/migrate/migrate_test.go @@ -1,6 +1,8 @@ package migrate_test import ( + "math/big" + "os" "testing" "time" @@ -11,13 +13,19 @@ import ( "gopkg.in/guregu/null.v4" "github.com/smartcontractkit/chainlink-relay/pkg/types" + + evmcfg "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" + "github.com/smartcontractkit/chainlink/v2/core/config/env" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest/heavyweight" + configtest "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest/v2" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/relay" "github.com/smartcontractkit/chainlink/v2/core/store/migrate" "github.com/smartcontractkit/chainlink/v2/core/store/models" + "github.com/smartcontractkit/chainlink/v2/core/utils" ) var migrationDir = "migrations" @@ -401,3 +409,31 @@ func TestMigrate(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(99), ver) } + +func TestSetMigrationENVVars(t *testing.T) { + t.Run("ValidEVMConfig", func(t *testing.T) { + chainID := utils.NewBig(big.NewInt(1337)) + testConfig := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { + evmEnabled := true + c.EVM = evmcfg.EVMConfigs{&evmcfg.EVMConfig{ + ChainID: chainID, + Enabled: &evmEnabled, + }} + }) + + require.NoError(t, migrate.SetMigrationENVVars(testConfig)) + + actualChainID := os.Getenv(env.EVMChainIDNotNullMigration0195) + require.Equal(t, actualChainID, chainID.String()) + }) + + t.Run("EVMConfigMissing", func(t *testing.T) { + chainID := utils.NewBig(big.NewInt(1337)) + testConfig := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.EVM = nil }) + + require.NoError(t, migrate.SetMigrationENVVars(testConfig)) + + actualChainID := os.Getenv(env.EVMChainIDNotNullMigration0195) + require.Equal(t, actualChainID, chainID.String()) + }) +} diff --git a/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go b/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go index 4b722c08ba2..b1387cc51f0 100644 --- a/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go +++ b/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go @@ -3,9 +3,12 @@ package migrations import ( "context" "database/sql" + "os" "github.com/pkg/errors" "github.com/pressly/goose/v3" + + "github.com/smartcontractkit/chainlink/v2/core/config/env" ) func init() { @@ -36,6 +39,25 @@ const ( // nolint func Up195(ctx context.Context, tx *sql.Tx) error { + chainID, set := os.LookupEnv(env.EVMChainIDNotNullMigration0195) + if set { + updateQueries := []string{ + `UPDATE direct_request_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE flux_monitor_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE ocr_oracle_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE keeper_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE vrf_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE blockhash_store_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE block_header_feeder_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + } + for i := range updateQueries { + _, err := tx.Exec(updateQueries[i], chainID) + if err != nil { + return errors.Wrap(err, "failed to set missing evm chain ids") + } + } + } + _, err := tx.ExecContext(ctx, addNullConstraintsToSpecs) return errors.Wrap(err, "failed to add null constraints") } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b95cc614fb4..5bc19cbab8f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Temporary helper function for proper migration of evmChainID not null in specs, that fills in missing chainIDs. This needs to be removed after one version after everyone migrated properly. For proper migrations all nodes must upgrade sequentially without skipping this version. +- Helper migrations function for injecting env vars into goose migrations. This was done to inject chainID into evm chain id not null in specs migrations. ### Removed diff --git a/sonar-project.properties b/sonar-project.properties index 28f9ca8a84c..ea142ce17fe 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,7 +5,7 @@ sonar.python.version=3.8 # Full exclusions from the static analysis sonar.exclusions=**/node_modules/**/*,**/mocks/**/*, **/testdata/**/*, **/contracts/typechain/**/*, **/contracts/artifacts/**/*, **/contracts/cache/**/*, **/contracts/scripts/**/*, **/generated/**/*, **/fixtures/**/*, **/docs/**/*, **/tools/**/*, **/*.pb.go, **/*report.xml, **/*.config.ts, **/*.txt, **/*.abi, **/*.bin, **/*_codecgen.go # Coverage exclusions -sonar.coverage.exclusions=**/*.test.ts, **/*_test.go, **/contracts/test/**/*, **/contracts/**/tests/**/*, **/core/**/testutils/**/*, **/core/**/mocks/**/*, **/core/**/cltest/**/*, **/integration-tests/**/*, **/generated/**/*, **/core/scripts**/* , **/*.pb.go, ./plugins/**/*, **/main.go +sonar.coverage.exclusions=**/*.test.ts, **/*_test.go, **/contracts/test/**/*, **/contracts/**/tests/**/*, **/core/**/testutils/**/*, **/core/**/mocks/**/*, **/core/**/cltest/**/*, **/integration-tests/**/*, **/generated/**/*, **/core/scripts**/* , **/*.pb.go, ./plugins/**/*, **/main.go, **/0195_add_not_null_to_evm_chain_id_in_job_specs.go # Duplication exclusions sonar.cpd.exclusions=**/contracts/**/*.sol, **/config.go, /core/services/ocr2/plugins/ocr2keeper/evm*/*