From a52b3a769005e25a6ca2a9a5d16770b4339f9009 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Feb 2024 11:01:58 +0100 Subject: [PATCH 1/2] stores: fix metric migrations --- stores/migrations.go | 60 ++------------------------- stores/migrations_metrics.go | 29 ++----------- stores/migrations_utils.go | 79 ++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 83 deletions(-) create mode 100644 stores/migrations_utils.go diff --git a/stores/migrations.go b/stores/migrations.go index e79c6c36b..05bda3f69 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/go-gormigrate/gormigrate/v2" - "go.sia.tech/renterd/api" "go.uber.org/zap" "gorm.io/gorm" ) @@ -16,32 +15,6 @@ var ( errMySQLNoSuperPrivilege = errors.New("You do not have the SUPER privilege and binary logging is enabled") ) -// initSchema is executed only on a clean database. Otherwise the individual -// migrations are executed. -func initSchema(tx *gorm.DB) (err error) { - // Pick the right migrations. - var schema []byte - if isSQLite(tx) { - schema, err = migrations.ReadFile("migrations/sqlite/main/schema.sql") - } else { - schema, err = migrations.ReadFile("migrations/mysql/main/schema.sql") - } - if err != nil { - return - } - - // Run it. - err = tx.Exec(string(schema)).Error - if err != nil { - return fmt.Errorf("failed to init schema: %w", err) - } - - // Add default bucket. - return tx.Create(&dbBucket{ - Name: api.DefaultBucketName, - }).Error -} - func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { migrations := []*gormigrate.Migration{ { @@ -51,13 +24,13 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { { ID: "00001_object_metadata", Migrate: func(tx *gorm.DB) error { - return performMigration(tx, "00001_object_metadata", logger) + return performMigration(tx, "00001_object_metadata", false, logger) }, }, { ID: "00002_prune_slabs_trigger", Migrate: func(tx *gorm.DB) error { - err := performMigration(tx, "00002_prune_slabs_trigger", logger) + err := performMigration(tx, "00002_prune_slabs_trigger", false, logger) if err != nil && strings.Contains(err.Error(), errMySQLNoSuperPrivilege.Error()) { logger.Warn("migration 00002_prune_slabs_trigger requires the user to have the SUPER privilege to register triggers") } @@ -70,7 +43,7 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { m := gormigrate.New(db, gormigrate.DefaultOptions, migrations) // Set init function. - m.InitSchema(initSchema) + m.InitSchema(initSchema(db, false, logger)) // Perform migrations. if err := m.Migrate(); err != nil { @@ -78,30 +51,3 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { } return nil } - -func performMigration(db *gorm.DB, name string, logger *zap.SugaredLogger) error { - logger.Infof("performing migration %s", name) - - // build path - var path string - if isSQLite(db) { - path = fmt.Sprintf("migrations/sqlite/main/migration_" + name + ".sql") - } else { - path = fmt.Sprintf("migrations/mysql/main/migration_" + name + ".sql") - } - - // read migration file - migration, err := migrations.ReadFile(path) - if err != nil { - return fmt.Errorf("migration %s failed: %w", name, err) - } - - // execute it - err = db.Exec(string(migration)).Error - if err != nil { - return fmt.Errorf("migration %s failed: %w", name, err) - } - - logger.Infof("migration %s complete", name) - return nil -} diff --git a/stores/migrations_metrics.go b/stores/migrations_metrics.go index a95d7b914..ddbbe8e4e 100644 --- a/stores/migrations_metrics.go +++ b/stores/migrations_metrics.go @@ -8,30 +8,7 @@ import ( "gorm.io/gorm" ) -// initMetricsSchema is executed only on a clean database. Otherwise the individual -// migrations are executed. -func initMetricsSchema(tx *gorm.DB) error { - // Pick the right migrations. - var schema []byte - var err error - if isSQLite(tx) { - schema, err = migrations.ReadFile("migrations/sqlite/metrics/schema.sql") - } else { - schema, err = migrations.ReadFile("migrations/mysql/metrics/schema.sql") - } - if err != nil { - return err - } - - // Run it. - err = tx.Exec(string(schema)).Error - if err != nil { - return fmt.Errorf("failed to init schema: %w", err) - } - return nil -} - -func performMetricsMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { +func performMetricsMigrations(tx *gorm.DB, logger *zap.SugaredLogger) error { migrations := []*gormigrate.Migration{ { ID: "00001_init", @@ -40,10 +17,10 @@ func performMetricsMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { } // Create migrator. - m := gormigrate.New(db, gormigrate.DefaultOptions, migrations) + m := gormigrate.New(tx, gormigrate.DefaultOptions, migrations) // Set init function. - m.InitSchema(initMetricsSchema) + m.InitSchema(initSchema(tx, true, logger)) // Perform migrations. if err := m.Migrate(); err != nil { diff --git a/stores/migrations_utils.go b/stores/migrations_utils.go new file mode 100644 index 000000000..4832d96a7 --- /dev/null +++ b/stores/migrations_utils.go @@ -0,0 +1,79 @@ +package stores + +import ( + "fmt" + + gormigrate "github.com/go-gormigrate/gormigrate/v2" + "go.sia.tech/renterd/api" + "go.uber.org/zap" + "gorm.io/gorm" +) + +// initSchema is executed only on a clean database. Otherwise the individual +// migrations are executed. +func initSchema(db *gorm.DB, metrics bool, logger *zap.SugaredLogger) gormigrate.InitSchemaFunc { + return func(tx *gorm.DB) error { + if metrics { + logger.Info("initializing metrics schema") + } else { + logger.Info("initializing schema") + } + + // build filename + filename := "schema" + err := execSQLFile(tx, metrics, filename) + if err != nil { + return fmt.Errorf("failed to init schema: %w", err) + } + + // add default bucket. + if !metrics { + if err := tx.Create(&dbBucket{ + Name: api.DefaultBucketName, + }).Error; err != nil { + return fmt.Errorf("failed to create default bucket: %v", err) + } + } + + logger.Info("initialization complete") + return nil + } +} + +func performMigration(db *gorm.DB, name string, metrics bool, logger *zap.SugaredLogger) error { + logger.Infof("performing migration %s", name) + + // build filename + filename := fmt.Sprintf("migration_%s", name) + + // execute migration + err := execSQLFile(db, metrics, filename) + if err != nil { + return fmt.Errorf("migration %s failed: %w", name, err) + } + + logger.Infof("migration %s complete", name) + return nil +} + +func execSQLFile(db *gorm.DB, metrics bool, filename string) error { + // build path + folder := "main" + if metrics { + folder = "metrics" + } + protocol := "mysql" + if isSQLite(db) { + protocol = "sqlite" + } + path := fmt.Sprintf("migrations/%s/%s/%s.sql", protocol, folder, filename) + + // read file + file, err := migrations.ReadFile(path) + if err != nil { + return err + } + + // execute it + return db.Exec(string(file)).Error +} From 9051e472f06409d5e4613375b47f17f0980933fb Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 16 Feb 2024 15:46:35 +0100 Subject: [PATCH 2/2] stores: get rid of migrate bool --- stores/migrations.go | 6 ++-- stores/migrations/mysql/main/schema.sql | 5 ++- stores/migrations/sqlite/main/schema.sql | 5 ++- stores/migrations_metrics.go | 2 +- stores/migrations_utils.go | 42 ++++++------------------ 5 files changed, 22 insertions(+), 38 deletions(-) diff --git a/stores/migrations.go b/stores/migrations.go index 05bda3f69..7f225ff17 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -24,13 +24,13 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { { ID: "00001_object_metadata", Migrate: func(tx *gorm.DB) error { - return performMigration(tx, "00001_object_metadata", false, logger) + return performMigration(tx, "main", "00001_object_metadata", logger) }, }, { ID: "00002_prune_slabs_trigger", Migrate: func(tx *gorm.DB) error { - err := performMigration(tx, "00002_prune_slabs_trigger", false, logger) + err := performMigration(tx, "main", "00002_prune_slabs_trigger", logger) if err != nil && strings.Contains(err.Error(), errMySQLNoSuperPrivilege.Error()) { logger.Warn("migration 00002_prune_slabs_trigger requires the user to have the SUPER privilege to register triggers") } @@ -43,7 +43,7 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { m := gormigrate.New(db, gormigrate.DefaultOptions, migrations) // Set init function. - m.InitSchema(initSchema(db, false, logger)) + m.InitSchema(initSchema(db, "main", logger)) // Perform migrations. if err := m.Migrate(); err != nil { diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 39bf279f0..d28bdd13f 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -452,4 +452,7 @@ AND NOT EXISTS ( SELECT 1 FROM slices WHERE slices.db_slab_id = OLD.db_slab_id -); \ No newline at end of file +); + +-- create default bucket +INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index df9fc9a83..daee619b4 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -183,4 +183,7 @@ BEGIN FROM slices WHERE slices.db_slab_id = OLD.db_slab_id ); -END; \ No newline at end of file +END; + +-- create default bucket +INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); diff --git a/stores/migrations_metrics.go b/stores/migrations_metrics.go index ddbbe8e4e..940917569 100644 --- a/stores/migrations_metrics.go +++ b/stores/migrations_metrics.go @@ -20,7 +20,7 @@ func performMetricsMigrations(tx *gorm.DB, logger *zap.SugaredLogger) error { m := gormigrate.New(tx, gormigrate.DefaultOptions, migrations) // Set init function. - m.InitSchema(initSchema(tx, true, logger)) + m.InitSchema(initSchema(tx, "metrics", logger)) // Perform migrations. if err := m.Migrate(); err != nil { diff --git a/stores/migrations_utils.go b/stores/migrations_utils.go index 4832d96a7..46d7f3dc4 100644 --- a/stores/migrations_utils.go +++ b/stores/migrations_utils.go @@ -4,64 +4,42 @@ import ( "fmt" gormigrate "github.com/go-gormigrate/gormigrate/v2" - "go.sia.tech/renterd/api" "go.uber.org/zap" "gorm.io/gorm" ) // initSchema is executed only on a clean database. Otherwise the individual // migrations are executed. -func initSchema(db *gorm.DB, metrics bool, logger *zap.SugaredLogger) gormigrate.InitSchemaFunc { +func initSchema(db *gorm.DB, name string, logger *zap.SugaredLogger) gormigrate.InitSchemaFunc { return func(tx *gorm.DB) error { - if metrics { - logger.Info("initializing metrics schema") - } else { - logger.Info("initializing schema") - } + logger.Infof("initializing '%s' schema", name) - // build filename - filename := "schema" - err := execSQLFile(tx, metrics, filename) + // init schema + err := execSQLFile(tx, name, "schema") if err != nil { return fmt.Errorf("failed to init schema: %w", err) } - // add default bucket. - if !metrics { - if err := tx.Create(&dbBucket{ - Name: api.DefaultBucketName, - }).Error; err != nil { - return fmt.Errorf("failed to create default bucket: %v", err) - } - } - logger.Info("initialization complete") return nil } } -func performMigration(db *gorm.DB, name string, metrics bool, logger *zap.SugaredLogger) error { - logger.Infof("performing migration %s", name) - - // build filename - filename := fmt.Sprintf("migration_%s", name) +func performMigration(db *gorm.DB, kind, migration string, logger *zap.SugaredLogger) error { + logger.Infof("performing %s migration '%s'", kind, migration) // execute migration - err := execSQLFile(db, metrics, filename) + err := execSQLFile(db, kind, fmt.Sprintf("migration_%s", migration)) if err != nil { - return fmt.Errorf("migration %s failed: %w", name, err) + return fmt.Errorf("migration '%s' failed: %w", migration, err) } - logger.Infof("migration %s complete", name) + logger.Infof("migration '%s' complete", migration) return nil } -func execSQLFile(db *gorm.DB, metrics bool, filename string) error { +func execSQLFile(db *gorm.DB, folder, filename string) error { // build path - folder := "main" - if metrics { - folder = "metrics" - } protocol := "mysql" if isSQLite(db) { protocol = "sqlite"