feat: embed MongoDB migrations in the binary (#4540)

* feat: use fs.FS for DbMigrator
* chore: add unit tests for getting MongoDB migrations from FS
* feat: embed MongoDB migrations in binary

cmd/api-server/main.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/kubeshop/testkube/internal/common"
 	"github.com/kubeshop/testkube/internal/config"
+	dbmigrations "github.com/kubeshop/testkube/internal/db-migrations"
 	parser "github.com/kubeshop/testkube/internal/template"
 	"github.com/kubeshop/testkube/pkg/version"
 
@@ -106,7 +107,7 @@ func runMigrations() (err error) {
 
 func runMongoMigrations(ctx context.Context, db *mongo.Database, migrationsDir string) error {
 	migrationsCollectionName := "__migrations"
-	activeMigrations, err := dbmigrator.GetDbMigrationsFromDir(migrationsDir)
+	activeMigrations, err := dbmigrator.GetDbMigrationsFromFs(dbmigrations.MongoMigrationsFs)
 	if err != nil {
 		return errors.Wrap(err, "failed to obtain MongoDB migrations from disk")
 	}

go.mod

@@ -118,6 +118,7 @@ require (
 	github.com/segmentio/backo-go v1.0.1 // indirect
 	github.com/shurcooL/githubv4 v0.0.0-20220922232305-70b4d362a8cb // indirect
 	github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
 	github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e // indirect
 	github.com/urfave/cli/v2 v2.24.4 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

internal/db-migrations/fs.go

@@ -0,0 +1,6 @@
+package db_migrations
+
+import "embed"
+
+//go:embed *.up.json *.down.json
+var MongoMigrationsFs embed.FS

pkg/dbmigrator/dbmigrator.go

@@ -2,10 +2,11 @@ package dbmigrator
 
 import (
 	"context"
-	"os"
+	"io/fs"
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"sort"
 
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
@@ -29,11 +30,14 @@ type DbMigrator struct {
 	list []DbMigration
 }
 
-func GetDbMigrationsFromDir(dirPath string) ([]DbMigration, error) {
-	filePaths, err := filepath.Glob(filepath.Join(dirPath, "*.json"))
+func GetDbMigrationsFromFs(fsys fs.FS) ([]DbMigration, error) {
+	filePaths, err := fs.Glob(fsys, "*.json")
 	if err != nil {
 		return nil, err
 	}
+	sort.Slice(filePaths, func(i, j int) bool {
+		return filePaths[i] < filePaths[j]
+	})
 	var list []DbMigration
 	upRe := regexp.MustCompile(`\.up\.json$`)
 	for _, filePath := range filePaths {
@@ -45,12 +49,12 @@ func GetDbMigrationsFromDir(dirPath string) ([]DbMigration, error) {
 		var upBytes []byte
 		downBytes := []byte("[]")
 		if slices.Contains(filePaths, downFilePath) {
-			downBytes, err = os.ReadFile(downFilePath)
+			downBytes, err = fs.ReadFile(fsys, downFilePath)
 			if err != nil {
 				return nil, err
 			}
 		}
-		upBytes, err = os.ReadFile(filePath)
+		upBytes, err = fs.ReadFile(fsys, filePath)
 		if err != nil {
 			return nil, err
 		}

pkg/dbmigrator/dbmigrator_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"go.mongodb.org/mongo-driver/bson"
 )
@@ -214,3 +215,25 @@ func TestDbMigrator_Apply_Downgrade_On_Apply_Error(t *testing.T) {
 
 	assert.Error(t, err, "test-failed")
 }
+
+func TestGetDbMigrationsFromFs_Empty(t *testing.T) {
+	fsys := &afero.IOFS{Fs: afero.NewMemMapFs()}
+	migrations, err := GetDbMigrationsFromFs(fsys)
+	assert.Equal(t, []DbMigration(nil), migrations)
+	assert.NoError(t, err)
+}
+
+func TestGetDbMigrationsFromFs_Files(t *testing.T) {
+	fsys := &afero.IOFS{Fs: afero.NewMemMapFs()}
+	_ = afero.WriteFile(fsys.Fs, "02_file.up.json", []byte(`[{"a": "2"}]`), 0644)
+	_ = afero.WriteFile(fsys.Fs, "01_file.up.json", []byte(`[{"a": "1"}]`), 0644)
+	_ = afero.WriteFile(fsys.Fs, "02_file.down.json", []byte(`[{"b": "2"}]`), 0644)
+	_ = afero.WriteFile(fsys.Fs, "no_suffix.json", []byte(`[{"b": "2"}]`), 0644)
+	_ = afero.WriteFile(fsys.Fs, "different_ext.up.js", []byte(`[{"b": "2"}]`), 0644)
+	migrations, err := GetDbMigrationsFromFs(fsys)
+	assert.Equal(t, []DbMigration{
+		{Name: "01_file", UpScript: []bson.D{{{Key: "a", Value: "1"}}}, DownScript: []bson.D{}},
+		{Name: "02_file", UpScript: []bson.D{{{Key: "a", Value: "2"}}}, DownScript: []bson.D{{{Key: "b", Value: "2"}}}},
+	}, migrations)
+	assert.NoError(t, err)
+}