parse config on start, remove os.Getenv entries

move folders

cmd/api/main.go

@@ -5,9 +5,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"os/signal"
 	"path"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -19,14 +21,14 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"soldr/pkg/app/api/server"
-	srvevents "soldr/pkg/app/api/server/events"
 	"soldr/pkg/app/api/storage/mem"
 	useraction "soldr/pkg/app/api/user_action"
-	"soldr/pkg/app/api/utils/meter"
 	"soldr/pkg/app/api/worker"
-	"soldr/pkg/logger"
+	"soldr/pkg/app/api/worker/events"
+	"soldr/pkg/log"
 	"soldr/pkg/observability"
 	"soldr/pkg/secret"
+	"soldr/pkg/storage"
 	"soldr/pkg/storage/mysql"
 	"soldr/pkg/system"
 	"soldr/pkg/version"
@@ -35,13 +37,14 @@ import (
 const serviceName = "vxapi"
 
 type Config struct {
-	Debug       bool `config:"debug"`
-	Develop     bool `config:"is_develop"`
-	Log         LogConfig
-	DB          DBConfig
-	Tracing     TracingConfig
-	PublicAPI   PublicAPIConfig
-	EventWorker EventWorkerConfig
+	Debug             bool `config:"debug"`
+	Develop           bool `config:"is_develop"`
+	Log               LogConfig
+	DB                DBConfig
+	Tracing           TracingConfig
+	PublicAPI         PublicAPIConfig
+	EventWorker       EventWorkerConfig
+	ServerEventWorker ServerEventWorkerConfig
 }
 
 type LogConfig struct {
@@ -51,11 +54,12 @@ type LogConfig struct {
 }
 
 type DBConfig struct {
-	User string `config:"db_user,required"`
-	Pass string `config:"db_pass,required"`
-	Name string `config:"db_name,required"`
-	Host string `config:"db_host,required"`
-	Port int    `config:"db_port,required"`
+	User         string `config:"db_user,required"`
+	Pass         string `config:"db_pass,required"`
+	Name         string `config:"db_name,required"`
+	Host         string `config:"db_host,required"`
+	Port         int    `config:"db_port,required"`
+	MigrationDir string `config:"migration_dir"`
 }
 
 type PublicAPIConfig struct {
@@ -65,6 +69,10 @@ type PublicAPIConfig struct {
 	CertFile        string        `config:"api_ssl_crt"`
 	KeyFile         string        `config:"api_ssl_key"`
 	GracefulTimeout time.Duration `config:"public_api_graceful_timeout"`
+	StaticPath      string        `config:"api_static_path"`
+	StaticURL       string        `config:"api_static_url"`
+	TemplatesDir    string        `config:"templates_dir"`
+	CertsPath       string        `config:"certs_path"`
 }
 
 type TracingConfig struct {
@@ -75,6 +83,10 @@ type EventWorkerConfig struct {
 	PollInterval time.Duration `config:"event_worker_poll_interval"`
 }
 
+type ServerEventWorkerConfig struct {
+	KeepDays int `config:"retention_events"`
+}
+
 func defaultConfig() Config {
 	return Config{
 		Log: LogConfig{
@@ -85,14 +97,23 @@ func defaultConfig() Config {
 		Tracing: TracingConfig{
 			Addr: "otel.local:8148",
 		},
+		DB: DBConfig{
+			MigrationDir: "db/api/migrations",
+		},
 		PublicAPI: PublicAPIConfig{
 			Addr:            ":8080",
 			AddrHTTPS:       ":8443",
 			GracefulTimeout: time.Minute,
+			TemplatesDir:    "templates",
+			StaticPath:      "static",
+			CertsPath:       filepath.Join("security", "certs", "api"),
 		},
 		EventWorker: EventWorkerConfig{
 			PollInterval: 30 * time.Second,
 		},
+		ServerEventWorker: ServerEventWorkerConfig{
+			KeepDays: 7,
+		},
 	}
 }
 
@@ -134,21 +155,15 @@ func main() {
 		cfg.Log.Level = "debug"
 		cfg.Log.Format = "text"
 	}
-
-	logDir := cfg.Log.Dir
-	if dir, ok := os.LookupEnv("LOG_DIR"); ok {
-		logDir = dir
-	}
 	logFile := &lumberjack.Logger{
-		Filename:   path.Join(logDir, "api.log"),
+		Filename:   path.Join(cfg.Log.Dir, "api.log"),
 		MaxSize:    10,
 		MaxBackups: 7,
 		MaxAge:     14,
 		Compress:   true,
 	}
-
-	logrus.SetLevel(logger.ParseLevel(cfg.Log.Level))
-	logrus.SetFormatter(logger.ParseFormat(cfg.Log.Format))
+	logrus.SetLevel(log.ParseLevel(cfg.Log.Level))
+	logrus.SetFormatter(log.ParseFormat(cfg.Log.Format))
 	logrus.SetOutput(io.MultiWriter(os.Stdout, logFile))
 
 	dsn := fmt.Sprintf("%s:%s@%s/%s?parseTime=true",
@@ -166,12 +181,7 @@ func main() {
 		logrus.WithError(err).Error("could not connect to database")
 		return
 	}
-
-	migrationDir := "db/api/migrations"
-	if dir, ok := os.LookupEnv("MIGRATION_DIR"); ok {
-		migrationDir = dir
-	}
-	if err = db.Migrate(migrationDir); err != nil {
+	if err = db.Migrate(cfg.DB.MigrationDir); err != nil {
 		logrus.WithError(err).Error("could not apply migrations")
 		return
 	}
@@ -244,7 +254,7 @@ func main() {
 	)
 
 	gormMeter := meterProvider.Meter("vxapi-meter")
-	if err = meter.InitGormMetrics(gormMeter); err != nil {
+	if err = storage.InitGormMetrics(gormMeter); err != nil {
 		logrus.WithError(err).Error("could not initialize vxapi-meter")
 		return
 	}
@@ -254,8 +264,8 @@ func main() {
 	observability.Observer.StartGoRuntimeMetricCollect(serviceName, version.GetBinaryVersion(), attr)
 	defer observability.Observer.Close()
 
-	exchanger := srvevents.NewExchanger()
-	eventWorker := srvevents.NewEventPoller(exchanger, cfg.EventWorker.PollInterval, dbWithORM)
+	exchanger := events.NewExchanger()
+	eventWorker := events.NewEventPoller(exchanger, cfg.EventWorker.PollInterval, dbWithORM)
 	go func() {
 		if err = eventWorker.Run(ctx); err != nil {
 			logrus.WithError(err).Error("could not start event worker")
@@ -269,27 +279,41 @@ func main() {
 	go worker.SyncModulesToPolicies(ctx, dbWithORM)
 
 	// run worker to synchronize events retention policy to all instance DB
-	go worker.SyncRetentionEvents(ctx, dbWithORM)
+	go worker.SyncRetentionEvents(ctx, dbWithORM, cfg.ServerEventWorker.KeepDays)
 
-	userActionWriter := useraction.NewLogWriter()
+	uiStaticURL, err := url.Parse(cfg.PublicAPI.StaticURL)
+	if err != nil {
+		logrus.WithError(err).Error("error on parsing URL to redirect requests to the UI static")
+		return
+	}
+	userActionLogger := useraction.NewLogger()
 
 	router := server.NewRouter(
+		server.RouterConfig{
+			BaseURL:      "/api/v1",
+			Debug:        cfg.Debug,
+			UseSSL:       cfg.PublicAPI.UseSSL,
+			StaticPath:   cfg.PublicAPI.StaticPath,
+			StaticURL:    uiStaticURL,
+			TemplatesDir: cfg.PublicAPI.TemplatesDir,
+			CertsPath:    cfg.PublicAPI.CertsPath,
+		},
 		dbWithORM,
 		exchanger,
-		userActionWriter,
+		userActionLogger,
 		dbConnectionStorage,
 		s3ConnectionStorage,
 	)
 
-	srvg, ctx := errgroup.WithContext(ctx)
-	srvg.Go(func() error {
+	group, ctx := errgroup.WithContext(ctx)
+	group.Go(func() error {
 		return server.Server{
 			Addr:            cfg.PublicAPI.Addr,
 			GracefulTimeout: cfg.PublicAPI.GracefulTimeout,
 		}.ListenAndServe(ctx, router)
 	})
 	if cfg.PublicAPI.UseSSL {
-		srvg.Go(func() error {
+		group.Go(func() error {
 			return server.Server{
 				Addr:            cfg.PublicAPI.AddrHTTPS,
 				CertFile:        cfg.PublicAPI.CertFile,
@@ -298,7 +322,7 @@ func main() {
 			}.ListenAndServeTLS(ctx, router)
 		})
 	}
-	if err = srvg.Wait(); err != nil {
-		logrus.WithError(err).Error("failed to start server")
+	if err = group.Wait(); err != nil {
+		logrus.WithError(err).Error("could not start services")
 	}
 }

cmd/server/main.go

@@ -33,7 +33,6 @@ import (
 	"soldr/pkg/app/server/config"
 	"soldr/pkg/app/server/mmodule"
 	"soldr/pkg/controller"
-	"soldr/pkg/db"
 	"soldr/pkg/observability"
 	"soldr/pkg/storage"
 	"soldr/pkg/system"
@@ -79,7 +78,7 @@ func (s *Server) Init(env svc.Environment) (err error) {
 		if err != nil {
 			return fmt.Errorf("failed to compose a DSN from config: %w", err)
 		}
-		gdb, err := db.New(dsn)
+		gdb, err := storage.New(dsn)
 		if err != nil {
 			return fmt.Errorf("failed to initialize a connection to DB: %w", err)
 		}
@@ -112,7 +111,7 @@ func (s *Server) Init(env svc.Environment) (err error) {
 		}
 		cl, err = controller.NewConfigFromS3(s3ConnParams)
 	case loaderTypeDB:
-		var dsn *db.DSN
+		var dsn *storage.DSN
 		dsn, err = dsnFromConfig(&s.config.DB)
 		if err != nil {
 			err = fmt.Errorf("failed to compose a DSN from config: %w", err)
@@ -186,7 +185,7 @@ func (s *Server) Init(env svc.Environment) (err error) {
 	return
 }
 
-func dsnFromConfig(c *config.DB) (*db.DSN, error) {
+func dsnFromConfig(c *config.DB) (*storage.DSN, error) {
 	if c == nil {
 		return nil, fmt.Errorf("passed config is nil")
 	}
@@ -205,7 +204,7 @@ func dsnFromConfig(c *config.DB) (*db.DSN, error) {
 	if len(c.Name) == 0 {
 		return nil, fmt.Errorf("db name is empty")
 	}
-	return &db.DSN{
+	return &storage.DSN{
 		Host:     c.Host,
 		Port:     c.Port,
 		User:     c.User,
@@ -269,7 +268,7 @@ func initCertProvider(c *config.CertsConfig, s3 storage.IFileReader) (certs.Prov
 	}
 }
 
-func initGorm(dsn *db.DSN, logDir string) (*gorm.DB, error) {
+func initGorm(dsn *storage.DSN, logDir string) (*gorm.DB, error) {
 	addr := fmt.Sprintf(
 		"%s:%s@%s/%s?parseTime=true",
 		dsn.User,

pkg/app/api/logger/logger.go

@@ -0,0 +1,11 @@
+package logger
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+// FromContext is function to get logrus Entry with context
+func FromContext(c *gin.Context) *logrus.Entry {
+	return logrus.WithContext(c.Request.Context())
+}

pkg/app/api/utils/json.go → pkg/app/api/modules/json.go

@@ -1,4 +1,4 @@
-package utils
+package modules
 
 import (
 	"encoding/json"
@@ -8,6 +8,7 @@ import (
 	"github.com/xeipuuv/gojsonreference"
 
 	"soldr/pkg/app/api/models"
+	"soldr/pkg/app/api/utils"
 )
 
 type jsonType int
@@ -299,7 +300,7 @@ func mergeArray(ov1, ov2 reflect.Value, sh models.Schema) reflect.Value {
 func addObjectRequiredKeys(v1, v2, mv reflect.Value, sh models.Schema) reflect.Value {
 	for _, k := range v2.MapKeys() {
 		kv1, kv2 := v1.MapIndex(k), v2.MapIndex(k)
-		if kv2.IsValid() && StringInSlice(k.String(), sh.Required) {
+		if kv2.IsValid() && utils.StringInSlice(k.String(), sh.Required) {
 			// value not found in current
 			if !kv1.IsValid() {
 				mv.SetMapIndex(k, kv2)

pkg/app/api/utils/json_test.go → pkg/app/api/modules/json_test.go

@@ -1,4 +1,4 @@
-package utils
+package modules
 
 import (
 	"encoding/json"

pkg/app/api/modules/modules.go

@@ -168,7 +168,7 @@ func mergeModuleACurrentConfig(cc, dc models.ModuleConfig, sh models.Schema) mod
 	}
 
 	mcsh := copySchema(&sh.Type, sh.Definitions)
-	icc := utils.MergeTwoInterfacesBySchema(cc, dc, mcsh)
+	icc := MergeTwoInterfacesBySchema(cc, dc, mcsh)
 	if res, err := mcsh.ValidateGo(icc); err != nil || !res.Valid() {
 		return dc
 	} else if rcc, ok := icc.(models.ModuleConfig); !ok {
@@ -192,7 +192,7 @@ func mergeModuleASecureCurrentConfig(cc, dc models.ModuleSecureConfig, sh models
 	}
 
 	mcsh := copySchema(&sh.Type, sh.Definitions)
-	icc := utils.MergeTwoInterfacesBySchema(cc, dc, mcsh)
+	icc := MergeTwoInterfacesBySchema(cc, dc, mcsh)
 	if res, err := mcsh.ValidateGo(icc); err != nil || !res.Valid() {
 		return dc
 	} else if rcc, ok := icc.(models.ModuleSecureConfig); !ok {
@@ -226,7 +226,7 @@ func mergeModuleACurrentActionConfig(cac, dac models.ActionConfig, sh models.Sch
 
 	rcac := models.ActionConfig{}
 	acsh := copySchema(&sh.Type, models.GetACSDefinitions(sh.Definitions))
-	icac := utils.MergeTwoInterfacesBySchema(convertToRawInterface(cac), convertToRawInterface(dac), acsh)
+	icac := MergeTwoInterfacesBySchema(convertToRawInterface(cac), convertToRawInterface(dac), acsh)
 	if res, err := acsh.ValidateGo(icac); err != nil || !res.Valid() {
 		return dac
 	} else if b, err := json.Marshal(icac); err != nil {
@@ -245,7 +245,7 @@ func mergeModuleACurrentActionConfig(cac, dac models.ActionConfig, sh models.Sch
 func mergeModuleAEventConfigItem(ceci, deci models.EventConfigItem, sh models.Schema) models.EventConfigItem {
 	reci := models.EventConfigItem{}
 	iceci, ideci := convertToRawInterface(ceci), convertToRawInterface(deci)
-	rieci := utils.MergeTwoInterfacesBySchema(iceci, ideci, sh)
+	rieci := MergeTwoInterfacesBySchema(iceci, ideci, sh)
 	if b, err := json.Marshal(rieci); err != nil {
 		return deci
 	} else if err = json.Unmarshal(b, &reci); err != nil {

pkg/app/api/server/middleware.go

@@ -15,17 +15,16 @@ import (
 
 	"soldr/pkg/app/api/models"
 	"soldr/pkg/app/api/server/context"
-	"soldr/pkg/app/api/server/private"
+	"soldr/pkg/app/api/server/protected"
 	"soldr/pkg/app/api/server/response"
-	"soldr/pkg/app/api/utils"
 	"soldr/pkg/app/api/utils/dbencryptor"
 	obs "soldr/pkg/observability"
 )
 
-func authTokenProtoRequired() gin.HandlerFunc {
+func authTokenProtoRequired(apiBaseURL string) gin.HandlerFunc {
 	privInteractive := "vxapi.modules.interactive"
 	connTypeRegexp := regexp.MustCompile(
-		fmt.Sprintf("%s/vxpws/(aggregate|browser|external)/.*", utils.PrefixPathAPI),
+		fmt.Sprintf("%s/vxpws/(aggregate|browser|external)/.*", apiBaseURL),
 	)
 	return func(c *gin.Context) {
 		if c.IsAborted() {
@@ -90,7 +89,7 @@ func authTokenProtoRequired() gin.HandlerFunc {
 			authFallback("must be used bearer schema")
 			return
 		}
-		claims, err := private.ValidateToken(token)
+		claims, err := protected.ValidateToken(token)
 		if err != nil {
 			authFallback("token invalid")
 			return