From c2001a4a09b621a8df50f4479b3efecd575daf76 Mon Sep 17 00:00:00 2001 From: Mikhail Swift Date: Thu, 21 Sep 2023 17:12:14 -0400 Subject: [PATCH] feat: add support for postgresql This allows users to connect to a postgresql backend for our ent library. The ent client creation was broken into it's own function so we can use the enttest library to pass in the test client for unit testing in the future. --- cmd/archivista/main.go | 9 ++- internal/config/config.go | 1 + internal/metadatastorage/sqlstore/client.go | 70 +++++++++++++++++++ .../mysql.go => sqlstore/store.go} | 34 +-------- 4 files changed, 80 insertions(+), 34 deletions(-) create mode 100644 internal/metadatastorage/sqlstore/client.go rename internal/metadatastorage/{mysqlstore/mysql.go => sqlstore/store.go} (88%) diff --git a/cmd/archivista/main.go b/cmd/archivista/main.go index 0c13cd98..74c8c8ba 100644 --- a/cmd/archivista/main.go +++ b/cmd/archivista/main.go @@ -39,7 +39,7 @@ import ( "github.com/sirupsen/logrus" "github.com/testifysec/archivista" "github.com/testifysec/archivista/internal/config" - "github.com/testifysec/archivista/internal/metadatastorage/mysqlstore" + "github.com/testifysec/archivista/internal/metadatastorage/sqlstore" "github.com/testifysec/archivista/internal/objectstorage/blobstore" "github.com/testifysec/archivista/internal/objectstorage/filestore" "github.com/testifysec/archivista/internal/server" @@ -86,7 +86,12 @@ func main() { logrus.Fatalf("error initializing storage clients: %+v", err) } - mysqlStore, mysqlStoreCh, err := mysqlstore.New(ctx, cfg.SQLStoreConnectionString) + entClient, err := sqlstore.NewEntClient(cfg.SQLBackend, cfg.SQLStoreConnectionString) + if err != nil { + logrus.Fatalf("could not create ent client: %+v", err) + } + + mysqlStore, mysqlStoreCh, err := sqlstore.New(ctx, entClient) if err != nil { logrus.Fatalf("error initializing mysql client: %+v", err) } diff --git a/internal/config/config.go b/internal/config/config.go index 7e0763d9..a3da763b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,6 +32,7 @@ type Config struct { SPIFFEAddress string `default:"unix:///tmp/spire-agent/public/api.sock" desc:"SPIFFE server address" split_words:"true"` SPIFFETrustedServerId string `default:"" desc:"Trusted SPIFFE server ID; defaults to any" split_words:"true"` SQLStoreConnectionString string `default:"root:example@tcp(db)/testify" desc:"SQL store connection string" split_words:"true"` + SQLBackend string `default:"MYSQL" desc:"SQL backend to use. Options are MYSQL, PSQL" split_words:"true"` StorageBackend string `default:"" desc:"Backend to use for attestation storage. Options are FILE, BLOB, or empty string for disabled." split_words:"true"` FileServeOn string `default:"" desc:"What address to serve files on. Only valid when using FILE storage backend." split_words:"true"` diff --git a/internal/metadatastorage/sqlstore/client.go b/internal/metadatastorage/sqlstore/client.go new file mode 100644 index 00000000..e5524258 --- /dev/null +++ b/internal/metadatastorage/sqlstore/client.go @@ -0,0 +1,70 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlstore + +import ( + "fmt" + "strings" + "time" + + "ariga.io/sqlcomment" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql" + "github.com/go-sql-driver/mysql" + "github.com/testifysec/archivista/ent" +) + +// NewEntClient creates an ent client for use in the sqlmetadata store. +// Valid backends are MYSQL and PSQL. +func NewEntClient(sqlBackend string, connectionString string) (*ent.Client, error) { + var entDialect string + switch strings.ToUpper(sqlBackend) { + case "MYSQL": + dbConfig, err := mysql.ParseDSN(connectionString) + if err != nil { + return nil, fmt.Errorf("could not parse mysql connection string: %w", err) + } + + // this tells the go-sql-driver to parse times from mysql to go's time.Time + // see https://github.com/go-sql-driver/mysql#timetime-support for details + dbConfig.ParseTime = true + entDialect = dialect.MySQL + connectionString = dbConfig.FormatDSN() + case "PSQL": + entDialect = dialect.Postgres + default: + return nil, fmt.Errorf("unknown sql backend: %s", sqlBackend) + } + + drv, err := sql.Open(entDialect, connectionString) + if err != nil { + return nil, fmt.Errorf("could not open sql connection: %w", err) + } + + db := drv.DB() + db.SetMaxIdleConns(10) + db.SetMaxOpenConns(100) + db.SetConnMaxLifetime(3 * time.Minute) + sqlcommentDrv := sqlcomment.NewDriver(drv, + sqlcomment.WithDriverVerTag(), + sqlcomment.WithTags(sqlcomment.Tags{ + sqlcomment.KeyApplication: "archivista", + sqlcomment.KeyFramework: "net/http", + }), + ) + + client := ent.NewClient(ent.Driver(sqlcommentDrv)) + return client, nil +} diff --git a/internal/metadatastorage/mysqlstore/mysql.go b/internal/metadatastorage/sqlstore/store.go similarity index 88% rename from internal/metadatastorage/mysqlstore/mysql.go rename to internal/metadatastorage/sqlstore/store.go index bb52cb8e..3fc0d34c 100644 --- a/internal/metadatastorage/mysqlstore/mysql.go +++ b/internal/metadatastorage/sqlstore/store.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mysqlstore +package sqlstore import ( "context" @@ -22,10 +22,7 @@ import ( "fmt" "time" - "ariga.io/sqlcomment" - "entgo.io/ent/dialect/sql" "github.com/digitorus/timestamp" - "github.com/go-sql-driver/mysql" "github.com/sirupsen/logrus" "github.com/testifysec/archivista/ent" "github.com/testifysec/archivista/internal/metadatastorage" @@ -47,34 +44,7 @@ type Store struct { client *ent.Client } -func New(ctx context.Context, connectionstring string) (*Store, <-chan error, error) { - dbConfig, err := mysql.ParseDSN(connectionstring) - if err != nil { - return nil, nil, err - } - - dbConfig.ParseTime = true - connectionstring = dbConfig.FormatDSN() - drv, err := sql.Open("mysql", connectionstring) - if err != nil { - return nil, nil, err - } - sqlcommentDrv := sqlcomment.NewDriver(drv, - sqlcomment.WithDriverVerTag(), - sqlcomment.WithTags(sqlcomment.Tags{ - sqlcomment.KeyApplication: "archivista", - sqlcomment.KeyFramework: "net/http", - }), - ) - - // TODO make sure these take affect in sqlcommentDrv - db := drv.DB() - db.SetMaxIdleConns(10) - db.SetMaxOpenConns(100) - db.SetConnMaxLifetime(3 * time.Minute) - - client := ent.NewClient(ent.Driver(sqlcommentDrv)) - +func New(ctx context.Context, client *ent.Client) (*Store, <-chan error, error) { errCh := make(chan error) go func() {