Add ability to use localstack for storage (#64)

* Add ability to use localstack for storage

* Fix storage tests

* Update pr.yml

.github/workflows/pr.yml

@@ -13,12 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          stable: 'true'
           go-version: 1.21
 
       - name: Test Code

cmd/allInOne.go

@@ -45,39 +45,39 @@ var allInOneCmd = &cobra.Command{
 	Long:  `This runs all the micro-services as part of a single process, useful for developing and for trying out Terrarium.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		dependencyServiceServer := &dependency_manager.DependencyManagerService{
-			Db:              storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+			Db:              storage.NewDynamoDbClient(awsSessionConfig),
 			ModuleTable:     dependency_manager.ModuleDependenciesTableName,
 			ModuleSchema:    dependency_manager.GetDependenciesSchema(dependency_manager.ModuleDependenciesTableName),
 			ContainerTable:  dependency_manager.ContainerDependenciesTableName,
 			ContainerSchema: dependency_manager.GetDependenciesSchema(dependency_manager.ContainerDependenciesTableName),
 		}
 
 		registrarServiceServer := &registrar.RegistrarService{
-			Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+			Db:     storage.NewDynamoDbClient(awsSessionConfig),
 			Table:  registrar.RegistrarTableName,
 			Schema: registrar.GetModulesSchema(registrar.RegistrarTableName),
 		}
 
 		storageServiceServer := &storage2.StorageService{
-			Client:     storage.NewS3Client(awsAccessKey, awsSecretKey, awsRegion),
+			Client:     storage.NewS3Client(awsSessionConfig),
 			BucketName: storage2.BucketName,
-			Region:     awsRegion,
+			Region:     awsSessionConfig.Region,
 		}
 
 		tagManagerServer := &tag_manager.TagManagerService{
-			Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+			Db:     storage.NewDynamoDbClient(awsSessionConfig),
 			Table:  tag_manager.TagTableName,
 			Schema: tag_manager.GetTagsSchema(tag_manager.TagTableName),
 		}
 
 		releaseServiceServer := &release.ReleaseService{
-			Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+			Db:     storage.NewDynamoDbClient(awsSessionConfig),
 			Table:  release.ReleaseTableName,
 			Schema: release.GetReleaseSchema(release.ReleaseTableName),
 		}
 
 		versionManagerServer := &version_manager.VersionManagerService{
-			Db:             storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+			Db:             storage.NewDynamoDbClient(awsSessionConfig),
 			Table:          version_manager.VersionsTableName,
 			Schema:         version_manager.GetModuleVersionsSchema(version_manager.VersionsTableName),
 			ReleaseService: release.NewPublisherGrpcClient(allInOneInternalEndpoint),

cmd/dependency_manager.go

@@ -23,7 +23,7 @@ func init() {
 func runDependencyManager(cmd *cobra.Command, args []string) {
 
 	dependencyServiceServer := &dependency_manager.DependencyManagerService{
-		Db:              storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+		Db:              storage.NewDynamoDbClient(awsSessionConfig),
 		ModuleTable:     dependency_manager.ModuleDependenciesTableName,
 		ModuleSchema:    dependency_manager.GetDependenciesSchema(dependency_manager.ModuleDependenciesTableName),
 		ContainerTable:  dependency_manager.ContainerDependenciesTableName,

cmd/registrar.go

@@ -22,7 +22,7 @@ func init() {
 func runRegistrarService(cmd *cobra.Command, args []string) {
 
 	registrarServiceServer := &registrar.RegistrarService{
-		Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+		Db:     storage.NewDynamoDbClient(awsSessionConfig),
 		Table:  registrar.RegistrarTableName,
 		Schema: registrar.GetModulesSchema(registrar.RegistrarTableName),
 	}

cmd/release.go

@@ -22,7 +22,7 @@ func init() {
 func runReleaseService(cmd *cobra.Command, args []string) {
 
 	releaseServiceServer := &release.ReleaseService{
-		Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+		Db:     storage.NewDynamoDbClient(awsSessionConfig),
 		Table:  release.ReleaseTableName,
 		Schema: release.GetReleaseSchema(release.ReleaseTableName),
 	}

cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/terrariumcloud/terrarium/internal/storage"
 	"log"
 	"net"
 	"net/http"
@@ -35,26 +36,23 @@ const (
 	defaultEndpoint = "0.0.0.0:3001"
 )
 
-var endpoint = defaultEndpoint
-var awsAccessKey string
-var awsSecretKey string
-var awsRegion string
-var opentelemetryInited = false
-
-var rootCmd = &cobra.Command{
-	Use:   "terrarium",
-	Short: "Terrarium Services",
-	Long:  "Runs backend that exposes Terrarium Services",
-}
+var (
+	endpoint            = defaultEndpoint
+	awsSessionConfig    = storage.AWSSessionConfig{}
+	opentelemetryInited = false
+	rootCmd             = &cobra.Command{
+		Use:   "terrarium",
+		Short: "Terrarium Services",
+		Long:  "Runs backend that exposes Terrarium Services",
+	}
+)
 
 func init() {
 	rootCmd.PersistentFlags().StringVarP(&endpoint, "endpoint", "e", defaultEndpoint, "Endpoint")
-	rootCmd.PersistentFlags().StringVarP(&awsAccessKey, "aws-access-key-id", "k", "", "AWS Access Key (required)")
-	rootCmd.MarkPersistentFlagRequired("aws-access-key-id")
-	rootCmd.PersistentFlags().StringVarP(&awsSecretKey, "aws-secret-access-key", "s", "", "AWS Secret Key (required)")
-	rootCmd.MarkPersistentFlagRequired("aws-secret-access-key")
-	rootCmd.PersistentFlags().StringVarP(&awsRegion, "aws-region", "r", "", "AWS Region (required)")
-	rootCmd.MarkPersistentFlagRequired("aws-region")
+	rootCmd.PersistentFlags().StringVarP(&awsSessionConfig.Key, "aws-access-key-id", "k", "", "AWS Access Key")
+	rootCmd.PersistentFlags().StringVarP(&awsSessionConfig.Secret, "aws-secret-access-key", "s", "", "AWS Secret Key")
+	rootCmd.PersistentFlags().StringVarP(&awsSessionConfig.Region, "aws-region", "r", "", "AWS Region")
+	rootCmd.PersistentFlags().BoolVar(&awsSessionConfig.UseLocalStack, "use-localstack", false, "Connect to a localstack instance rather than AWS.")
 }
 
 func newTraceExporter(ctx context.Context) (*otlptrace.Exporter, error) {

cmd/storage.go

@@ -22,9 +22,9 @@ func init() {
 func runStorageService(cmd *cobra.Command, args []string) {
 
 	storageServiceServer := &storage2.StorageService{
-		Client:     storage.NewS3Client(awsAccessKey, awsSecretKey, awsRegion),
+		Client:     storage.NewS3Client(awsSessionConfig),
 		BucketName: storage2.BucketName,
-		Region:     awsRegion,
+		Region:     awsSessionConfig.Region,
 	}
 
 	startGRPCService("storage-s3", storageServiceServer)

cmd/tag_manager.go

@@ -22,7 +22,7 @@ func init() {
 func runTagManager(cmd *cobra.Command, args []string) {
 
 	tagManagerServer := &tag_manager.TagManagerService{
-		Db:     storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+		Db:     storage.NewDynamoDbClient(awsSessionConfig),
 		Table:  tag_manager.TagTableName,
 		Schema: tag_manager.GetTagsSchema(tag_manager.TagTableName),
 	}

cmd/version_manager.go

@@ -23,7 +23,7 @@ func init() {
 func runVersionManager(cmd *cobra.Command, args []string) {
 
 	versionManagerServer := &version_manager.VersionManagerService{
-		Db:             storage.NewDynamoDbClient(awsAccessKey, awsSecretKey, awsRegion),
+		Db:             storage.NewDynamoDbClient(awsSessionConfig),
 		Table:          version_manager.VersionsTableName,
 		Schema:         version_manager.GetModuleVersionsSchema(version_manager.VersionsTableName),
 		ReleaseService: release.NewPublisherGrpcClient(release.ReleaseServiceEndpoint),

internal/storage/aws_session.go

@@ -6,13 +6,43 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws"
+	"os"
 )
 
+type AWSSessionConfig struct {
+	UseLocalStack bool
+	Region        string
+	Key           string
+	Secret        string
+}
+
 // Create new AWS Session with provided API key, Secret and Region
-func NewAwsSession(key string, secret string, region string) (*aws.Config, error) {
+func NewAwsSession(sessionConfig AWSSessionConfig) (*aws.Config, error) {
+	awsRegion := sessionConfig.Region
+	if envRegion := os.Getenv("AWS_REGION"); envRegion != "" {
+		awsRegion = envRegion
+	}
+
+	if sessionConfig.UseLocalStack {
+		sessionConfig.Key = "test"
+		sessionConfig.Secret = "test"
+	}
+
+	customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		if sessionConfig.UseLocalStack {
+			return aws.Endpoint{
+				PartitionID:   "aws",
+				URL:           "http://localhost:4566",
+				SigningRegion: awsRegion,
+			}, nil
+		}
+		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+	})
+
 	cfg, err := config.LoadDefaultConfig(context.TODO(),
-		config.WithRegion(region),
-		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(key, secret, "")))
+		config.WithRegion(awsRegion),
+		config.WithEndpointResolverWithOptions(customResolver),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(sessionConfig.Key, sessionConfig.Secret, "")))
 	if err != nil {
 		return nil, err
 	}

internal/storage/aws_session_test.go

@@ -1,8 +1,7 @@
-package storage_test
+package storage
 
 import (
 	"context"
-	"github.com/terrariumcloud/terrarium/internal/storage"
 	"testing"
 )
 
@@ -11,27 +10,57 @@ import (
 func Test_NewAwsSession(t *testing.T) {
 
 	t.Run("returns AWS session`", func(t *testing.T) {
-		var key, secret, region string
-		key = "test_key"
-		secret = "test_secret"
-		region = "eu-west-1"
+		sessionConfig := AWSSessionConfig{
+			UseLocalStack: false,
+			Region:        "eu-west-1",
+			Key:           "test_key",
+			Secret:        "test_secret",
+		}
+
+		cfg, err := NewAwsSession(sessionConfig)
+
+		if err != nil {
+			t.Errorf("Expected no error, got %v.", err)
+		}
+
+		if cfg.Region != sessionConfig.Region {
+			t.Errorf("Expected %v, got %v.", sessionConfig.Region, cfg.Region)
+		}
+		creds, _ := cfg.Credentials.Retrieve(context.TODO())
+
+		if creds.AccessKeyID != sessionConfig.Key {
+			t.Errorf("Expected %v, got %v.", sessionConfig.Key, creds.AccessKeyID)
+
+			if creds.SecretAccessKey != sessionConfig.Secret {
+				t.Errorf("Expected %v, got %v.", sessionConfig.Secret, creds.SecretAccessKey)
+			}
+		}
+	})
+
+	t.Run("use localstack", func(t *testing.T) {
+		sessionConfig := AWSSessionConfig{
+			UseLocalStack: true,
+			Region:        "eu-west-1",
+			Key:           "super_secret_key",
+			Secret:        "super_secret_secret_key",
+		}
 
-		cfg, err := storage.NewAwsSession(key, secret, region)
+		cfg, err := NewAwsSession(sessionConfig)
 
 		if err != nil {
 			t.Errorf("Expected no error, got %v.", err)
 		}
 
-		if cfg.Region != region {
-			t.Errorf("Expected %v, got %v.", region, cfg.Region)
+		if cfg.Region != sessionConfig.Region {
+			t.Errorf("Expected %v, got %v.", sessionConfig.Region, cfg.Region)
 		}
 		creds, _ := cfg.Credentials.Retrieve(context.TODO())
 
-		if creds.AccessKeyID != key {
-			t.Errorf("Expected %v, got %v.", key, creds.AccessKeyID)
+		if creds.AccessKeyID != "test" {
+			t.Errorf("Expected %v, got %v.", sessionConfig.Key, creds.AccessKeyID)
 
-			if creds.SecretAccessKey != secret {
-				t.Errorf("Expected %v, got %v.", secret, creds.SecretAccessKey)
+			if creds.SecretAccessKey != "test" {
+				t.Errorf("Expected %v, got %v.", sessionConfig.Secret, creds.SecretAccessKey)
 			}
 		}
 	})

internal/storage/dynamodb.go

@@ -20,8 +20,8 @@ type DynamoDBTableCreator interface {
 }
 
 // Create new DynamoDB client
-func NewDynamoDbClient(key string, secret string, region string) *dynamodb.Client {
-	cfg, err := NewAwsSession(key, secret, region)
+func NewDynamoDbClient(sessionConfig AWSSessionConfig) *dynamodb.Client {
+	cfg, err := NewAwsSession(sessionConfig)
 	if err != nil {
 		log.Fatalf("Unable to create AWS Session: %s", err.Error())
 	}

internal/storage/dynamodb_test.go

@@ -1,10 +1,9 @@
-package storage_test
+package storage
 
 import (
 	"errors"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
-	"github.com/terrariumcloud/terrarium/internal/storage"
 	"github.com/terrariumcloud/terrarium/internal/storage/mocks"
 	"testing"
 )
@@ -22,7 +21,7 @@ func Test_InitializeDynamoDb(t *testing.T) {
 		schema := &dynamodb.CreateTableInput{}
 		db := &mocks.DynamoDB{}
 
-		err := storage.InitializeDynamoDb(table, schema, db)
+		err := InitializeDynamoDb(table, schema, db)
 
 		if db.DescribeTableInvocations != 1 {
 			t.Errorf("Expected 1 call to DescribeTable, got %v.", db.DescribeTableInvocations)
@@ -44,7 +43,7 @@ func Test_InitializeDynamoDb(t *testing.T) {
 			DescribeTableErrors: []error{&types.TableNotFoundException{}},
 		}
 
-		err := storage.InitializeDynamoDb(table, schema, db)
+		err := InitializeDynamoDb(table, schema, db)
 
 		if db.DescribeTableInvocations != 1 {
 			t.Errorf("Expected 1 call to DescribeTable, got %v.", db.DescribeTableInvocations)
@@ -74,7 +73,7 @@ func Test_InitializeDynamoDb(t *testing.T) {
 			DescribeTableErrors: []error{errors.New("some error")},
 		}
 
-		err := storage.InitializeDynamoDb(table, schema, db)
+		err := InitializeDynamoDb(table, schema, db)
 
 		if db.DescribeTableInvocations != 1 {
 			t.Errorf("Expected 1 call to DescribeTable, got %v.", db.DescribeTableInvocations)
@@ -98,7 +97,7 @@ func Test_InitializeDynamoDb(t *testing.T) {
 			CreateTableError:    someError,
 		}
 
-		err := storage.InitializeDynamoDb(table, schema, db)
+		err := InitializeDynamoDb(table, schema, db)
 
 		if db.DescribeTableInvocations != 1 {
 			t.Errorf("Expected 1 call to DescribeTable, got %v.", db.DescribeTableInvocations)

internal/storage/s3.go

@@ -17,12 +17,14 @@ type AWSS3BucketClient interface {
 }
 
 // NewS3Client Create new S3 client
-func NewS3Client(key string, secret string, region string) *s3.Client {
-	cfg, err := NewAwsSession(key, secret, region)
+func NewS3Client(sessionConfig AWSSessionConfig) *s3.Client {
+	cfg, err := NewAwsSession(sessionConfig)
 	if err != nil {
 		log.Fatalf("Unable to create AWS Session: %s", err.Error())
 	}
-	return s3.NewFromConfig(*cfg)
+	return s3.NewFromConfig(*cfg, func(options *s3.Options) {
+		options.UsePathStyle = sessionConfig.UseLocalStack
+	})
 }
 
 // InitializeS3Bucket - checks if bucket exists, in case it doesn't it creates it