Skip to content

Commit

Permalink
[Terrarium - Providers]: Implementation of Storage Feature - 2 (#71)
Browse files Browse the repository at this point in the history
* implementation of consumer service

* update in functions as we remove consumer service

* go fmt

* Update internal/provider/services/storage/storage.go

Co-authored-by: Adam Charrett <[email protected]>

* updated storage.go

* enhance error logs

* updated storage_test

* fix

* fix

* updated storage_test and mock server

* [Terrarium - Providers]: Implementation of Storage Feature - 3 (#72)

---------

Co-authored-by: Anjani Kumar Srivastava <[email protected]>
Co-authored-by: Adam Charrett <[email protected]>
  • Loading branch information
3 people authored Apr 30, 2024
1 parent e37b1b1 commit bb6b5b7
Show file tree
Hide file tree
Showing 14 changed files with 1,205 additions and 11 deletions.
12 changes: 11 additions & 1 deletion cmd/allInOne.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
storage2 "github.com/terrariumcloud/terrarium/internal/module/services/storage"
"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
"github.com/terrariumcloud/terrarium/internal/release/services/release"
"github.com/terrariumcloud/terrarium/internal/restapi/browse"
Expand Down Expand Up @@ -92,6 +93,12 @@ var allInOneCmd = &cobra.Command{
Schema: providerVersionManager.GetProviderVersionsSchema(providerVersionManager.VersionsTableName),
}

providerStorageServiceServer := &providerStorage.StorageService{
Client: storage.NewS3Client(awsSessionConfig),
BucketName: providerStorage.BucketName,
Region: awsSessionConfig.Region,
}

services := []grpcServices.Service{
dependencyServiceServer,
registrarServiceServer,
Expand All @@ -100,6 +107,7 @@ var allInOneCmd = &cobra.Command{
releaseServiceServer,
versionManagerServer,
providerVersionManagerServer,
providerStorageServiceServer,
}

otelShutdown := initOpenTelemetry("all-in-one")
Expand All @@ -114,6 +122,7 @@ var allInOneCmd = &cobra.Command{
dependency_manager.NewDependencyManagerGrpcClient(allInOneInternalEndpoint),
release.NewPublisherGrpcClient(allInOneInternalEndpoint),
providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint),
providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint),
)

startAllInOneGrpcServices([]grpcServices.Service{gatewayServer}, allInOneGrpcGatewayEndpoint)
Expand All @@ -124,7 +133,7 @@ var allInOneCmd = &cobra.Command{
providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))

modulesAPIServer := modulesv1.New(version_manager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), storage2.NewStorageGrpcClient(allInOneInternalEndpoint))
providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))
providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint))

router := mux.NewRouter()
router.PathPrefix("/modules").Handler(modulesAPIServer.GetHttpHandler("/modules"))
Expand All @@ -146,6 +155,7 @@ func init() {
allInOneCmd.Flags().StringVar(&dependency_manager.ModuleDependenciesTableName, "module-dependencies-table", dependency_manager.DefaultModuleDependenciesTableName, "Module dependencies table name")
allInOneCmd.Flags().StringVar(&dependency_manager.ContainerDependenciesTableName, "container-dependencies-table", dependency_manager.DefaultContainerDependenciesTableName, "Module container dependencies table name")
allInOneCmd.Flags().StringVar(&providerVersionManager.VersionsTableName, "provider-table", providerVersionManager.DefaultProviderVersionsTableName, "Provider versions table name")
allInOneCmd.Flags().StringVar(&providerStorage.BucketName, "provider-storage-bucket", providerStorage.DefaultBucketName, "Provider bucket name")
}

func startAllInOneGrpcServices(services []grpcServices.Service, endpoint string) {
Expand Down
5 changes: 4 additions & 1 deletion cmd/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/terrariumcloud/terrarium/internal/module/services/storage"
"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
"github.com/terrariumcloud/terrarium/internal/release/services/release"

Expand All @@ -25,10 +26,11 @@ func init() {
gatewayCmd.Flags().StringVarP(&registrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Registrar Service")
gatewayCmd.Flags().StringVarP(&dependency_manager.DependencyManagerEndpoint, "dependency-manager", "", dependency_manager.DefaultDependencyManagerEndpoint, "GRPC Endpoint for Dependency Manager Service")
gatewayCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Module Version Manager Service")
gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Storage Service")
gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Module Storage Service")
gatewayCmd.Flags().StringVarP(&tag_manager.TagManagerEndpoint, "tag-manager", "", tag_manager.DefaultTagManagerEndpoint, "GRPC Endpoint for Tag Service")
gatewayCmd.Flags().StringVarP(&release.ReleaseServiceEndpoint, "release", "", release.DefaultReleaseServiceEndpoint, "GRPC Endpoint for Release Service")
gatewayCmd.Flags().StringVarP(&providerVersionManager.VersionManagerEndpoint, "provider-version-manager", "", providerVersionManager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Provider Version Manager Service")
gatewayCmd.Flags().StringVarP(&providerStorage.StorageServiceEndpoint, "provider-storage", "", providerStorage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service")
}

func runGateway(cmd *cobra.Command, args []string) {
Expand All @@ -40,6 +42,7 @@ func runGateway(cmd *cobra.Command, args []string) {
dependency_manager.NewDependencyManagerGrpcClient(dependency_manager.DependencyManagerEndpoint),
release.NewPublisherGrpcClient(release.ReleaseServiceEndpoint),
providerVersionManager.NewVersionManagerGrpcClient(providerVersionManager.VersionManagerEndpoint),
providerStorage.NewStorageGrpcClient(providerStorage.StorageServiceEndpoint),
)

startGRPCService("api-gateway", gatewayServer)
Expand Down
31 changes: 31 additions & 0 deletions cmd/provider_storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package cmd

import (
providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
"github.com/terrariumcloud/terrarium/internal/storage"

"github.com/spf13/cobra"
)

var providerStorageServiceCmd = &cobra.Command{
Use: "provider-storage",
Short: "Starts the Terrarium GRPC Provider Storage service",
Long: "Runs the Terrarium GRPC Provider Storage server.",
Run: runProviderStorageService,
}

func init() {
rootCmd.AddCommand(providerStorageServiceCmd)
providerStorageServiceCmd.Flags().StringVarP(&providerStorage.BucketName, "bucket", "b", providerStorage.DefaultBucketName, "Provider bucket name")
}

func runProviderStorageService(cmd *cobra.Command, args []string) {

storageServiceServer := &providerStorage.StorageService{
Client: storage.NewS3Client(awsSessionConfig),
BucketName: providerStorage.BucketName,
Region: awsSessionConfig.Region,
}

startGRPCService("provider-storage-s3", storageServiceServer)
}
8 changes: 7 additions & 1 deletion cmd/rest_providers_v1.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"github.com/terrariumcloud/terrarium/internal/provider/services/storage"
"github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
providersv1 "github.com/terrariumcloud/terrarium/internal/restapi/providers/v1"

Expand All @@ -25,10 +26,15 @@ func init() {
"Mount path for the rest API server used to process request relative to a particular URL in a reverse proxy type setup",
)
providersV1Cmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "provider-version-manager", "", version_manager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service")
providersV1Cmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "provider-storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service")

rootCmd.AddCommand(providersV1Cmd)
}

func runRESTProvidersV1Server(cmd *cobra.Command, args []string) {
restAPIServer := providersv1.New(version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint))
restAPIServer := providersv1.New(
version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint),
storage.NewStorageGrpcClient(storage.StorageServiceEndpoint),
)
startRESTAPIService("rest-providers-v1", mountPathProviders, restAPIServer)
}
21 changes: 21 additions & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,27 @@ services:
- "$AWS_SECRET_ACCESS_KEY"
- "--aws-region"
- "$AWS_DEFAULT_REGION"
provider-storage:
build: .
image: terrarium:dev
container_name: terrarium-provider-storage-service
environment:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- AWS_DEFAULT_REGION
- OTEL_EXPORTER_OTLP_ENDPOINT=jaeger:4317
ports:
- 50010:3001
networks:
- terrarium
command:
- provider-storage
- "--aws-access-key-id"
- "$AWS_ACCESS_KEY_ID"
- "--aws-secret-access-key"
- "$AWS_SECRET_ACCESS_KEY"
- "--aws-region"
- "$AWS_DEFAULT_REGION"
release:
build: .
image: terrarium:dev
Expand Down
5 changes: 4 additions & 1 deletion internal/common/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type TerrariumGrpcGateway struct {
storageClient moduleServices.StorageClient
dependencyManagerClient moduleServices.DependencyManagerClient
releasePublisherClient release.PublisherClient
providerStorageClient providerServices.StorageClient
}

func New(registrarClient moduleServices.RegistrarClient,
Expand All @@ -52,7 +53,8 @@ func New(registrarClient moduleServices.RegistrarClient,
storageClient moduleServices.StorageClient,
dependencyManagerClient moduleServices.DependencyManagerClient,
releasePublisherClient release.PublisherClient,
providerVersionManagerClient providerServices.VersionManagerClient) *TerrariumGrpcGateway {
providerVersionManagerClient providerServices.VersionManagerClient,
providerStorageClient providerServices.StorageClient) *TerrariumGrpcGateway {
return &TerrariumGrpcGateway{
registrarClient: registrarClient,
tagManagerClient: tagManagerClient,
Expand All @@ -61,6 +63,7 @@ func New(registrarClient moduleServices.RegistrarClient,
dependencyManagerClient: dependencyManagerClient,
releasePublisherClient: releasePublisherClient,
providerVersionManagerClient: providerVersionManagerClient,
providerStorageClient: providerStorageClient,
}
}

Expand Down
94 changes: 94 additions & 0 deletions internal/provider/services/mocks/mock_clients.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package mocks

import (
"context"

providerServices "github.com/terrariumcloud/terrarium/internal/provider/services"

"google.golang.org/grpc"
)

type MockProviderStorageClient struct {
providerServices.StorageClient
DownloadSourceZipInvocations int
DownloadSourceZipClient providerServices.Storage_DownloadProviderSourceZipClient
DownloadSourceZipError error
DownloadShasumInvocations int
DownloadShasumClient providerServices.Storage_DownloadShasumClient
DownloadShasumError error
DownloadShasumSignatureInvocations int
DownloadShasumSignatureClient providerServices.Storage_DownloadShasumSignatureClient
DownloadShasumSignatureError error
}

func (m *MockProviderStorageClient) DownloadProviderSourceZip(ctx context.Context, in *providerServices.DownloadSourceZipRequest, opts ...grpc.CallOption) (providerServices.Storage_DownloadProviderSourceZipClient, error) {
m.DownloadSourceZipInvocations++
return m.DownloadSourceZipClient, m.DownloadSourceZipError
}

func (m *MockProviderStorageClient) DownloadShasum(ctx context.Context, in *providerServices.DownloadShasumRequest, opts ...grpc.CallOption) (providerServices.Storage_DownloadShasumClient, error) {
m.DownloadShasumInvocations++
return m.DownloadShasumClient, m.DownloadShasumError
}

func (m *MockProviderStorageClient) DownloadShasumSignature(ctx context.Context, in *providerServices.DownloadShasumRequest, opts ...grpc.CallOption) (providerServices.Storage_DownloadShasumSignatureClient, error) {
m.DownloadShasumSignatureInvocations++
return m.DownloadShasumSignatureClient, m.DownloadShasumSignatureError
}

type MockStorage_DownloadProviderSourceZipClient struct {
providerServices.Storage_DownloadProviderSourceZipClient
RecvInvocations int
RecvResponse *providerServices.SourceZipResponse
RecvError error
CloseSendInvocations int
CloseSendError error
}

func (m *MockStorage_DownloadProviderSourceZipClient) Recv() (*providerServices.SourceZipResponse, error) {
m.RecvInvocations++
return m.RecvResponse, m.RecvError
}

func (m *MockStorage_DownloadProviderSourceZipClient) CloseSend() error {
m.CloseSendInvocations++
return m.CloseSendError
}

type MockStorage_DownloadProviderShasumClient struct {
providerServices.Storage_DownloadShasumClient
RecvInvocations int
RecvResponse *providerServices.DownloadShasumResponse
RecvError error
CloseSendInvocations int
CloseSendError error
}

func (m *MockStorage_DownloadProviderShasumClient) Recv() (*providerServices.DownloadShasumResponse, error) {
m.RecvInvocations++
return m.RecvResponse, m.RecvError
}

func (m *MockStorage_DownloadProviderShasumClient) CloseSend() error {
m.CloseSendInvocations++
return m.CloseSendError
}

type MockStorage_DownloadProviderShasumSignatureClient struct {
providerServices.Storage_DownloadShasumSignatureClient
RecvInvocations int
RecvResponse *providerServices.DownloadShasumResponse
RecvError error
CloseSendInvocations int
CloseSendError error
}

func (m *MockStorage_DownloadProviderShasumSignatureClient) Recv() (*providerServices.DownloadShasumResponse, error) {
m.RecvInvocations++
return m.RecvResponse, m.RecvError
}

func (m *MockStorage_DownloadProviderShasumSignatureClient) CloseSend() error {
m.CloseSendInvocations++
return m.CloseSendError
}
64 changes: 64 additions & 0 deletions internal/provider/services/mocks/mock_servers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package mocks

import (
"context"

providerServices "github.com/terrariumcloud/terrarium/internal/provider/services"
)

type MockDownloadProviderSourceZipServer struct {
providerServices.Storage_DownloadProviderSourceZipServer
SendInvocations int
SendResponse *providerServices.SourceZipResponse
SendError error
TotalReceived []byte
}

func (mds *MockDownloadProviderSourceZipServer) Context() context.Context {
return context.TODO()
}

func (mds *MockDownloadProviderSourceZipServer) Send(res *providerServices.SourceZipResponse) error {
mds.SendInvocations++
mds.SendResponse = res
mds.TotalReceived = append(mds.TotalReceived, mds.SendResponse.ZipDataChunk...)
return mds.SendError
}

type MockDownloadProviderShasumServer struct {
providerServices.Storage_DownloadShasumServer
SendInvocations int
SendResponse *providerServices.DownloadShasumResponse
SendError error
TotalReceived []byte
}

func (mds *MockDownloadProviderShasumServer) Context() context.Context {
return context.TODO()
}

func (mds *MockDownloadProviderShasumServer) Send(res *providerServices.DownloadShasumResponse) error {
mds.SendInvocations++
mds.SendResponse = res
mds.TotalReceived = append(mds.TotalReceived, mds.SendResponse.ShasumDataChunk...)
return mds.SendError
}

type MockDownloadProviderShasumSignatureServer struct {
providerServices.Storage_DownloadShasumSignatureServer
SendInvocations int
SendResponse *providerServices.DownloadShasumResponse
SendError error
TotalReceived []byte
}

func (mds *MockDownloadProviderShasumSignatureServer) Context() context.Context {
return context.TODO()
}

func (mds *MockDownloadProviderShasumSignatureServer) Send(res *providerServices.DownloadShasumResponse) error {
mds.SendInvocations++
mds.SendResponse = res
mds.TotalReceived = append(mds.TotalReceived, mds.SendResponse.ShasumDataChunk...)
return mds.SendError
}
Loading

0 comments on commit bb6b5b7

Please sign in to comment.