diff --git a/cmd/allInOne.go b/cmd/allInOne.go index d7e9e6a..132c81d 100644 --- a/cmd/allInOne.go +++ b/cmd/allInOne.go @@ -17,6 +17,7 @@ import ( storage2 "github.com/terrariumcloud/terrarium/internal/module/services/storage" "github.com/terrariumcloud/terrarium/internal/module/services/tag_manager" "github.com/terrariumcloud/terrarium/internal/module/services/version_manager" + providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage" providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager" "github.com/terrariumcloud/terrarium/internal/release/services/release" "github.com/terrariumcloud/terrarium/internal/restapi/browse" @@ -92,6 +93,12 @@ var allInOneCmd = &cobra.Command{ Schema: providerVersionManager.GetProviderVersionsSchema(providerVersionManager.VersionsTableName), } + providerStorageServiceServer := &providerStorage.StorageService{ + Client: storage.NewS3Client(awsSessionConfig), + BucketName: providerStorage.BucketName, + Region: awsSessionConfig.Region, + } + services := []grpcServices.Service{ dependencyServiceServer, registrarServiceServer, @@ -100,6 +107,7 @@ var allInOneCmd = &cobra.Command{ releaseServiceServer, versionManagerServer, providerVersionManagerServer, + providerStorageServiceServer, } otelShutdown := initOpenTelemetry("all-in-one") @@ -114,6 +122,7 @@ var allInOneCmd = &cobra.Command{ dependency_manager.NewDependencyManagerGrpcClient(allInOneInternalEndpoint), release.NewPublisherGrpcClient(allInOneInternalEndpoint), providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), + providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint), ) startAllInOneGrpcServices([]grpcServices.Service{gatewayServer}, allInOneGrpcGatewayEndpoint) @@ -124,7 +133,7 @@ var allInOneCmd = &cobra.Command{ providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint)) modulesAPIServer := modulesv1.New(version_manager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), storage2.NewStorageGrpcClient(allInOneInternalEndpoint)) - providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint)) + providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint)) router := mux.NewRouter() router.PathPrefix("/modules").Handler(modulesAPIServer.GetHttpHandler("/modules")) @@ -146,6 +155,7 @@ func init() { allInOneCmd.Flags().StringVar(&dependency_manager.ModuleDependenciesTableName, "module-dependencies-table", dependency_manager.DefaultModuleDependenciesTableName, "Module dependencies table name") allInOneCmd.Flags().StringVar(&dependency_manager.ContainerDependenciesTableName, "container-dependencies-table", dependency_manager.DefaultContainerDependenciesTableName, "Module container dependencies table name") allInOneCmd.Flags().StringVar(&providerVersionManager.VersionsTableName, "provider-table", providerVersionManager.DefaultProviderVersionsTableName, "Provider versions table name") + allInOneCmd.Flags().StringVar(&providerStorage.BucketName, "provider-storage-bucket", providerStorage.DefaultBucketName, "Provider bucket name") } func startAllInOneGrpcServices(services []grpcServices.Service, endpoint string) { diff --git a/cmd/gateway.go b/cmd/gateway.go index a0121e7..2a556ee 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -7,6 +7,7 @@ import ( "github.com/terrariumcloud/terrarium/internal/module/services/storage" "github.com/terrariumcloud/terrarium/internal/module/services/tag_manager" "github.com/terrariumcloud/terrarium/internal/module/services/version_manager" + providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage" providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager" "github.com/terrariumcloud/terrarium/internal/release/services/release" @@ -25,10 +26,11 @@ func init() { gatewayCmd.Flags().StringVarP(®istrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Registrar Service") gatewayCmd.Flags().StringVarP(&dependency_manager.DependencyManagerEndpoint, "dependency-manager", "", dependency_manager.DefaultDependencyManagerEndpoint, "GRPC Endpoint for Dependency Manager Service") gatewayCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Module Version Manager Service") - gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Storage Service") + gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Module Storage Service") gatewayCmd.Flags().StringVarP(&tag_manager.TagManagerEndpoint, "tag-manager", "", tag_manager.DefaultTagManagerEndpoint, "GRPC Endpoint for Tag Service") gatewayCmd.Flags().StringVarP(&release.ReleaseServiceEndpoint, "release", "", release.DefaultReleaseServiceEndpoint, "GRPC Endpoint for Release Service") gatewayCmd.Flags().StringVarP(&providerVersionManager.VersionManagerEndpoint, "provider-version-manager", "", providerVersionManager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Provider Version Manager Service") + gatewayCmd.Flags().StringVarP(&providerStorage.StorageServiceEndpoint, "provider-storage", "", providerStorage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service") } func runGateway(cmd *cobra.Command, args []string) { @@ -40,6 +42,7 @@ func runGateway(cmd *cobra.Command, args []string) { dependency_manager.NewDependencyManagerGrpcClient(dependency_manager.DependencyManagerEndpoint), release.NewPublisherGrpcClient(release.ReleaseServiceEndpoint), providerVersionManager.NewVersionManagerGrpcClient(providerVersionManager.VersionManagerEndpoint), + providerStorage.NewStorageGrpcClient(providerStorage.StorageServiceEndpoint), ) startGRPCService("api-gateway", gatewayServer) diff --git a/cmd/provider_storage.go b/cmd/provider_storage.go new file mode 100644 index 0000000..9018243 --- /dev/null +++ b/cmd/provider_storage.go @@ -0,0 +1,31 @@ +package cmd + +import ( + providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage" + "github.com/terrariumcloud/terrarium/internal/storage" + + "github.com/spf13/cobra" +) + +var providerStorageServiceCmd = &cobra.Command{ + Use: "provider-storage", + Short: "Starts the Terrarium GRPC Provider Storage service", + Long: "Runs the Terrarium GRPC Provider Storage server.", + Run: runProviderStorageService, +} + +func init() { + rootCmd.AddCommand(providerStorageServiceCmd) + providerStorageServiceCmd.Flags().StringVarP(&providerStorage.BucketName, "bucket", "b", providerStorage.DefaultBucketName, "Provider bucket name") +} + +func runProviderStorageService(cmd *cobra.Command, args []string) { + + storageServiceServer := &providerStorage.StorageService{ + Client: storage.NewS3Client(awsSessionConfig), + BucketName: providerStorage.BucketName, + Region: awsSessionConfig.Region, + } + + startGRPCService("provider-storage-s3", storageServiceServer) +} diff --git a/cmd/rest_providers_v1.go b/cmd/rest_providers_v1.go index 405ca32..e800cdd 100644 --- a/cmd/rest_providers_v1.go +++ b/cmd/rest_providers_v1.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/terrariumcloud/terrarium/internal/provider/services/storage" "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager" providersv1 "github.com/terrariumcloud/terrarium/internal/restapi/providers/v1" @@ -25,10 +26,15 @@ func init() { "Mount path for the rest API server used to process request relative to a particular URL in a reverse proxy type setup", ) providersV1Cmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "provider-version-manager", "", version_manager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service") + providersV1Cmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "provider-storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service") + rootCmd.AddCommand(providersV1Cmd) } func runRESTProvidersV1Server(cmd *cobra.Command, args []string) { - restAPIServer := providersv1.New(version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint)) + restAPIServer := providersv1.New( + version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint), + storage.NewStorageGrpcClient(storage.StorageServiceEndpoint), + ) startRESTAPIService("rest-providers-v1", mountPathProviders, restAPIServer) } diff --git a/docker-compose.yaml b/docker-compose.yaml index 880cf74..2ef4237 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -170,6 +170,27 @@ services: - "$AWS_SECRET_ACCESS_KEY" - "--aws-region" - "$AWS_DEFAULT_REGION" + provider-storage: + build: . + image: terrarium:dev + container_name: terrarium-provider-storage-service + environment: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_DEFAULT_REGION + - OTEL_EXPORTER_OTLP_ENDPOINT=jaeger:4317 + ports: + - 50010:3001 + networks: + - terrarium + command: + - provider-storage + - "--aws-access-key-id" + - "$AWS_ACCESS_KEY_ID" + - "--aws-secret-access-key" + - "$AWS_SECRET_ACCESS_KEY" + - "--aws-region" + - "$AWS_DEFAULT_REGION" release: build: . image: terrarium:dev diff --git a/internal/common/gateway/gateway.go b/internal/common/gateway/gateway.go index 8fd7b52..4c657b9 100644 --- a/internal/common/gateway/gateway.go +++ b/internal/common/gateway/gateway.go @@ -44,6 +44,7 @@ type TerrariumGrpcGateway struct { storageClient moduleServices.StorageClient dependencyManagerClient moduleServices.DependencyManagerClient releasePublisherClient release.PublisherClient + providerStorageClient providerServices.StorageClient } func New(registrarClient moduleServices.RegistrarClient, @@ -52,7 +53,8 @@ func New(registrarClient moduleServices.RegistrarClient, storageClient moduleServices.StorageClient, dependencyManagerClient moduleServices.DependencyManagerClient, releasePublisherClient release.PublisherClient, - providerVersionManagerClient providerServices.VersionManagerClient) *TerrariumGrpcGateway { + providerVersionManagerClient providerServices.VersionManagerClient, + providerStorageClient providerServices.StorageClient) *TerrariumGrpcGateway { return &TerrariumGrpcGateway{ registrarClient: registrarClient, tagManagerClient: tagManagerClient, @@ -61,6 +63,7 @@ func New(registrarClient moduleServices.RegistrarClient, dependencyManagerClient: dependencyManagerClient, releasePublisherClient: releasePublisherClient, providerVersionManagerClient: providerVersionManagerClient, + providerStorageClient: providerStorageClient, } } diff --git a/internal/restapi/providers/v1/handler.go b/internal/restapi/providers/v1/handler.go index c6d7ff0..1cd1f68 100644 --- a/internal/restapi/providers/v1/handler.go +++ b/internal/restapi/providers/v1/handler.go @@ -2,7 +2,9 @@ package v1 import ( "encoding/json" + "errors" "fmt" + "io" "log" "net/http" "os" @@ -20,12 +22,16 @@ import ( type providersV1HttpService struct { versionManagerClient services.VersionManagerClient + storageClient services.StorageClient responseHandler restapi.ResponseHandler errorHandler restapi.ErrorHandler } -func New(versionManagerClient services.VersionManagerClient) *providersV1HttpService { - return &providersV1HttpService{versionManagerClient: versionManagerClient} +func New(versionManagerClient services.VersionManagerClient, storageClient services.StorageClient) *providersV1HttpService { + return &providersV1HttpService{ + versionManagerClient: versionManagerClient, + storageClient: storageClient, + } } func (h *providersV1HttpService) GetHttpHandler(mountPath string) http.Handler { @@ -43,6 +49,9 @@ func (h *providersV1HttpService) createRouter(mountPath string) *mux.Router { sr.StrictSlash(true) sr.Handle("/{organization_name}/{name}/versions", h.getProviderVersionHandler()).Methods(http.MethodGet) sr.Handle("/{organization_name}/{name}/{version}/download/{os}/{arch}", h.downloadProviderHandler()).Methods(http.MethodGet) + sr.Handle("/{organization_name}/{name}/{version}/{os}/{arch}/terraform-provider-{name}_{version}_{os}_{arch}.zip", h.archiveHandler()).Methods(http.MethodGet) + sr.Handle("/{organization_name}/{name}/{version}/terraform-provider-{name}_{version}_SHA256SUMS", h.shasumHandler()).Methods(http.MethodGet) + sr.Handle("/{organization_name}/{name}/{version}/terraform-provider-{name}_{version}_SHA256SUMS.sig", h.shasumSignatureHandler()).Methods(http.MethodGet) return r } @@ -120,3 +129,105 @@ func (h *providersV1HttpService) downloadProviderHandler() http.Handler { _, _ = rw.Write(data) }) } + +// archiveHandler performs a fetch of the provider binary from the chosen backing store and presents it to the client. +func (h *providersV1HttpService) archiveHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + providerName := GetProviderNameFromRequest(r) + + ctx := r.Context() + span := trace.SpanFromContext(ctx) + providerVersion, providerOS, providerArch := GetProviderInputsFromRequest(r) + span.SetAttributes( + attribute.String("provider.name", providerName), + attribute.String("provider.version", providerVersion), + attribute.String("provider.os", providerOS), + attribute.String("provider.arch", providerArch), + ) + downloadStream, err := h.storageClient.DownloadProviderSourceZip(r.Context(), &services.DownloadSourceZipRequest{ + Provider: GetProviderLocationFromRequest(r), + }) + if err != nil { + log.Printf("Failed to connect: %v", err) + span.RecordError(err) + h.errorHandler.Write(rw, errors.New("failed to initiate the download of the archive from storage backend service"), http.StatusInternalServerError) + return + } + r.Header.Set("Content-Type", "application/zip") + for { + chunk, err := downloadStream.Recv() + if err == io.EOF { + return + } + _, _ = rw.Write(chunk.ZipDataChunk) + } + }) +} + +// shasumHandler performs a fetch of the shasum file from the chosen backing store and presents it to the client. +func (h *providersV1HttpService) shasumHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + providerName := GetProviderNameFromRequest(r) + + ctx := r.Context() + span := trace.SpanFromContext(ctx) + providerVersion, _, _ := GetProviderInputsFromRequest(r) + span.SetAttributes( + attribute.String("provider.name", providerName), + attribute.String("provider.version", providerVersion), + ) + + downloadStream, err := h.storageClient.DownloadShasum(r.Context(), &services.DownloadShasumRequest{ + Provider: GetVersionedProviderFromRequest(r), + }) + if err != nil { + log.Printf("Failed to connect: %v", err) + span.RecordError(err) + h.errorHandler.Write(rw, errors.New("failed to initiate the download of the shasum file from storage backend service"), http.StatusInternalServerError) + return + } + + r.Header.Set("Content-Type", "text/plain") + for { + chunk, err := downloadStream.Recv() + if err == io.EOF { + return + } + _, _ = rw.Write(chunk.ShasumDataChunk) + } + }) +} + +// shasumSignatureHandler performs a fetch of the shasum signature file from the chosen backing store and presents it to the client. +func (h *providersV1HttpService) shasumSignatureHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + providerName := GetProviderNameFromRequest(r) + + ctx := r.Context() + span := trace.SpanFromContext(ctx) + providerVersion, _, _ := GetProviderInputsFromRequest(r) + span.SetAttributes( + attribute.String("provider.name", providerName), + attribute.String("provider.version", providerVersion), + ) + + downloadStream, err := h.storageClient.DownloadShasumSignature(r.Context(), &services.DownloadShasumRequest{ + Provider: GetVersionedProviderFromRequest(r), + }) + if err != nil { + log.Printf("Failed to connect: %v", err) + span.RecordError(err) + h.errorHandler.Write(rw, errors.New("failed to initiate the download of the shasum signature file from storage backend service"), http.StatusInternalServerError) + return + } + + r.Header.Set("Content-Type", "text/plain") + for { + chunk, err := downloadStream.Recv() + if err == io.EOF { + return + } + _, _ = rw.Write(chunk.ShasumDataChunk) + } + }) +} diff --git a/internal/restapi/providers/v1/helpers.go b/internal/restapi/providers/v1/helpers.go index 1fb5d22..d92ab71 100644 --- a/internal/restapi/providers/v1/helpers.go +++ b/internal/restapi/providers/v1/helpers.go @@ -5,6 +5,9 @@ import ( "net/http" "github.com/gorilla/mux" + + "github.com/terrariumcloud/terrarium/internal/provider/services" + pb "github.com/terrariumcloud/terrarium/pkg/terrarium/provider" ) func GetProviderNameFromRequest(r *http.Request) string { @@ -18,3 +21,29 @@ func GetProviderInputsFromRequest(r *http.Request) (string, string, string) { params := mux.Vars(r) return params["version"], params["os"], params["arch"] } + +func GetProviderLocationFromRequest(r *http.Request) *services.ProviderRequest { + params := mux.Vars(r) + orgName := params["organization_name"] + providerName := params["name"] + version := params["version"] + os := params["os"] + arch := params["arch"] + return &services.ProviderRequest{ + Name: fmt.Sprintf("%s/%s", orgName, providerName), + Version: version, + Os: os, + Arch: arch, + } +} + +func GetVersionedProviderFromRequest(r *http.Request) *pb.Provider { + params := mux.Vars(r) + orgName := params["organization_name"] + providerName := params["name"] + version := params["version"] + return &pb.Provider{ + Name: fmt.Sprintf("%s/%s", orgName, providerName), + Version: version, + } +} diff --git a/internal/restapi/providers/v1/helpers_test.go b/internal/restapi/providers/v1/helpers_test.go index 5961589..c17f25b 100644 --- a/internal/restapi/providers/v1/helpers_test.go +++ b/internal/restapi/providers/v1/helpers_test.go @@ -8,22 +8,22 @@ import ( ) func Test_GetProviderNameFromRequest(t *testing.T) { - req := httptest.NewRequest("GET", "/providers/v1/hashicorp/random/versions", nil) + req := httptest.NewRequest("GET", "/providers/v1/test-org/test-provider/versions", nil) req = mux.SetURLVars(req, map[string]string{ - "organization_name": "hashicorp", - "name": "random", + "organization_name": "test-org", + "name": "test-provider", }) providerName := GetProviderNameFromRequest(req) - expectedProviderName := "hashicorp/random" + expectedProviderName := "test-org/test-provider" if providerName != expectedProviderName { t.Errorf("Expected provider name to be %s, but got %s", expectedProviderName, providerName) } } func Test_GetProviderInputsFromRequest(t *testing.T) { - req := httptest.NewRequest("GET", "/providers/v1/hashicorp/random/2.0.0/download/linux/amd64", nil) + req := httptest.NewRequest("GET", "/providers/v1/test-org/test-provider/2.0.0/download/linux/amd64", nil) req = mux.SetURLVars(req, map[string]string{ "version": "2.0.0", "os": "linux", @@ -47,3 +47,57 @@ func Test_GetProviderInputsFromRequest(t *testing.T) { t.Errorf("Expected arch. to be %s, but got %s", expectedArch, arch) } } + +func Test_GetProviderLocationFromRequest(t *testing.T) { + req := httptest.NewRequest("GET", "/providers/v1/test-org/test-provider/2.0.0/linux/amd64/terraform-provider-test-provider_2.0.0_linux_amd64.zip", nil) + req = mux.SetURLVars(req, map[string]string{ + "organization_name": "test-org", + "name": "test-provider", + "version": "2.0.0", + "os": "linux", + "arch": "amd64", + }) + + provider := GetProviderLocationFromRequest(req) + + expectedProviderName := "test-org/test-provider" + if provider.Name != expectedProviderName { + t.Errorf("Expected provider name to be %s, but got %s", expectedProviderName, provider.Name) + } + + expectedVersion := "2.0.0" + if provider.Version != expectedVersion { + t.Errorf("Expected version to be %s, but got %s", expectedVersion, provider.Version) + } + + expectedOS := "linux" + if provider.Os != expectedOS { + t.Errorf("Expected OS to be %s, but got %s", expectedOS, provider.Os) + } + + expectedArch := "amd64" + if provider.Arch != expectedArch { + t.Errorf("Expected arch. to be %s, but got %s", expectedArch, provider.Arch) + } +} + +func Test_GetVersionedProviderFromRequest(t *testing.T) { + req := httptest.NewRequest("GET", "/providers/v1/test-org/test-provider/2.0.0/terraform-provider-test-provider_2.0.0_SHA256SUMS", nil) + req = mux.SetURLVars(req, map[string]string{ + "organization_name": "test-org", + "name": "test-provider", + "version": "2.0.0", + }) + + provider := GetVersionedProviderFromRequest(req) + + expectedProviderName := "test-org/test-provider" + if provider.Name != expectedProviderName { + t.Errorf("Expected provider name to be %s, but got %s", expectedProviderName, provider.Name) + } + + expectedVersion := "2.0.0" + if provider.Version != expectedVersion { + t.Errorf("Expected version to be %s, but got %s", expectedVersion, provider.Version) + } +}