[Terrarium - Providers]: Implementation of Storage Feature - 3

cmd/allInOne.go

@@ -17,6 +17,7 @@ import (
 	storage2 "github.com/terrariumcloud/terrarium/internal/module/services/storage"
 	"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
 	"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
+	providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
 	providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	"github.com/terrariumcloud/terrarium/internal/release/services/release"
 	"github.com/terrariumcloud/terrarium/internal/restapi/browse"
@@ -92,6 +93,12 @@ var allInOneCmd = &cobra.Command{
 			Schema: providerVersionManager.GetProviderVersionsSchema(providerVersionManager.VersionsTableName),
 		}
 
+		providerStorageServiceServer := &providerStorage.StorageService{
+			Client:     storage.NewS3Client(awsSessionConfig),
+			BucketName: providerStorage.BucketName,
+			Region:     awsSessionConfig.Region,
+		}
+
 		services := []grpcServices.Service{
 			dependencyServiceServer,
 			registrarServiceServer,
@@ -100,6 +107,7 @@ var allInOneCmd = &cobra.Command{
 			releaseServiceServer,
 			versionManagerServer,
 			providerVersionManagerServer,
+			providerStorageServiceServer,
 		}
 
 		otelShutdown := initOpenTelemetry("all-in-one")
@@ -114,6 +122,7 @@ var allInOneCmd = &cobra.Command{
 			dependency_manager.NewDependencyManagerGrpcClient(allInOneInternalEndpoint),
 			release.NewPublisherGrpcClient(allInOneInternalEndpoint),
 			providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint),
+			providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint),
 		)
 
 		startAllInOneGrpcServices([]grpcServices.Service{gatewayServer}, allInOneGrpcGatewayEndpoint)
@@ -124,7 +133,7 @@ var allInOneCmd = &cobra.Command{
 			providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))
 
 		modulesAPIServer := modulesv1.New(version_manager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), storage2.NewStorageGrpcClient(allInOneInternalEndpoint))
-		providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))
+		providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), providerStorage.NewStorageGrpcClient(allInOneInternalEndpoint))
 
 		router := mux.NewRouter()
 		router.PathPrefix("/modules").Handler(modulesAPIServer.GetHttpHandler("/modules"))
@@ -146,6 +155,7 @@ func init() {
 	allInOneCmd.Flags().StringVar(&dependency_manager.ModuleDependenciesTableName, "module-dependencies-table", dependency_manager.DefaultModuleDependenciesTableName, "Module dependencies table name")
 	allInOneCmd.Flags().StringVar(&dependency_manager.ContainerDependenciesTableName, "container-dependencies-table", dependency_manager.DefaultContainerDependenciesTableName, "Module container dependencies table name")
 	allInOneCmd.Flags().StringVar(&providerVersionManager.VersionsTableName, "provider-table", providerVersionManager.DefaultProviderVersionsTableName, "Provider versions table name")
+	allInOneCmd.Flags().StringVar(&providerStorage.BucketName, "provider-storage-bucket", providerStorage.DefaultBucketName, "Provider bucket name")
 }
 
 func startAllInOneGrpcServices(services []grpcServices.Service, endpoint string) {

cmd/gateway.go

@@ -7,6 +7,7 @@ import (
 	"github.com/terrariumcloud/terrarium/internal/module/services/storage"
 	"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
 	"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
+	providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
 	providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	"github.com/terrariumcloud/terrarium/internal/release/services/release"
 
@@ -25,10 +26,11 @@ func init() {
 	gatewayCmd.Flags().StringVarP(&registrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Registrar Service")
 	gatewayCmd.Flags().StringVarP(&dependency_manager.DependencyManagerEndpoint, "dependency-manager", "", dependency_manager.DefaultDependencyManagerEndpoint, "GRPC Endpoint for Dependency Manager Service")
 	gatewayCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Module Version Manager Service")
-	gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Storage Service")
+	gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Module Storage Service")
 	gatewayCmd.Flags().StringVarP(&tag_manager.TagManagerEndpoint, "tag-manager", "", tag_manager.DefaultTagManagerEndpoint, "GRPC Endpoint for Tag Service")
 	gatewayCmd.Flags().StringVarP(&release.ReleaseServiceEndpoint, "release", "", release.DefaultReleaseServiceEndpoint, "GRPC Endpoint for Release Service")
 	gatewayCmd.Flags().StringVarP(&providerVersionManager.VersionManagerEndpoint, "provider-version-manager", "", providerVersionManager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Provider Version Manager Service")
+	gatewayCmd.Flags().StringVarP(&providerStorage.StorageServiceEndpoint, "provider-storage", "", providerStorage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service")
 }
 
 func runGateway(cmd *cobra.Command, args []string) {
@@ -40,6 +42,7 @@ func runGateway(cmd *cobra.Command, args []string) {
 		dependency_manager.NewDependencyManagerGrpcClient(dependency_manager.DependencyManagerEndpoint),
 		release.NewPublisherGrpcClient(release.ReleaseServiceEndpoint),
 		providerVersionManager.NewVersionManagerGrpcClient(providerVersionManager.VersionManagerEndpoint),
+		providerStorage.NewStorageGrpcClient(providerStorage.StorageServiceEndpoint),
 	)
 
 	startGRPCService("api-gateway", gatewayServer)

cmd/provider_storage.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	providerStorage "github.com/terrariumcloud/terrarium/internal/provider/services/storage"
+	"github.com/terrariumcloud/terrarium/internal/storage"
+
+	"github.com/spf13/cobra"
+)
+
+var providerStorageServiceCmd = &cobra.Command{
+	Use:   "provider-storage",
+	Short: "Starts the Terrarium GRPC Provider Storage service",
+	Long:  "Runs the Terrarium GRPC Provider Storage server.",
+	Run:   runProviderStorageService,
+}
+
+func init() {
+	rootCmd.AddCommand(providerStorageServiceCmd)
+	providerStorageServiceCmd.Flags().StringVarP(&providerStorage.BucketName, "bucket", "b", providerStorage.DefaultBucketName, "Provider bucket name")
+}
+
+func runProviderStorageService(cmd *cobra.Command, args []string) {
+
+	storageServiceServer := &providerStorage.StorageService{
+		Client:     storage.NewS3Client(awsSessionConfig),
+		BucketName: providerStorage.BucketName,
+		Region:     awsSessionConfig.Region,
+	}
+
+	startGRPCService("provider-storage-s3", storageServiceServer)
+}

cmd/rest_providers_v1.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/terrariumcloud/terrarium/internal/provider/services/storage"
 	"github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	providersv1 "github.com/terrariumcloud/terrarium/internal/restapi/providers/v1"
 
@@ -25,10 +26,15 @@ func init() {
 		"Mount path for the rest API server used to process request relative to a particular URL in a reverse proxy type setup",
 	)
 	providersV1Cmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "provider-version-manager", "", version_manager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service")
+	providersV1Cmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "provider-storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Provider Storage Service")
+
 	rootCmd.AddCommand(providersV1Cmd)
 }
 
 func runRESTProvidersV1Server(cmd *cobra.Command, args []string) {
-	restAPIServer := providersv1.New(version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint))
+	restAPIServer := providersv1.New(
+		version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint),
+		storage.NewStorageGrpcClient(storage.StorageServiceEndpoint),
+	)
 	startRESTAPIService("rest-providers-v1", mountPathProviders, restAPIServer)
 }

docker-compose.yaml

@@ -170,6 +170,27 @@ services:
       - "$AWS_SECRET_ACCESS_KEY"
       - "--aws-region"
       - "$AWS_DEFAULT_REGION"
+  provider-storage:
+    build: .
+    image: terrarium:dev
+    container_name: terrarium-provider-storage-service
+    environment:
+     - AWS_ACCESS_KEY_ID
+     - AWS_SECRET_ACCESS_KEY
+     - AWS_DEFAULT_REGION
+     - OTEL_EXPORTER_OTLP_ENDPOINT=jaeger:4317
+    ports:
+      - 50010:3001
+    networks:
+      - terrarium
+    command:
+      - provider-storage
+      - "--aws-access-key-id"
+      - "$AWS_ACCESS_KEY_ID"
+      - "--aws-secret-access-key"
+      - "$AWS_SECRET_ACCESS_KEY"
+      - "--aws-region"
+      - "$AWS_DEFAULT_REGION"
   release:
     build: .
     image: terrarium:dev

internal/common/gateway/gateway.go

@@ -44,6 +44,7 @@ type TerrariumGrpcGateway struct {
 	storageClient                moduleServices.StorageClient
 	dependencyManagerClient      moduleServices.DependencyManagerClient
 	releasePublisherClient       release.PublisherClient
+	providerStorageClient        providerServices.StorageClient
 }
 
 func New(registrarClient moduleServices.RegistrarClient,
@@ -52,7 +53,8 @@ func New(registrarClient moduleServices.RegistrarClient,
 	storageClient moduleServices.StorageClient,
 	dependencyManagerClient moduleServices.DependencyManagerClient,
 	releasePublisherClient release.PublisherClient,
-	providerVersionManagerClient providerServices.VersionManagerClient) *TerrariumGrpcGateway {
+	providerVersionManagerClient providerServices.VersionManagerClient,
+	providerStorageClient providerServices.StorageClient) *TerrariumGrpcGateway {
 	return &TerrariumGrpcGateway{
 		registrarClient:              registrarClient,
 		tagManagerClient:             tagManagerClient,
@@ -61,6 +63,7 @@ func New(registrarClient moduleServices.RegistrarClient,
 		dependencyManagerClient:      dependencyManagerClient,
 		releasePublisherClient:       releasePublisherClient,
 		providerVersionManagerClient: providerVersionManagerClient,
+		providerStorageClient:        providerStorageClient,
 	}
 }
 

internal/restapi/providers/v1/handler.go

@@ -2,7 +2,9 @@ package v1
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -20,12 +22,16 @@ import (
 
 type providersV1HttpService struct {
 	versionManagerClient services.VersionManagerClient
+	storageClient        services.StorageClient
 	responseHandler      restapi.ResponseHandler
 	errorHandler         restapi.ErrorHandler
 }
 
-func New(versionManagerClient services.VersionManagerClient) *providersV1HttpService {
-	return &providersV1HttpService{versionManagerClient: versionManagerClient}
+func New(versionManagerClient services.VersionManagerClient, storageClient services.StorageClient) *providersV1HttpService {
+	return &providersV1HttpService{
+		versionManagerClient: versionManagerClient,
+		storageClient:        storageClient,
+	}
 }
 
 func (h *providersV1HttpService) GetHttpHandler(mountPath string) http.Handler {
@@ -43,6 +49,9 @@ func (h *providersV1HttpService) createRouter(mountPath string) *mux.Router {
 	sr.StrictSlash(true)
 	sr.Handle("/{organization_name}/{name}/versions", h.getProviderVersionHandler()).Methods(http.MethodGet)
 	sr.Handle("/{organization_name}/{name}/{version}/download/{os}/{arch}", h.downloadProviderHandler()).Methods(http.MethodGet)
+	sr.Handle("/{organization_name}/{name}/{version}/{os}/{arch}/terraform-provider-{name}_{version}_{os}_{arch}.zip", h.archiveHandler()).Methods(http.MethodGet)
+	sr.Handle("/{organization_name}/{name}/{version}/terraform-provider-{name}_{version}_SHA256SUMS", h.shasumHandler()).Methods(http.MethodGet)
+	sr.Handle("/{organization_name}/{name}/{version}/terraform-provider-{name}_{version}_SHA256SUMS.sig", h.shasumSignatureHandler()).Methods(http.MethodGet)
 	return r
 }
 
@@ -120,3 +129,105 @@ func (h *providersV1HttpService) downloadProviderHandler() http.Handler {
 		_, _ = rw.Write(data)
 	})
 }
+
+// archiveHandler performs a fetch of the provider binary from the chosen backing store and presents it to the client.
+func (h *providersV1HttpService) archiveHandler() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		providerName := GetProviderNameFromRequest(r)
+
+		ctx := r.Context()
+		span := trace.SpanFromContext(ctx)
+		providerVersion, providerOS, providerArch := GetProviderInputsFromRequest(r)
+		span.SetAttributes(
+			attribute.String("provider.name", providerName),
+			attribute.String("provider.version", providerVersion),
+			attribute.String("provider.os", providerOS),
+			attribute.String("provider.arch", providerArch),
+		)
+		downloadStream, err := h.storageClient.DownloadProviderSourceZip(r.Context(), &services.DownloadSourceZipRequest{
+			Provider: GetProviderLocationFromRequest(r),
+		})
+		if err != nil {
+			log.Printf("Failed to connect: %v", err)
+			span.RecordError(err)
+			h.errorHandler.Write(rw, errors.New("failed to initiate the download of the archive from storage backend service"), http.StatusInternalServerError)
+			return
+		}
+		r.Header.Set("Content-Type", "application/zip")
+		for {
+			chunk, err := downloadStream.Recv()
+			if err == io.EOF {
+				return
+			}
+			_, _ = rw.Write(chunk.ZipDataChunk)
+		}
+	})
+}
+
+// shasumHandler performs a fetch of the shasum file from the chosen backing store and presents it to the client.
+func (h *providersV1HttpService) shasumHandler() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		providerName := GetProviderNameFromRequest(r)
+
+		ctx := r.Context()
+		span := trace.SpanFromContext(ctx)
+		providerVersion, _, _ := GetProviderInputsFromRequest(r)
+		span.SetAttributes(
+			attribute.String("provider.name", providerName),
+			attribute.String("provider.version", providerVersion),
+		)
+
+		downloadStream, err := h.storageClient.DownloadShasum(r.Context(), &services.DownloadShasumRequest{
+			Provider: GetVersionedProviderFromRequest(r),
+		})
+		if err != nil {
+			log.Printf("Failed to connect: %v", err)
+			span.RecordError(err)
+			h.errorHandler.Write(rw, errors.New("failed to initiate the download of the shasum file from storage backend service"), http.StatusInternalServerError)
+			return
+		}
+
+		r.Header.Set("Content-Type", "text/plain")
+		for {
+			chunk, err := downloadStream.Recv()
+			if err == io.EOF {
+				return
+			}
+			_, _ = rw.Write(chunk.ShasumDataChunk)
+		}
+	})
+}
+
+// shasumSignatureHandler performs a fetch of the shasum signature file from the chosen backing store and presents it to the client.
+func (h *providersV1HttpService) shasumSignatureHandler() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		providerName := GetProviderNameFromRequest(r)
+
+		ctx := r.Context()
+		span := trace.SpanFromContext(ctx)
+		providerVersion, _, _ := GetProviderInputsFromRequest(r)
+		span.SetAttributes(
+			attribute.String("provider.name", providerName),
+			attribute.String("provider.version", providerVersion),
+		)
+
+		downloadStream, err := h.storageClient.DownloadShasumSignature(r.Context(), &services.DownloadShasumRequest{
+			Provider: GetVersionedProviderFromRequest(r),
+		})
+		if err != nil {
+			log.Printf("Failed to connect: %v", err)
+			span.RecordError(err)
+			h.errorHandler.Write(rw, errors.New("failed to initiate the download of the shasum signature file from storage backend service"), http.StatusInternalServerError)
+			return
+		}
+
+		r.Header.Set("Content-Type", "text/plain")
+		for {
+			chunk, err := downloadStream.Recv()
+			if err == io.EOF {
+				return
+			}
+			_, _ = rw.Write(chunk.ShasumDataChunk)
+		}
+	})
+}

internal/restapi/providers/v1/helpers.go

@@ -5,6 +5,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+
+	"github.com/terrariumcloud/terrarium/internal/provider/services"
+	pb "github.com/terrariumcloud/terrarium/pkg/terrarium/provider"
 )
 
 func GetProviderNameFromRequest(r *http.Request) string {
@@ -18,3 +21,29 @@ func GetProviderInputsFromRequest(r *http.Request) (string, string, string) {
 	params := mux.Vars(r)
 	return params["version"], params["os"], params["arch"]
 }
+
+func GetProviderLocationFromRequest(r *http.Request) *services.ProviderRequest {
+	params := mux.Vars(r)
+	orgName := params["organization_name"]
+	providerName := params["name"]
+	version := params["version"]
+	os := params["os"]
+	arch := params["arch"]
+	return &services.ProviderRequest{
+		Name:    fmt.Sprintf("%s/%s", orgName, providerName),
+		Version: version,
+		Os:      os,
+		Arch:    arch,
+	}
+}
+
+func GetVersionedProviderFromRequest(r *http.Request) *pb.Provider {
+	params := mux.Vars(r)
+	orgName := params["organization_name"]
+	providerName := params["name"]
+	version := params["version"]
+	return &pb.Provider{
+		Name:    fmt.Sprintf("%s/%s", orgName, providerName),
+		Version: version,
+	}
+}