[Terrarium - Providers]: Implementation of gRPC backend microservice …

…integrated with DynamoDB for Data Persistence (#68)

* Implementation of the provider registry protocol v1 (#65)

* changes for testing, adding a metadata for windows amd64 plat

* removed provider stuff from docker compose that was added for testing

* updated terraform.json

* worked on review comments

* go fmt

* removed unused code

* added support for v3.6.0 linux amd

* worked on review comment suggestions

* Web ui & rest microservice for providers

* updated pushed changes

* go fmt

* updatedone of the params of provider list item

* worked on review comments

* version-manager-service

* go fmt

* update in gen proto script

* correction "updating previously existing sourceUrl to sourceRepoUrl as we have a sourceUrl as providerMetadata"

* pushing test changes here, not the final one, this branch will be deleted here, as the main branch is ciedev-3252

* update in version_manager

* adding unit tests

* removed providers.json

* created startGRPCService for providers

* modified schema & updated services according to it, removed abort provider function, and updated provider structure

* go fmt

* updated unit tests based on updated schema, and fixed repeated list of providers in UI issue.

* optimised the query to DB for list providers & get provider to improve time complexity.

* go fmt

* fix

* Update cmd/allInOne.go

Co-authored-by: Adam Charrett <[email protected]>

* Update cmd/allInOne.go

Co-authored-by: Adam Charrett <[email protected]>

* Update cmd/gateway_provider.go

Co-authored-by: Adam Charrett <[email protected]>

* introduced a new package for service and common gateway for both modules & providers, small fix in ui(source_repo_url fix), addition of span attributes, renaming of moduleServices, update in docker compose, browse cmd, allInOne, root cmds, added test functions for providers in gateway, worked for review comments

* go fmt

* updated grpc service naming convention from camel case to snake case

* added missing opentelemetry span

---------

Co-authored-by: Anjani <[email protected]>
Co-authored-by: Anjani Kumar Srivastava <[email protected]>
Co-authored-by: Adam Charrett <[email protected]>

Dockerfile

@@ -15,7 +15,6 @@ RUN apt-get update && \
     apt-get install -y ca-certificates
 
 FROM scratch
-COPY --from=build /workspace/providers.json /
 COPY --from=build /workspace/terrarium /
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT [ "/terrarium" ]

cmd/allInOne.go

@@ -4,25 +4,27 @@ Copyright © 2023 NAME HERE <EMAIL ADDRESS>
 package cmd
 
 import (
+	"log"
+	"net"
+	"net/http"
+
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
-	services2 "github.com/terrariumcloud/terrarium/internal/module/services"
+	"github.com/terrariumcloud/terrarium/internal/common/gateway"
+	grpcServices "github.com/terrariumcloud/terrarium/internal/common/grpc_service"
 	"github.com/terrariumcloud/terrarium/internal/module/services/dependency_manager"
-	"github.com/terrariumcloud/terrarium/internal/module/services/gateway"
 	"github.com/terrariumcloud/terrarium/internal/module/services/registrar"
 	storage2 "github.com/terrariumcloud/terrarium/internal/module/services/storage"
 	"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
 	"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
-	providers_services "github.com/terrariumcloud/terrarium/internal/provider/services"
+	providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	"github.com/terrariumcloud/terrarium/internal/release/services/release"
 	"github.com/terrariumcloud/terrarium/internal/restapi/browse"
 	modulesv1 "github.com/terrariumcloud/terrarium/internal/restapi/modules/v1"
+	providersv1 "github.com/terrariumcloud/terrarium/internal/restapi/providers/v1"
 	"github.com/terrariumcloud/terrarium/internal/storage"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
-	"log"
-	"net"
-	"net/http"
 )
 
 const (
@@ -84,13 +86,20 @@ var allInOneCmd = &cobra.Command{
 			ReleaseService: release.NewPublisherGrpcClient(allInOneInternalEndpoint),
 		}
 
-		services := []services2.Service{
+		providerVersionManagerServer := &providerVersionManager.VersionManagerService{
+			Db:     storage.NewDynamoDbClient(awsSessionConfig),
+			Table:  providerVersionManager.VersionsTableName,
+			Schema: providerVersionManager.GetProviderVersionsSchema(providerVersionManager.VersionsTableName),
+		}
+
+		services := []grpcServices.Service{
 			dependencyServiceServer,
 			registrarServiceServer,
 			storageServiceServer,
 			tagManagerServer,
 			releaseServiceServer,
 			versionManagerServer,
+			providerVersionManagerServer,
 		}
 
 		otelShutdown := initOpenTelemetry("all-in-one")
@@ -104,27 +113,26 @@ var allInOneCmd = &cobra.Command{
 			storage2.NewStorageGrpcClient(allInOneInternalEndpoint),
 			dependency_manager.NewDependencyManagerGrpcClient(allInOneInternalEndpoint),
 			release.NewPublisherGrpcClient(allInOneInternalEndpoint),
+			providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint),
 		)
-		startAllInOneGrpcServices([]services2.Service{gatewayServer}, allInOneGrpcGatewayEndpoint)
 
-		version_manager_svc, err := providers_services.NewJSONFileProviderVersionManager()
-		if err != nil {
-			panic(err)
-		}
+		startAllInOneGrpcServices([]grpcServices.Service{gatewayServer}, allInOneGrpcGatewayEndpoint)
 
 		restAPIServer := browse.New(registrar.NewRegistrarGrpcClient(allInOneInternalEndpoint),
 			version_manager.NewVersionManagerGrpcClient(allInOneInternalEndpoint),
-			release.NewBrowseGrpcClient(allInOneInternalEndpoint), version_manager_svc)
+			release.NewBrowseGrpcClient(allInOneInternalEndpoint),
+			providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))
 
 		modulesAPIServer := modulesv1.New(version_manager.NewVersionManagerGrpcClient(allInOneInternalEndpoint), storage2.NewStorageGrpcClient(allInOneInternalEndpoint))
+		providersAPIServer := providersv1.New(providerVersionManager.NewVersionManagerGrpcClient(allInOneInternalEndpoint))
 
 		router := mux.NewRouter()
 		router.PathPrefix("/modules").Handler(modulesAPIServer.GetHttpHandler("/modules"))
+		router.PathPrefix("/providers").Handler(providersAPIServer.GetHttpHandler("/providers"))
 		router.PathPrefix("/").Handler(restAPIServer.GetHttpHandler(""))
 
 		endpoint = allInOneHTTPEndpoint
 		startRESTAPIService("browse", "", allInOneRestHandler{router: router})
-
 	},
 }
 
@@ -137,9 +145,10 @@ func init() {
 	allInOneCmd.Flags().StringVar(&registrar.RegistrarTableName, "registrar-table", registrar.DefaultRegistrarTableName, "Module Registrar table name")
 	allInOneCmd.Flags().StringVar(&dependency_manager.ModuleDependenciesTableName, "module-dependencies-table", dependency_manager.DefaultModuleDependenciesTableName, "Module dependencies table name")
 	allInOneCmd.Flags().StringVar(&dependency_manager.ContainerDependenciesTableName, "container-dependencies-table", dependency_manager.DefaultContainerDependenciesTableName, "Module container dependencies table name")
+	allInOneCmd.Flags().StringVar(&providerVersionManager.VersionsTableName, "provider-table", providerVersionManager.DefaultProviderVersionsTableName, "Provider versions table name")
 }
 
-func startAllInOneGrpcServices(services []services2.Service, endpoint string) {
+func startAllInOneGrpcServices(services []grpcServices.Service, endpoint string) {
 	listener, err := net.Listen("tcp4", endpoint)
 	if err != nil {
 		log.Fatalf("Failed to start: %v", err)

cmd/browse.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"github.com/spf13/cobra"
+
 	"github.com/terrariumcloud/terrarium/internal/module/services/registrar"
 	"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
-	providerServices "github.com/terrariumcloud/terrarium/internal/provider/services"
+	providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	"github.com/terrariumcloud/terrarium/internal/release/services/release"
 	"github.com/terrariumcloud/terrarium/internal/restapi/browse"
 )
@@ -17,21 +18,17 @@ var browseCmd = &cobra.Command{
 }
 
 func init() {
-	browseCmd.Flags().StringVarP(&registrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Registrar Service")
-	browseCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service")
+	browseCmd.Flags().StringVarP(&registrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Module Registrar Service")
+	browseCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Module Version Manager Service")
 	browseCmd.Flags().StringVarP(&release.ReleaseServiceEndpoint, "release", "", release.DefaultReleaseServiceEndpoint, "GRPC Endpoint for Release Service")
+	browseCmd.Flags().StringVarP(&providerVersionManager.VersionManagerEndpoint, "provider-version-manager", "", providerVersionManager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Provider Version Manager Service")
 	rootCmd.AddCommand(browseCmd)
 }
 
 func runBrowseServer(cmd *cobra.Command, args []string) {
-
-	version_manager_svc, err := providerServices.NewJSONFileProviderVersionManager()
-	if err != nil {
-		panic(err)
-	}
-
 	restAPIServer := browse.New(registrar.NewRegistrarGrpcClient(registrar.RegistrarServiceEndpoint),
 		version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint),
-		release.NewBrowseGrpcClient(release.ReleaseServiceEndpoint), version_manager_svc)
+		release.NewBrowseGrpcClient(release.ReleaseServiceEndpoint),
+		providerVersionManager.NewVersionManagerGrpcClient(providerVersionManager.VersionManagerEndpoint))
 	startRESTAPIService("browse", "", restAPIServer)
 }

cmd/gateway.go

@@ -1,12 +1,13 @@
 package cmd
 
 import (
+	"github.com/terrariumcloud/terrarium/internal/common/gateway"
 	"github.com/terrariumcloud/terrarium/internal/module/services/dependency_manager"
-	"github.com/terrariumcloud/terrarium/internal/module/services/gateway"
 	"github.com/terrariumcloud/terrarium/internal/module/services/registrar"
 	"github.com/terrariumcloud/terrarium/internal/module/services/storage"
 	"github.com/terrariumcloud/terrarium/internal/module/services/tag_manager"
 	"github.com/terrariumcloud/terrarium/internal/module/services/version_manager"
+	providerVersionManager "github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	"github.com/terrariumcloud/terrarium/internal/release/services/release"
 
 	"github.com/spf13/cobra"
@@ -23,10 +24,11 @@ func init() {
 	rootCmd.AddCommand(gatewayCmd)
 	gatewayCmd.Flags().StringVarP(&registrar.RegistrarServiceEndpoint, "registrar", "", registrar.DefaultRegistrarServiceEndpoint, "GRPC Endpoint for Registrar Service")
 	gatewayCmd.Flags().StringVarP(&dependency_manager.DependencyManagerEndpoint, "dependency-manager", "", dependency_manager.DefaultDependencyManagerEndpoint, "GRPC Endpoint for Dependency Manager Service")
-	gatewayCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service")
+	gatewayCmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "version-manager", "", version_manager.DefaultVersionManagerEndpoint, "GRPC Endpoint for Module Version Manager Service")
 	gatewayCmd.Flags().StringVarP(&storage.StorageServiceEndpoint, "storage", "", storage.DefaultStorageServiceDefaultEndpoint, "GRPC Endpoint for Storage Service")
 	gatewayCmd.Flags().StringVarP(&tag_manager.TagManagerEndpoint, "tag-manager", "", tag_manager.DefaultTagManagerEndpoint, "GRPC Endpoint for Tag Service")
 	gatewayCmd.Flags().StringVarP(&release.ReleaseServiceEndpoint, "release", "", release.DefaultReleaseServiceEndpoint, "GRPC Endpoint for Release Service")
+	gatewayCmd.Flags().StringVarP(&providerVersionManager.VersionManagerEndpoint, "provider-version-manager", "", providerVersionManager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Provider Version Manager Service")
 }
 
 func runGateway(cmd *cobra.Command, args []string) {
@@ -37,6 +39,7 @@ func runGateway(cmd *cobra.Command, args []string) {
 		storage.NewStorageGrpcClient(storage.StorageServiceEndpoint),
 		dependency_manager.NewDependencyManagerGrpcClient(dependency_manager.DependencyManagerEndpoint),
 		release.NewPublisherGrpcClient(release.ReleaseServiceEndpoint),
+		providerVersionManager.NewVersionManagerGrpcClient(providerVersionManager.VersionManagerEndpoint),
 	)
 
 	startGRPCService("api-gateway", gatewayServer)

cmd/provider_version_manager.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
+	"github.com/terrariumcloud/terrarium/internal/storage"
+
+	"github.com/spf13/cobra"
+)
+
+var providerVersionManagerServiceCmd = &cobra.Command{
+	Use:   "provider-version-manager",
+	Short: "Starts the Terrarium GRPC Provider Version Manager service",
+	Long:  "Runs the Terrarium GRPC Provider Version Manager server.",
+	Run:   runProviderVersionManagerService,
+}
+
+func init() {
+	rootCmd.AddCommand(providerVersionManagerServiceCmd)
+	providerVersionManagerServiceCmd.Flags().StringVarP(&version_manager.VersionsTableName, "table", "t", version_manager.DefaultProviderVersionsTableName, "Provider Version Manager table name")
+}
+
+func runProviderVersionManagerService(cmd *cobra.Command, args []string) {
+
+	versionManagerServiceServer := &version_manager.VersionManagerService{
+		Db:     storage.NewDynamoDbClient(awsSessionConfig),
+		Table:  version_manager.VersionsTableName,
+		Schema: version_manager.GetProviderVersionsSchema(version_manager.VersionsTableName),
+	}
+
+	startGRPCService("provider-version-manager", versionManagerServiceServer)
+}

cmd/rest_providers_v1.go

@@ -1,10 +1,9 @@
 package cmd
 
 import (
+	"github.com/terrariumcloud/terrarium/internal/provider/services/version_manager"
 	providersv1 "github.com/terrariumcloud/terrarium/internal/restapi/providers/v1"
 
-	"github.com/terrariumcloud/terrarium/internal/provider/services"
-
 	"github.com/spf13/cobra"
 )
 
@@ -25,14 +24,11 @@ func init() {
 		"providers",
 		"Mount path for the rest API server used to process request relative to a particular URL in a reverse proxy type setup",
 	)
+	providersV1Cmd.Flags().StringVarP(&version_manager.VersionManagerEndpoint, "provider-version-manager", "", version_manager.DefaultProviderVersionManagerEndpoint, "GRPC Endpoint for Version Manager Service")
 	rootCmd.AddCommand(providersV1Cmd)
 }
 
 func runRESTProvidersV1Server(cmd *cobra.Command, args []string) {
-	version_manager_svc, err := services.NewJSONFileProviderVersionManager()
-	if err != nil {
-		panic(err)
-	}
-	restAPIServer := providersv1.New(version_manager_svc)
+	restAPIServer := providersv1.New(version_manager.NewVersionManagerGrpcClient(version_manager.VersionManagerEndpoint))
 	startRESTAPIService("rest-providers-v1", mountPathProviders, restAPIServer)
 }

cmd/root.go

@@ -11,10 +11,9 @@ import (
 
 	"go.opentelemetry.io/otel/propagation"
 
+	"github.com/terrariumcloud/terrarium/internal/common/grpc_service"
 	"github.com/terrariumcloud/terrarium/internal/restapi"
 
-	"github.com/terrariumcloud/terrarium/internal/module/services"
-
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -119,7 +118,7 @@ func initOpenTelemetry(name string) func() {
 	}
 }
 
-func startGRPCService(name string, service services.Service) {
+func startGRPCService(name string, service grpc_service.Service) {
 	log.Printf("Starting %s", name)
 	if !opentelemetryInited {
 		otelShutdown := initOpenTelemetry(name)

docker-compose.yaml

@@ -107,6 +107,27 @@ services:
       - "$AWS_SECRET_ACCESS_KEY"
       - "--aws-region"
       - "$AWS_DEFAULT_REGION"
+  provider_version_manager:
+    build: .
+    image: terrarium:dev
+    container_name: provider-version-service
+    environment:
+     - AWS_ACCESS_KEY_ID
+     - AWS_SECRET_ACCESS_KEY
+     - AWS_DEFAULT_REGION
+     - OTEL_EXPORTER_OTLP_ENDPOINT=jaeger:4317
+    ports:
+      - 50009:3001
+    networks:
+      - terrarium
+    command:
+      - provider-version-manager
+      - "--aws-access-key-id"
+      - "$AWS_ACCESS_KEY_ID"
+      - "--aws-secret-access-key"
+      - "$AWS_SECRET_ACCESS_KEY"
+      - "--aws-region"
+      - "$AWS_DEFAULT_REGION"
   dependency_manager:
     build: .
     image: terrarium:dev