Support multiple Feast endpoints on Standard Transformer (#169)

* Update proto and re-generate it

* Update UI to support feast serving endpoint selection

* Transformer read feast serving endpoint from YAML spec

* Cleanup image building jobs (#165)

* Move cronjob package to pkg/

* Add new cronjob: Image Builder Janitor

* Add IMG_BUILDER_RETENTION to helm template

* Address review -- changing cronjob frequency and reduce retention duration

* Use batch deletion for image build jobs. Add unit tests

* Revert to deletion one-by-one. Add prometheus metrics

* Fix package sources reference. For further reading:

https://www.mail-archive.com/[email protected]/msg1813968.html
https://superuser.com/questions/1456989/how-to-configure-apt-in-debian-buster-after-release

* Add test for CleanJobs

* Safely delete only succeeded image building jobs (#171)

* Safely delete only succeeded image building jobs

* Address review:

1. Use job.Status.StartTime
2. Keep deleting active or failed job
3. On metrics tracking, also track job status as label

* Little refactoring and add validation on version endpoint api validation

- Rename FeastServingEndpoint to FeastServingURL
- Reformat some tansformer yaml spec
- Validate if servingURL provided by user is valid

* Remove grpc tools setup from Makefile

* Add more API tests

* Update charts

* Update proto and re-generate it

* Update UI to support feast serving endpoint selection

* Transformer read feast serving endpoint from YAML spec

* Little refactoring and add validation on version endpoint api validation

- Rename FeastServingEndpoint to FeastServingURL
- Reformat some tansformer yaml spec
- Validate if servingURL provided by user is valid

* Remove grpc tools setup from Makefile

* Add more API tests

* Update charts

* Add python tests for multiple feast serving url

* Address Tio's review. Fix setting appContext's std transformer config

* Remove unused transformer env var

* Remove unused file

* Update propagation policy on image builder janitor

* Update environment on .env.sample for development purpose

Makefile

@@ -174,4 +174,7 @@ generate-client-python:
 .PHONY: gen-proto
 gen-proto:
 	@echo "> Generating specification configuration from Proto file..."
-	@cd protos/merlin && protoc -I=. --go_out=../../api --go-json_out=../../api/pkg --go_opt=module=github.com/gojek/merlin transformer/**/*.proto
+	@cd protos/merlin && \
+		protoc -I=. --go_out=../../api --go_opt=module=github.com/gojek/merlin \
+		--go-json_out=../../api --go-json_opt=module=github.com/gojek/merlin \
+		transformer/**/*.proto

api/api/router.go

@@ -41,6 +41,9 @@ import (
 
 // AppContext contains the services of the Merlin application.
 type AppContext struct {
+	DB       *gorm.DB
+	Enforcer enforcer.Enforcer
+
 	EnvironmentService        service.EnvironmentService
 	ProjectsService           service.ProjectsService
 	ModelsService             service.ModelsService
@@ -51,13 +54,15 @@ type AppContext struct {
 	PredictionJobService      service.PredictionJobService
 	SecretService             service.SecretService
 	ModelEndpointAlertService service.ModelEndpointAlertService
-	DB                        *gorm.DB
-	AuthorizationEnabled      bool
-	MonitoringConfig          config.MonitoringConfig
-	AlertEnabled              bool
-	Enforcer                  enforcer.Enforcer
-	FeastCoreClient           core.CoreServiceClient
-	MlflowClient              mlflow.Client
+
+	AuthorizationEnabled bool
+	AlertEnabled         bool
+	MonitoringConfig     config.MonitoringConfig
+
+	StandardTransformerConfig config.StandardTransformerConfig
+
+	FeastCoreClient core.CoreServiceClient
+	MlflowClient    mlflow.Client
 }
 
 // Handler handles the API requests and responses.

api/api/version_endpoints_api.go

@@ -28,6 +28,7 @@ import (
 	"github.com/gojek/merlin/config"
 	"github.com/gojek/merlin/models"
 	"github.com/gojek/merlin/pkg/transformer"
+	"github.com/gojek/merlin/pkg/transformer/feast"
 	"github.com/gojek/merlin/pkg/transformer/pipeline"
 	"github.com/gojek/merlin/pkg/transformer/spec"
 )
@@ -406,5 +407,10 @@ func (c *EndpointsController) validateStandardTransformerConfig(ctx context.Cont
 		return err
 	}
 
-	return pipeline.ValidateTransformerConfig(ctx, c.FeastCoreClient, stdTransformerConfig)
+	feastOptions := &feast.Options{
+		DefaultServingURL: c.StandardTransformerConfig.DefaultFeastServingURL,
+		ServingURLs:       c.StandardTransformerConfig.FeastServingURLs.URLs(),
+	}
+
+	return pipeline.ValidateTransformerConfig(ctx, c.FeastCoreClient, stdTransformerConfig, feastOptions)
 }