Skip to content

Commit

Permalink
Fail the request when user try to deploy with model observability ena…
Browse files Browse the repository at this point in the history 
…bled but the associated model is not support observability
  • Loading branch information
@tiopramayudi
tiopramayudi committed Dec 2, 2024
1 parent 429aeb9 commit 310e8dc
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 25 deletions.
9 changes: 8 additions & 1 deletion api/api/validator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,16 @@ func deploymentModeValidation(prev *models.VersionEndpoint, new *models.VersionE

func modelObservabilityValidation(endpoint *models.VersionEndpoint, model *models.Model) requestValidator {
return newFuncValidate(func() error {
if endpoint.IsModelMonitoringEnabled() && !slices.Contains(supportedObservabilityModelTypes, model.Type) {
if !endpoint.IsModelMonitoringEnabled() {
return nil
}
if !slices.Contains(supportedObservabilityModelTypes, model.Type) {
return fmt.Errorf("%s: %w", model.Type, ErrUnsupportedObservabilityModelType)
}

if !model.ObservabilitySupported {
return fmt.Errorf("model observability is not supported for this model")
}
return nil
})
}
150 changes: 126 additions & 24 deletions api/api/version_endpoints_api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1073,14 +1073,15 @@ func TestCreateEndpoint(t *testing.T) {
modelService: func() *mocks.ModelsService {
svc := &mocks.ModelsService{}
svc.On("FindByID", mock.Anything, models.ID(1)).Return(&models.Model{
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
ObservabilitySupported: true,
}, nil)
return svc
},
Expand Down Expand Up @@ -1199,6 +1200,105 @@ func TestCreateEndpoint(t *testing.T) {
},
},
},
{
desc: "Fail when try to enable model observability but the model is not supported yet",
vars: map[string]string{
"model_id": "1",
"version_id": "1",
},
requestBody: &models.VersionEndpoint{
ID: uuid,
VersionID: models.ID(1),
VersionModelID: models.ID(1),
ServiceName: "sample",
Namespace: "sample",
EnvironmentName: "dev",
Message: "",
ResourceRequest: &models.ResourceRequest{
MinReplica: 1,
MaxReplica: 4,
CPURequest: resource.MustParse("1"),
MemoryRequest: resource.MustParse("1Gi"),
},
EnvVars: models.EnvVars([]models.EnvVar{
{
Name: "WORKER",
Value: "1",
},
}),
EnableModelObservability: true,
},
modelService: func() *mocks.ModelsService {
svc := &mocks.ModelsService{}
svc.On("FindByID", mock.Anything, models.ID(1)).Return(&models.Model{
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
ObservabilitySupported: false,
}, nil)
return svc
},
versionService: func() *mocks.VersionsService {
svc := &mocks.VersionsService{}
svc.On("FindByID", mock.Anything, models.ID(1), models.ID(1), mock.Anything).Return(&models.Version{
ID: models.ID(1),
ModelID: models.ID(1),
Model: &models.Model{
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
},
}, nil)
return svc
},
envService: func() *mocks.EnvironmentService {
svc := &mocks.EnvironmentService{}
svc.On("GetDefaultEnvironment").Return(&models.Environment{
ID: models.ID(1),
Name: "dev",
Cluster: "dev",
IsDefault: &trueBoolean,
Region: "id",
GcpProject: "dev-proj",
MaxCPU: "1",
MaxMemory: "1Gi",
}, nil)
svc.On("GetEnvironment", "dev").Return(&models.Environment{
ID: models.ID(1),
Name: "dev",
Cluster: "dev",
IsDefault: &trueBoolean,
Region: "id",
GcpProject: "dev-proj",
MaxCPU: "1",
MaxMemory: "1Gi",
}, nil)
return svc
},
endpointService: func() *mocks.EndpointsService {
svc := &mocks.EndpointsService{}
svc.On("CountEndpoints", context.Background(), mock.Anything, mock.Anything).Return(0, nil)
return svc
},
monitoringConfig: config.MonitoringConfig{},
feastCoreMock: func() *feastmocks.CoreServiceClient {
return &feastmocks.CoreServiceClient{}
},
expected: &Response{
code: http.StatusBadRequest,
data: Error{Message: "Request validation failed: model observability is not supported for this model"},
},
},
{
desc: "Should return 400 if UPI is not supported",
vars: map[string]string{
Expand Down Expand Up @@ -3909,14 +4009,15 @@ func TestUpdateEndpoint(t *testing.T) {
modelService: func() *mocks.ModelsService {
svc := &mocks.ModelsService{}
svc.On("FindByID", context.Background(), models.ID(1)).Return(&models.Model{
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "pyfunc",
MlflowURL: "",
Endpoints: nil,
ObservabilitySupported: true,
}, nil)
return svc
},
Expand Down Expand Up @@ -4600,14 +4701,15 @@ func TestUpdateEndpoint(t *testing.T) {
modelService: func() *mocks.ModelsService {
svc := &mocks.ModelsService{}
svc.On("FindByID", context.Background(), models.ID(1)).Return(&models.Model{
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "tensorflow",
MlflowURL: "",
Endpoints: nil,
ID: models.ID(1),
Name: "model-1",
ProjectID: models.ID(1),
Project: mlp.Project{},
ExperimentID: 1,
Type: "tensorflow",
MlflowURL: "",
Endpoints: nil,
ObservabilitySupported: true,
}, nil)
return svc
},
Expand Down
3 changes: 3 additions & 0 deletions scripts/e2e/debug-e2e.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ echo "::group::Merlin log"
kubectl logs deploy/merlin -n caraml -c merlin
echo "::endgroup::"

echo "::group::Merlin MLFlow log"
kubectl logs deploy/merlin-mlflow -n caraml -c merlin-mlflow

echo "::group::Get Inference Service in merlin-e2e namespace"
kubectl get isvc -n merlin-e2e -o wide
echo "::endgroup::"
Expand Down

0 comments on commit 310e8dc

Please sign in to comment.