From f062c1d86ad8ffc6acb4ef171af96ed9257953ff Mon Sep 17 00:00:00 2001 From: Daniele Tria <36860433+dtria91@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:33:12 +0200 Subject: [PATCH] feat: add latest current and reference job status to modelout (#59) * feat: add latest current and reference job status to modelOut * fix: align with main * fix: remove unused model definition fields into the sdk --- api/app/models/model_dto.py | 30 +++++++++++++++-- api/app/services/model_service.py | 33 ++++++++----------- api/tests/services/model_service_test.py | 4 +-- .../models/model_definition.py | 2 -- sdk/tests/apis/model_test.py | 16 --------- sdk/tests/client_test.py | 10 +----- sdk/tests/models/model_definition_test.py | 8 +---- 7 files changed, 46 insertions(+), 57 deletions(-) diff --git a/api/app/models/model_dto.py b/api/app/models/model_dto.py index 60a9e441..fd4faa64 100644 --- a/api/app/models/model_dto.py +++ b/api/app/models/model_dto.py @@ -6,8 +6,11 @@ from pydantic import BaseModel, ConfigDict, model_validator from pydantic.alias_generators import to_camel +from app.db.dao.current_dataset_dao import CurrentDataset from app.db.dao.model_dao import Model +from app.db.dao.reference_dataset_dao import ReferenceDataset from app.models.inferred_schema_dto import SupportedTypes +from app.models.job_status import JobStatus from app.models.utils import is_none, is_number, is_number_or_string, is_optional_float @@ -42,6 +45,7 @@ class OutputType(BaseModel, validate_assignment=True): prediction: ColumnDefinition prediction_proba: Optional[ColumnDefinition] = None output: List[ColumnDefinition] + model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) def to_dict(self): @@ -175,6 +179,8 @@ class ModelOut(BaseModel): updated_at: str latest_reference_uuid: Optional[UUID] latest_current_uuid: Optional[UUID] + latest_reference_job_status: JobStatus + latest_current_job_status: JobStatus model_config = ConfigDict( populate_by_name=True, alias_generator=to_camel, protected_namespaces=() @@ -183,9 +189,27 @@ class ModelOut(BaseModel): @staticmethod def from_model( model: Model, - latest_reference_uuid: Optional[UUID] = None, - latest_current_uuid: Optional[UUID] = None, + latest_reference_dataset: Optional[ReferenceDataset] = None, + latest_current_dataset: Optional[CurrentDataset] = None, ): + latest_reference_uuid = ( + latest_reference_dataset.uuid if latest_reference_dataset else None + ) + latest_current_uuid = ( + latest_current_dataset.uuid if latest_current_dataset else None + ) + + latest_reference_job_status = ( + latest_reference_dataset.status + if latest_reference_dataset + else JobStatus.MISSING_REFERENCE + ) + latest_current_job_status = ( + latest_current_dataset.status + if latest_current_dataset + else JobStatus.MISSING_CURRENT + ) + return ModelOut( uuid=model.uuid, name=model.name, @@ -203,4 +227,6 @@ def from_model( updated_at=str(model.updated_at), latest_reference_uuid=latest_reference_uuid, latest_current_uuid=latest_current_uuid, + latest_reference_job_status=latest_reference_job_status, + latest_current_job_status=latest_current_job_status, ) diff --git a/api/app/services/model_service.py b/api/app/services/model_service.py index 4e7ed4e0..9dd77758 100644 --- a/api/app/services/model_service.py +++ b/api/app/services/model_service.py @@ -6,7 +6,9 @@ from app.db.dao.current_dataset_dao import CurrentDatasetDAO from app.db.dao.model_dao import ModelDAO from app.db.dao.reference_dataset_dao import ReferenceDatasetDAO +from app.db.tables.current_dataset_table import CurrentDataset from app.db.tables.model_table import Model +from app.db.tables.reference_dataset_table import ReferenceDataset from app.models.exceptions import ModelInternalError, ModelNotFoundError from app.models.model_dto import ModelIn, ModelOut from app.models.model_order import OrderType @@ -35,13 +37,13 @@ def create_model(self, model_in: ModelIn) -> ModelOut: def get_model_by_uuid(self, model_uuid: UUID) -> Optional[ModelOut]: model = self.check_and_get_model(model_uuid) - latest_reference_uuid, latest_current_uuid = self.get_latest_dataset_uuids( + latest_reference_dataset, latest_current_dataset = self.get_latest_datasets( model_uuid ) return ModelOut.from_model( model=model, - latest_reference_uuid=latest_reference_uuid, - latest_current_uuid=latest_current_uuid, + latest_reference_dataset=latest_reference_dataset, + latest_current_dataset=latest_current_dataset, ) def delete_model(self, model_uuid: UUID) -> Optional[ModelOut]: @@ -55,13 +57,13 @@ def get_all_models( models = self.model_dao.get_all() model_out_list = [] for model in models: - latest_reference_uuid, latest_current_uuid = self.get_latest_dataset_uuids( + latest_reference_dataset, latest_current_dataset = self.get_latest_datasets( model.uuid ) model_out = ModelOut.from_model( model=model, - latest_reference_uuid=latest_reference_uuid, - latest_current_uuid=latest_current_uuid, + latest_reference_dataset=latest_reference_dataset, + latest_current_dataset=latest_current_dataset, ) model_out_list.append(model_out) return model_out_list @@ -78,13 +80,13 @@ def get_all_models_paginated( _items = [] for model in models.items: - latest_reference_uuid, latest_current_uuid = self.get_latest_dataset_uuids( + latest_reference_dataset, latest_current_dataset = self.get_latest_datasets( model.uuid ) model_out = ModelOut.from_model( model=model, - latest_reference_uuid=latest_reference_uuid, - latest_current_uuid=latest_current_uuid, + latest_reference_dataset=latest_reference_dataset, + latest_current_dataset=latest_current_dataset, ) _items.append(model_out) @@ -96,9 +98,9 @@ def check_and_get_model(self, model_uuid: UUID) -> Model: raise ModelNotFoundError(f'Model {model_uuid} not found') return model - def get_latest_dataset_uuids( + def get_latest_datasets( self, model_uuid: UUID - ) -> (Optional[UUID], Optional[UUID]): + ) -> (Optional[ReferenceDataset], Optional[CurrentDataset]): latest_reference_dataset = ( self.rd_dao.get_latest_reference_dataset_by_model_uuid(model_uuid) ) @@ -106,11 +108,4 @@ def get_latest_dataset_uuids( model_uuid ) - latest_reference_uuid = ( - latest_reference_dataset.uuid if latest_reference_dataset else None - ) - latest_current_uuid = ( - latest_current_dataset.uuid if latest_current_dataset else None - ) - - return latest_reference_uuid, latest_current_uuid + return latest_reference_dataset, latest_current_dataset diff --git a/api/tests/services/model_service_test.py b/api/tests/services/model_service_test.py index 1564c195..02e6a669 100644 --- a/api/tests/services/model_service_test.py +++ b/api/tests/services/model_service_test.py @@ -55,8 +55,8 @@ def test_get_model_by_uuid_ok(self): assert res == ModelOut.from_model( model=model, - latest_reference_uuid=reference_dataset.uuid, - latest_current_uuid=current_dataset.uuid, + latest_reference_dataset=reference_dataset, + latest_current_dataset=current_dataset, ) def test_get_model_by_uuid_not_found(self): diff --git a/sdk/radicalbit_platform_sdk/models/model_definition.py b/sdk/radicalbit_platform_sdk/models/model_definition.py index 3fb6b4ba..eb47445c 100644 --- a/sdk/radicalbit_platform_sdk/models/model_definition.py +++ b/sdk/radicalbit_platform_sdk/models/model_definition.py @@ -68,7 +68,5 @@ class ModelDefinition(BaseModelDefinition): uuid: uuid_lib.UUID = Field(default_factory=lambda: uuid_lib.uuid4()) created_at: str = Field(alias='createdAt') updated_at: str = Field(alias='updatedAt') - latest_reference_uuid: Optional[uuid_lib.UUID] = Field(alias='latestReferenceUuid') - latest_current_uuid: Optional[uuid_lib.UUID] = Field(alias='latestCurrentUuid') model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) diff --git a/sdk/tests/apis/model_test.py b/sdk/tests/apis/model_test.py index a33adc83..fd93c8b2 100644 --- a/sdk/tests/apis/model_test.py +++ b/sdk/tests/apis/model_test.py @@ -43,8 +43,6 @@ def test_delete_model(self): timestamp=column_def, created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) responses.add( @@ -82,8 +80,6 @@ def test_load_reference_dataset_without_object_name(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) response = ReferenceFileUpload( @@ -129,8 +125,6 @@ def test_load_reference_dataset_with_different_separator(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) response = ReferenceFileUpload( @@ -176,8 +170,6 @@ def test_load_reference_dataset_with_object_name(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) response = ReferenceFileUpload( @@ -214,8 +206,6 @@ def test_load_reference_dataset_wrong_headers(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=None, - latest_current_uuid=None, ), ) with pytest.raises(ClientError): @@ -249,8 +239,6 @@ def test_load_current_dataset_without_object_name(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) response = CurrentFileUpload( @@ -302,8 +290,6 @@ def test_load_current_dataset_with_object_name(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=uuid.uuid4(), - latest_current_uuid=uuid.uuid4(), ), ) response = CurrentFileUpload( @@ -346,8 +332,6 @@ def test_load_current_dataset_wrong_headers(self): timestamp=ColumnDefinition(name='created_at', type='str'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=None, - latest_current_uuid=None, ), ) with pytest.raises(ClientError): diff --git a/sdk/tests/client_test.py b/sdk/tests/client_test.py index 1cf8fb24..c07d28be 100644 --- a/sdk/tests/client_test.py +++ b/sdk/tests/client_test.py @@ -39,8 +39,6 @@ def test_get_model(self): timestamp_name = 'when' timestamp_type = 'str' ts = str(time.time()) - latest_reference_uuid = uuid.uuid4() - latest_current_uuid = uuid.uuid4() json_string = f"""{{ "uuid": "{str(model_id)}", "name": "{name}", @@ -77,9 +75,7 @@ def test_get_model(self): "algorithm": "{algorithm}", "frameworks": "{frameworks}", "createdAt": "{ts}", - "updatedAt": "{ts}", - "latestReferenceUuid": "{str(latest_reference_uuid)}", - "latestCurrentUuid": "{str(latest_current_uuid)}" + "updatedAt": "{ts}" }}""" responses.add( method=responses.GET, @@ -156,8 +152,6 @@ def test_create_model(self): timestamp=model.timestamp, created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=None, - latest_current_uuid=None, ) responses.add( method=responses.POST, @@ -199,8 +193,6 @@ def test_search_models(self): timestamp=ColumnDefinition(name='tst_column', type='string'), created_at=str(time.time()), updated_at=str(time.time()), - latest_reference_uuid=None, - latest_current_uuid=None, ) responses.add( diff --git a/sdk/tests/models/model_definition_test.py b/sdk/tests/models/model_definition_test.py index 7d904ff0..c7a9a48b 100644 --- a/sdk/tests/models/model_definition_test.py +++ b/sdk/tests/models/model_definition_test.py @@ -30,8 +30,6 @@ def test_model_definition_from_json(self): timestamp_name = 'when' timestamp_type = 'str' ts = str(time.time()) - latest_reference_uuid = uuid.uuid4() - latest_current_uuid = uuid.uuid4() json_string = f"""{{ "uuid": "{str(id)}", "name": "{name}", @@ -68,9 +66,7 @@ def test_model_definition_from_json(self): "algorithm": "{algorithm}", "frameworks": "{frameworks}", "createdAt": "{ts}", - "updatedAt": "{ts}", - "latestReferenceUuid": "{str(latest_reference_uuid)}", - "latestCurrentUuid": "{str(latest_current_uuid)}" + "updatedAt": "{ts}" }}""" model_definition = ModelDefinition.model_validate(json.loads(json_string)) assert model_definition.uuid == id @@ -83,8 +79,6 @@ def test_model_definition_from_json(self): assert model_definition.frameworks == frameworks assert model_definition.created_at == ts assert model_definition.updated_at == ts - assert model_definition.latest_reference_uuid == latest_reference_uuid - assert model_definition.latest_current_uuid == latest_current_uuid assert len(model_definition.features) == 1 assert model_definition.features[0].name == feature_name assert model_definition.features[0].type == feature_type