From af7c03ea5019de2a34b376e7e5b86e9813464804 Mon Sep 17 00:00:00 2001 From: Stefano Zamboni Date: Mon, 28 Oct 2024 14:24:34 +0100 Subject: [PATCH 1/2] fix: fixed get percentages with loading/deleted models --- api/app/db/dao/model_dao.py | 3 +- api/app/routes/metrics_route.py | 2 +- api/app/services/metrics_service.py | 10 +++--- api/app/services/model_service.py | 47 +++++++++++++++-------------- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/api/app/db/dao/model_dao.py b/api/app/db/dao/model_dao.py index 653659a8..bf1c56d0 100644 --- a/api/app/db/dao/model_dao.py +++ b/api/app/db/dao/model_dao.py @@ -80,10 +80,11 @@ def get_last_n_percentages(self, n_models=None): (CurrentDataset.model_uuid == subq.c.model_uuid) & (CurrentDataset.date == subq.c.maxdate), ) - .join( + .outerjoin( CurrentDatasetMetrics, CurrentDatasetMetrics.current_uuid == CurrentDataset.uuid, ) + .filter(Model.deleted.is_(False)) .order_by(Model.updated_at.desc()) ) if n_models: diff --git a/api/app/routes/metrics_route.py b/api/app/routes/metrics_route.py index 869d8302..c906d50c 100644 --- a/api/app/routes/metrics_route.py +++ b/api/app/routes/metrics_route.py @@ -145,7 +145,7 @@ def get_latest_current_percentages_by_model_by_uuid(model_uuid: UUID): response_model=PercentagesDTO, ) def get_current_percentages_by_model_by_uuid( - model_uuid: UUID, current_uuid: UUID + model_uuid: UUID, current_uuid: UUID ): return metrics_service.get_current_percentages_by_model_by_uuid( model_uuid, current_uuid diff --git a/api/app/services/metrics_service.py b/api/app/services/metrics_service.py index d0c585ba..973a4f06 100644 --- a/api/app/services/metrics_service.py +++ b/api/app/services/metrics_service.py @@ -108,7 +108,7 @@ def get_current_data_quality_by_model_by_uuid( def get_current_percentages_by_model_by_uuid( self, model_uuid: UUID, current_uuid: Optional[UUID] ) -> PercentagesDTO: - """Retrieve current data quality for a model by its UUID and an optional current dataset UUID.""" + """Retrieve current percentages for a model by its UUID and an optional current dataset UUID.""" return self._get_percentages_by_model_uuid( model_uuid=model_uuid, dataset_and_metrics_getter=lambda uuid: self.check_and_get_current_dataset_and_metrics( @@ -348,10 +348,10 @@ def _create_data_quality_dto( @staticmethod def _create_percentages_dto( - model_type: ModelType, - dataset: Optional[ReferenceDataset | CurrentDataset], - metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics], - missing_status, + model_type: ModelType, + dataset: Optional[ReferenceDataset | CurrentDataset], + metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics], + missing_status, ) -> PercentagesDTO: """Create a PercentagesDTO from the provided dataset and metrics.""" if not dataset: diff --git a/api/app/services/model_service.py b/api/app/services/model_service.py index fc4a5376..710fcf4c 100644 --- a/api/app/services/model_service.py +++ b/api/app/services/model_service.py @@ -96,28 +96,29 @@ def get_summarized_percentages(self) -> TotPercentagesDTO: mq, mq_c = 0, 0 dr, dr_c = 0, 0 for _, metrics in models: - dq = dq + ( - metrics.percentages['data_quality']['value'] - if metrics.percentages['data_quality']['value'] >= 0 - else 0 - ) - dq_c = dq_c + ( - 1 if metrics.percentages['data_quality']['value'] >= 0 else 0 - ) - mq = mq + ( - metrics.percentages['model_quality']['value'] - if metrics.percentages['model_quality']['value'] >= 0 - else 0 - ) - mq_c = mq_c + ( - 1 if metrics.percentages['model_quality']['value'] >= 0 else 0 - ) - dr = dr + ( - metrics.percentages['drift']['value'] - if metrics.percentages['drift']['value'] >= 0 - else 0 - ) - dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0) + if metrics: + dq = dq + ( + metrics.percentages['data_quality']['value'] + if metrics.percentages['data_quality']['value'] >= 0 + else 0 + ) + dq_c = dq_c + ( + 1 if metrics.percentages['data_quality']['value'] >= 0 else 0 + ) + mq = mq + ( + metrics.percentages['model_quality']['value'] + if metrics.percentages['model_quality']['value'] >= 0 + else 0 + ) + mq_c = mq_c + ( + 1 if metrics.percentages['model_quality']['value'] >= 0 else 0 + ) + dr = dr + ( + metrics.percentages['drift']['value'] + if metrics.percentages['drift']['value'] >= 0 + else 0 + ) + dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0) return TotPercentagesDTO.from_dict( { 'data_quality': dq / dq_c if dq_c > 0 else 0, @@ -178,7 +179,7 @@ def get_last_n_models_percentages(self, n_models) -> List[ModelOut]: model=model, latest_reference_dataset=latest_reference_dataset, latest_current_dataset=latest_current_dataset, - percentages=metrics.percentages, + percentages=metrics.percentages if metrics else None, ) model_out_list_tmp.append(model_out) return model_out_list_tmp From 2505c771925383f2c63bc3b4ff3fd91ff93088a6 Mon Sep 17 00:00:00 2001 From: Stefano Zamboni Date: Mon, 28 Oct 2024 14:41:53 +0100 Subject: [PATCH 2/2] fix: fixed alerts when importing --- api/app/services/model_service.py | 55 ++++++++++++++++--------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/api/app/services/model_service.py b/api/app/services/model_service.py index 710fcf4c..5793e5c5 100644 --- a/api/app/services/model_service.py +++ b/api/app/services/model_service.py @@ -137,34 +137,35 @@ def get_last_n_alerts(self, n_alerts) -> List[AlertDTO]: latest_reference_dataset, latest_current_dataset = self.get_latest_datasets( model.uuid ) - for perc in ['data_quality', 'model_quality', 'drift']: - if count_alerts == n_alerts: - return res - if 0 <= metrics.percentages[perc]['value'] < 1: - res.append( - AlertDTO.from_dict( - { - 'model_uuid': model.uuid, - 'reference_uuid': latest_reference_dataset.uuid - if latest_reference_dataset - else None, - 'current_uuid': latest_current_dataset.uuid - if latest_current_dataset - else None, - 'anomaly_type': AnomalyType[perc.upper()], - 'anomaly_features': [ - x['feature_name'] - for x in sorted( - metrics.percentages[perc]['details'], - key=lambda e: e['score'], - reverse=True, - ) - if x['score'] > 0 - ], - } + if metrics: + for perc in ['data_quality', 'model_quality', 'drift']: + if count_alerts == n_alerts: + return res + if 0 <= metrics.percentages[perc]['value'] < 1: + res.append( + AlertDTO.from_dict( + { + 'model_uuid': model.uuid, + 'reference_uuid': latest_reference_dataset.uuid + if latest_reference_dataset + else None, + 'current_uuid': latest_current_dataset.uuid + if latest_current_dataset + else None, + 'anomaly_type': AnomalyType[perc.upper()], + 'anomaly_features': [ + x['feature_name'] + for x in sorted( + metrics.percentages[perc]['details'], + key=lambda e: e['score'], + reverse=True, + ) + if x['score'] > 0 + ], + } + ) ) - ) - count_alerts += 1 + count_alerts += 1 return res