Skip to content

Commit

Permalink
fix: fix get percentages models (#185)
Browse files Browse the repository at this point in the history 
* fix: fixed get percentages with loading/deleted models

* fix: fixed alerts when importing
  • Loading branch information
@SteZamboni
SteZamboni authored Oct 28, 2024
1 parent 82eea2b commit b422cb3
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 57 deletions.
3 changes: 2 additions & 1 deletion api/app/db/dao/model_dao.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ def get_last_n_percentages(self, n_models=None):
(CurrentDataset.model_uuid == subq.c.model_uuid)
& (CurrentDataset.date == subq.c.maxdate),
)
.join(
.outerjoin(
CurrentDatasetMetrics,
CurrentDatasetMetrics.current_uuid == CurrentDataset.uuid,
)
.filter(Model.deleted.is_(False))
.order_by(Model.updated_at.desc())
)
if n_models:
Expand Down
2 changes: 1 addition & 1 deletion api/app/routes/metrics_route.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ def get_latest_current_percentages_by_model_by_uuid(model_uuid: UUID):
response_model=PercentagesDTO,
)
def get_current_percentages_by_model_by_uuid(
model_uuid: UUID, current_uuid: UUID
model_uuid: UUID, current_uuid: UUID
):
return metrics_service.get_current_percentages_by_model_by_uuid(
model_uuid, current_uuid
Expand Down
10 changes: 5 additions & 5 deletions api/app/services/metrics_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def get_current_data_quality_by_model_by_uuid(
def get_current_percentages_by_model_by_uuid(
self, model_uuid: UUID, current_uuid: Optional[UUID]
) -> PercentagesDTO:
"""Retrieve current data quality for a model by its UUID and an optional current dataset UUID."""
"""Retrieve current percentages for a model by its UUID and an optional current dataset UUID."""
return self._get_percentages_by_model_uuid(
model_uuid=model_uuid,
dataset_and_metrics_getter=lambda uuid: self.check_and_get_current_dataset_and_metrics(
Expand Down Expand Up @@ -348,10 +348,10 @@ def _create_data_quality_dto(

@staticmethod
def _create_percentages_dto(
model_type: ModelType,
dataset: Optional[ReferenceDataset | CurrentDataset],
metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics],
missing_status,
model_type: ModelType,
dataset: Optional[ReferenceDataset | CurrentDataset],
metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics],
missing_status,
) -> PercentagesDTO:
"""Create a PercentagesDTO from the provided dataset and metrics."""
if not dataset:
Expand Down
102 changes: 52 additions & 50 deletions api/app/services/model_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,28 +96,29 @@ def get_summarized_percentages(self) -> TotPercentagesDTO:
mq, mq_c = 0, 0
dr, dr_c = 0, 0
for _, metrics in models:
dq = dq + (
metrics.percentages['data_quality']['value']
if metrics.percentages['data_quality']['value'] >= 0
else 0
)
dq_c = dq_c + (
1 if metrics.percentages['data_quality']['value'] >= 0 else 0
)
mq = mq + (
metrics.percentages['model_quality']['value']
if metrics.percentages['model_quality']['value'] >= 0
else 0
)
mq_c = mq_c + (
1 if metrics.percentages['model_quality']['value'] >= 0 else 0
)
dr = dr + (
metrics.percentages['drift']['value']
if metrics.percentages['drift']['value'] >= 0
else 0
)
dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0)
if metrics:
dq = dq + (
metrics.percentages['data_quality']['value']
if metrics.percentages['data_quality']['value'] >= 0
else 0
)
dq_c = dq_c + (
1 if metrics.percentages['data_quality']['value'] >= 0 else 0
)
mq = mq + (
metrics.percentages['model_quality']['value']
if metrics.percentages['model_quality']['value'] >= 0
else 0
)
mq_c = mq_c + (
1 if metrics.percentages['model_quality']['value'] >= 0 else 0
)
dr = dr + (
metrics.percentages['drift']['value']
if metrics.percentages['drift']['value'] >= 0
else 0
)
dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0)
return TotPercentagesDTO.from_dict(
{
'data_quality': dq / dq_c if dq_c > 0 else 0,
Expand All @@ -136,34 +137,35 @@ def get_last_n_alerts(self, n_alerts) -> List[AlertDTO]:
latest_reference_dataset, latest_current_dataset = self.get_latest_datasets(
model.uuid
)
for perc in ['data_quality', 'model_quality', 'drift']:
if count_alerts == n_alerts:
return res
if 0 <= metrics.percentages[perc]['value'] < 1:
res.append(
AlertDTO.from_dict(
{
'model_uuid': model.uuid,
'reference_uuid': latest_reference_dataset.uuid
if latest_reference_dataset
else None,
'current_uuid': latest_current_dataset.uuid
if latest_current_dataset
else None,
'anomaly_type': AnomalyType[perc.upper()],
'anomaly_features': [
x['feature_name']
for x in sorted(
metrics.percentages[perc]['details'],
key=lambda e: e['score'],
reverse=True,
)
if x['score'] > 0
],
}
if metrics:
for perc in ['data_quality', 'model_quality', 'drift']:
if count_alerts == n_alerts:
return res
if 0 <= metrics.percentages[perc]['value'] < 1:
res.append(
AlertDTO.from_dict(
{
'model_uuid': model.uuid,
'reference_uuid': latest_reference_dataset.uuid
if latest_reference_dataset
else None,
'current_uuid': latest_current_dataset.uuid
if latest_current_dataset
else None,
'anomaly_type': AnomalyType[perc.upper()],
'anomaly_features': [
x['feature_name']
for x in sorted(
metrics.percentages[perc]['details'],
key=lambda e: e['score'],
reverse=True,
)
if x['score'] > 0
],
}
)
)
)
count_alerts += 1
count_alerts += 1

return res

Expand All @@ -178,7 +180,7 @@ def get_last_n_models_percentages(self, n_models) -> List[ModelOut]:
model=model,
latest_reference_dataset=latest_reference_dataset,
latest_current_dataset=latest_current_dataset,
percentages=metrics.percentages,
percentages=metrics.percentages if metrics else None,
)
model_out_list_tmp.append(model_out)
return model_out_list_tmp
Expand Down

0 comments on commit b422cb3

Please sign in to comment.