Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix get percentages models #185

Merged
merged 2 commits into from
Oct 28, 2024
Merged

fix: fix get percentages models #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af7c03e
fix: fixed get percentages with loading/deleted models
SteZamboni Oct 28, 2024
2505c77
fix: fixed alerts when importing
SteZamboni Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/app/db/dao/model_dao.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ def get_last_n_percentages(self, n_models=None):
(CurrentDataset.model_uuid == subq.c.model_uuid)
& (CurrentDataset.date == subq.c.maxdate),
)
.join(
.outerjoin(
CurrentDatasetMetrics,
CurrentDatasetMetrics.current_uuid == CurrentDataset.uuid,
)
.filter(Model.deleted.is_(False))
.order_by(Model.updated_at.desc())
)
if n_models:
Expand Down
2 changes: 1 addition & 1 deletion api/app/routes/metrics_route.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ def get_latest_current_percentages_by_model_by_uuid(model_uuid: UUID):
response_model=PercentagesDTO,
)
def get_current_percentages_by_model_by_uuid(
model_uuid: UUID, current_uuid: UUID
model_uuid: UUID, current_uuid: UUID
):
return metrics_service.get_current_percentages_by_model_by_uuid(
model_uuid, current_uuid
Expand Down
10 changes: 5 additions & 5 deletions api/app/services/metrics_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def get_current_data_quality_by_model_by_uuid(
def get_current_percentages_by_model_by_uuid(
self, model_uuid: UUID, current_uuid: Optional[UUID]
) -> PercentagesDTO:
"""Retrieve current data quality for a model by its UUID and an optional current dataset UUID."""
"""Retrieve current percentages for a model by its UUID and an optional current dataset UUID."""
return self._get_percentages_by_model_uuid(
model_uuid=model_uuid,
dataset_and_metrics_getter=lambda uuid: self.check_and_get_current_dataset_and_metrics(
Expand Down Expand Up @@ -348,10 +348,10 @@ def _create_data_quality_dto(

@staticmethod
def _create_percentages_dto(
model_type: ModelType,
dataset: Optional[ReferenceDataset | CurrentDataset],
metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics],
missing_status,
model_type: ModelType,
dataset: Optional[ReferenceDataset | CurrentDataset],
metrics: Optional[ReferenceDatasetMetrics | CurrentDatasetMetrics],
missing_status,
) -> PercentagesDTO:
"""Create a PercentagesDTO from the provided dataset and metrics."""
if not dataset:
Expand Down
102 changes: 52 additions & 50 deletions api/app/services/model_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,28 +96,29 @@ def get_summarized_percentages(self) -> TotPercentagesDTO:
mq, mq_c = 0, 0
dr, dr_c = 0, 0
for _, metrics in models:
dq = dq + (
metrics.percentages['data_quality']['value']
if metrics.percentages['data_quality']['value'] >= 0
else 0
)
dq_c = dq_c + (
1 if metrics.percentages['data_quality']['value'] >= 0 else 0
)
mq = mq + (
metrics.percentages['model_quality']['value']
if metrics.percentages['model_quality']['value'] >= 0
else 0
)
mq_c = mq_c + (
1 if metrics.percentages['model_quality']['value'] >= 0 else 0
)
dr = dr + (
metrics.percentages['drift']['value']
if metrics.percentages['drift']['value'] >= 0
else 0
)
dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0)
if metrics:
dq = dq + (
metrics.percentages['data_quality']['value']
if metrics.percentages['data_quality']['value'] >= 0
else 0
)
dq_c = dq_c + (
1 if metrics.percentages['data_quality']['value'] >= 0 else 0
)
mq = mq + (
metrics.percentages['model_quality']['value']
if metrics.percentages['model_quality']['value'] >= 0
else 0
)
mq_c = mq_c + (
1 if metrics.percentages['model_quality']['value'] >= 0 else 0
)
dr = dr + (
metrics.percentages['drift']['value']
if metrics.percentages['drift']['value'] >= 0
else 0
)
dr_c = dr_c + (1 if metrics.percentages['drift']['value'] >= 0 else 0)
return TotPercentagesDTO.from_dict(
{
'data_quality': dq / dq_c if dq_c > 0 else 0,
Expand All @@ -136,34 +137,35 @@ def get_last_n_alerts(self, n_alerts) -> List[AlertDTO]:
latest_reference_dataset, latest_current_dataset = self.get_latest_datasets(
model.uuid
)
for perc in ['data_quality', 'model_quality', 'drift']:
if count_alerts == n_alerts:
return res
if 0 <= metrics.percentages[perc]['value'] < 1:
res.append(
AlertDTO.from_dict(
{
'model_uuid': model.uuid,
'reference_uuid': latest_reference_dataset.uuid
if latest_reference_dataset
else None,
'current_uuid': latest_current_dataset.uuid
if latest_current_dataset
else None,
'anomaly_type': AnomalyType[perc.upper()],
'anomaly_features': [
x['feature_name']
for x in sorted(
metrics.percentages[perc]['details'],
key=lambda e: e['score'],
reverse=True,
)
if x['score'] > 0
],
}
if metrics:
for perc in ['data_quality', 'model_quality', 'drift']:
if count_alerts == n_alerts:
return res
if 0 <= metrics.percentages[perc]['value'] < 1:
res.append(
AlertDTO.from_dict(
{
'model_uuid': model.uuid,
'reference_uuid': latest_reference_dataset.uuid
if latest_reference_dataset
else None,
'current_uuid': latest_current_dataset.uuid
if latest_current_dataset
else None,
'anomaly_type': AnomalyType[perc.upper()],
'anomaly_features': [
x['feature_name']
for x in sorted(
metrics.percentages[perc]['details'],
key=lambda e: e['score'],
reverse=True,
)
if x['score'] > 0
],
}
)
)
)
count_alerts += 1
count_alerts += 1

return res

Expand All @@ -178,7 +180,7 @@ def get_last_n_models_percentages(self, n_models) -> List[ModelOut]:
model=model,
latest_reference_dataset=latest_reference_dataset,
latest_current_dataset=latest_current_dataset,
percentages=metrics.percentages,
percentages=metrics.percentages if metrics else None,
)
model_out_list_tmp.append(model_out)
return model_out_list_tmp
Expand Down