diff --git a/quartz_solar_forecast/eval/metrics.py b/quartz_solar_forecast/eval/metrics.py index 9e96a76c..7a2742ea 100644 --- a/quartz_solar_forecast/eval/metrics.py +++ b/quartz_solar_forecast/eval/metrics.py @@ -11,7 +11,7 @@ def metrics(results_df): """ - mae = (results_df["forecast_power"] - results_df['generation_power']).abs().mean() + mae = (results_df["forecast_power"] - results_df['generation_power']).abs().mean().round(4) print(f"MAE: {mae}") # calculate metrics over the different horizons hours @@ -20,8 +20,10 @@ def metrics(results_df): for horizon_hour in horizon_hours: # filter results_df to only include the horizon_hour results_df_horizon = results_df[results_df["horizon_hour"] == horizon_hour] - mae = (results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().mean() - print(f"MAE for horizon {horizon_hour}: {mae}") + mae = (results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().mean().round(3) + sem = ((results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().std() / len(results_df_horizon)**0.5).round(3) + + print(f"MAE for horizon {horizon_hour}: {mae} +- {1.96*sem}") # TODO add more metrics using ocf_ml_metrics diff --git a/quartz_solar_forecast/evaluation.py b/quartz_solar_forecast/evaluation.py index 9918a73b..e353d7b9 100644 --- a/quartz_solar_forecast/evaluation.py +++ b/quartz_solar_forecast/evaluation.py @@ -59,7 +59,8 @@ def run_eval(testset_path: str = "quartz_solar_forecast/dataset/testset.csv"): # Calculate and print metrics: MAE metrics(results_df) - # Visulisations + # Visualizations + # TODO # run_eval()