add 95% CI on mean in metrics

openclimatefix · Dec 14, 2023 · 2414dc7 · 2414dc7
1 parent fc231d3
commit 2414dc7
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/quartz_solar_forecast/eval/metrics.py b/quartz_solar_forecast/eval/metrics.py
@@ -11,7 +11,7 @@ def metrics(results_df):
 
     """
 
-    mae = (results_df["forecast_power"] - results_df['generation_power']).abs().mean()
+    mae = (results_df["forecast_power"] - results_df['generation_power']).abs().mean().round(4)
     print(f"MAE: {mae}")
 
     # calculate metrics over the different horizons hours
@@ -20,8 +20,10 @@ def metrics(results_df):
     for horizon_hour in horizon_hours:
         # filter results_df to only include the horizon_hour
         results_df_horizon = results_df[results_df["horizon_hour"] == horizon_hour]
-        mae = (results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().mean()
-        print(f"MAE for horizon {horizon_hour}: {mae}")
+        mae = (results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().mean().round(3)
+        sem = ((results_df_horizon["forecast_power"] - results_df_horizon['generation_power']).abs().std() / len(results_df_horizon)**0.5).round(3)
+
+        print(f"MAE for horizon {horizon_hour}: {mae} +- {1.96*sem}")
 
     # TODO add more metrics using ocf_ml_metrics
 

diff --git a/quartz_solar_forecast/evaluation.py b/quartz_solar_forecast/evaluation.py
@@ -59,7 +59,8 @@ def run_eval(testset_path: str = "quartz_solar_forecast/dataset/testset.csv"):
     # Calculate and print metrics: MAE
     metrics(results_df)
 
-    # Visulisations
+    # Visualizations
+    # TODO
 
 
 # run_eval()