From 0c23083988fda083c040c1a7731561edd24504f3 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Nov 2024 15:27:19 +0000 Subject: [PATCH 1/5] add smooth blocks options --- india_forecast_app/app.py | 11 ++++++++++- india_forecast_app/models/all_models.yaml | 2 ++ india_forecast_app/models/dummy.py | 3 +++ india_forecast_app/models/pvnet/model.py | 12 ++++++++++++ india_forecast_app/models/pydantic_models.py | 16 ++++++++++++---- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/india_forecast_app/app.py b/india_forecast_app/app.py index 9889e4f..324032c 100644 --- a/india_forecast_app/app.py +++ b/india_forecast_app/app.py @@ -149,6 +149,7 @@ def get_model( hf_repo: str, hf_version: str, name: str, + smooth_blocks: Optional[int] = 0, ) -> PVNetModel: """ Instantiates and returns the forecast model ready for running inference @@ -160,6 +161,7 @@ def get_model( hf_repo: ID of the ML model used for the forecast hf_version: Version of the ML model used for the forecast name: Name of the ML model used for the forecast + smooth_blocks: Number of blocks to smooth the forecast by Returns: A forecasting model @@ -169,7 +171,13 @@ def get_model( model_cls = PVNetModel model = model_cls( - asset_type, timestamp, generation_data, hf_repo=hf_repo, hf_version=hf_version, name=name + asset_type, + timestamp, + generation_data, + hf_repo=hf_repo, + hf_version=hf_version, + name=name, + smooth_blocks=smooth_blocks, ) return model @@ -363,6 +371,7 @@ def app_run(timestamp: dt.datetime | None, write_to_db: bool = False, log_level: hf_repo=model_config.id, hf_version=model_config.version, name=model_config.name, + smooth_blocks=model_config.smooth_blocks, ) ml_model.site_uuid = site.site_uuid diff --git a/india_forecast_app/models/all_models.yaml b/india_forecast_app/models/all_models.yaml index 053224c..642ab1b 100644 --- a/india_forecast_app/models/all_models.yaml +++ b/india_forecast_app/models/all_models.yaml @@ -13,6 +13,7 @@ models: version: 546baded3d4216736d8ee8d6798d47235bd72b08 client: ruvnl asset_type: wind + smooth_blocks: 5 - name: windnet_india_mo_v2 type: pvnet id: openclimatefix/windnet_india @@ -25,6 +26,7 @@ models: version: 990bed9ad1dbb10515f830c181609b10e72c975e client: ruvnl asset_type: wind + smooth_blocks: 5 # RU client solar - name: pvnet_india type: pvnet diff --git a/india_forecast_app/models/dummy.py b/india_forecast_app/models/dummy.py index 704cd84..4425d9c 100644 --- a/india_forecast_app/models/dummy.py +++ b/india_forecast_app/models/dummy.py @@ -6,6 +6,8 @@ import math import random +from typing import Optional + import pandas as pd import pytz @@ -28,6 +30,7 @@ def __init__( hf_version: str = None, hf_repo: str = None, name: str = None, + smooth_blocks: Optional[int] = 0, ): """Initializer for the model""" self.asset_type = asset_type diff --git a/india_forecast_app/models/pvnet/model.py b/india_forecast_app/models/pvnet/model.py index 086ecb0..b808510 100644 --- a/india_forecast_app/models/pvnet/model.py +++ b/india_forecast_app/models/pvnet/model.py @@ -7,6 +7,7 @@ import os import shutil import tempfile +from typing import Optional import numpy as np import pandas as pd @@ -63,6 +64,7 @@ def __init__( hf_repo: str, hf_version: str, name: str, + smooth_blocks: Optional[int] = 0, ): """Initializer for the model""" @@ -72,6 +74,7 @@ def __init__( self.name = name self.site_uuid = None self.t0 = timestamp + self.smooth_blocks = smooth_blocks log.info(f"Model initialised at t0={self.t0}") self.client = os.getenv("CLIENT_NAME", "ruvnl") @@ -189,6 +192,15 @@ def predict(self, site_id: str, timestamp: dt.datetime): values_df["forecast_power_kw"].rolling(4, min_periods=1).mean().astype(int) ) + if self.smooth_blocks: + log.info(f"Smoothing the forecast with {self.smooth_blocks} blocks") + values_df["forecast_power_kw"] = ( + values_df["forecast_power_kw"] + .rolling(window=self.smooth_blocks, min_periods=1, center=True) + .mean() + .astype(int) + ) + # remove any negative values values_df["forecast_power_kw"] = values_df["forecast_power_kw"].clip(lower=0.0) diff --git a/india_forecast_app/models/pydantic_models.py b/india_forecast_app/models/pydantic_models.py index 1de3a50..eb95f7a 100644 --- a/india_forecast_app/models/pydantic_models.py +++ b/india_forecast_app/models/pydantic_models.py @@ -32,14 +32,22 @@ class Model(BaseModel): 60, title="Average Minutes", description="The number of minutes that results are average over when " - "calculating adjuster values. " - "For solar site with regular data, 15 should be used. " - "For wind sites, 60 minutes should be used.", + "calculating adjuster values. " + "For solar site with regular data, 15 should be used. " + "For wind sites, 60 minutes should be used.", + ) + smooth_blocks: int = Field( + 0, + title="Smooth Blocks", + description="The number of blocks to smooth the forecast by. " + "We use a central smoothing and each block is 15mins, " + "so 5 would be smoothing from 30s before and afterwards", ) class Models(BaseModel): - """ A group of ml models """ + """A group of ml models""" + models: List[Model] = Field( ..., title="Models", description="A list of models to use for the forecast" ) From 28ce1b7d20240c2177e177e03e554bf0b338fdfb Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Nov 2024 15:30:56 +0000 Subject: [PATCH 2/5] lint --- india_forecast_app/models/dummy.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/india_forecast_app/models/dummy.py b/india_forecast_app/models/dummy.py index 4425d9c..830d456 100644 --- a/india_forecast_app/models/dummy.py +++ b/india_forecast_app/models/dummy.py @@ -3,11 +3,10 @@ """ import datetime as dt -import math import random - from typing import Optional +import math import pandas as pd import pytz @@ -23,14 +22,14 @@ def version(self): return "0.0.0" def __init__( - self, - asset_type: str, - timestamp: dt.datetime, - generation_data: dict[str, pd.DataFrame] = None, - hf_version: str = None, - hf_repo: str = None, - name: str = None, - smooth_blocks: Optional[int] = 0, + self, + asset_type: str, + timestamp: dt.datetime, + generation_data: dict[str, pd.DataFrame] = None, + hf_version: str = None, + hf_repo: str = None, + name: str = None, + smooth_blocks: Optional[int] = 0, ): """Initializer for the model""" self.asset_type = asset_type From 65f0296aa0c3f80fcdae00754a0674e05e88ed31 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Nov 2024 15:35:18 +0000 Subject: [PATCH 3/5] isort --- india_forecast_app/models/dummy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/india_forecast_app/models/dummy.py b/india_forecast_app/models/dummy.py index 830d456..39d22c6 100644 --- a/india_forecast_app/models/dummy.py +++ b/india_forecast_app/models/dummy.py @@ -3,10 +3,10 @@ """ import datetime as dt +import math import random from typing import Optional -import math import pandas as pd import pytz From 4c7b3d24ba3caa99121f0cc856fc73f5cc07dd91 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 26 Nov 2024 16:09:01 +0000 Subject: [PATCH 4/5] smooth --- india_forecast_app/models/all_models.yaml | 6 ++++-- india_forecast_app/models/pvnet/model.py | 8 -------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/india_forecast_app/models/all_models.yaml b/india_forecast_app/models/all_models.yaml index 642ab1b..bbd85c1 100644 --- a/india_forecast_app/models/all_models.yaml +++ b/india_forecast_app/models/all_models.yaml @@ -7,26 +7,28 @@ models: version: ae07c15de064e1d03cf4bc02618b65c6d5b17e8e client: ruvnl asset_type: wind + smooth_blocks: 4 - name: windnet_india_mo type: pvnet id: openclimatefix/windnet_india version: 546baded3d4216736d8ee8d6798d47235bd72b08 client: ruvnl asset_type: wind - smooth_blocks: 5 + smooth_blocks: 7 - name: windnet_india_mo_v2 type: pvnet id: openclimatefix/windnet_india version: 165267d34500cf0c881ed70d9318421f4e0d10f1 client: ruvnl asset_type: wind + smooth_blocks: 4 - name: windnet_india_mo_v3 type: pvnet id: openclimatefix/windnet_india version: 990bed9ad1dbb10515f830c181609b10e72c975e client: ruvnl asset_type: wind - smooth_blocks: 5 + smooth_blocks: 7 # RU client solar - name: pvnet_india type: pvnet diff --git a/india_forecast_app/models/pvnet/model.py b/india_forecast_app/models/pvnet/model.py index b808510..b432b29 100644 --- a/india_forecast_app/models/pvnet/model.py +++ b/india_forecast_app/models/pvnet/model.py @@ -184,14 +184,6 @@ def predict(self, site_id: str, timestamp: dt.datetime): ) * smooth_values[final_gen_index + idx] log.debug(f"New values are {values_df['forecast_power_kw']}") - if self.asset_type == "wind": - # Smooth with a 1 hour rolling window - # Only smooth the wind else we introduce too much of a lag in the solar - # going up and down throughout the day - values_df["forecast_power_kw"] = ( - values_df["forecast_power_kw"].rolling(4, min_periods=1).mean().astype(int) - ) - if self.smooth_blocks: log.info(f"Smoothing the forecast with {self.smooth_blocks} blocks") values_df["forecast_power_kw"] = ( From cbfadc85348752cd1ee798575f3e8778a4a87c1d Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Wed, 27 Nov 2024 13:09:09 +0000 Subject: [PATCH 5/5] make sure forecst values are ints --- india_forecast_app/models/pvnet/model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/india_forecast_app/models/pvnet/model.py b/india_forecast_app/models/pvnet/model.py index b432b29..fb416a1 100644 --- a/india_forecast_app/models/pvnet/model.py +++ b/india_forecast_app/models/pvnet/model.py @@ -184,15 +184,17 @@ def predict(self, site_id: str, timestamp: dt.datetime): ) * smooth_values[final_gen_index + idx] log.debug(f"New values are {values_df['forecast_power_kw']}") - if self.smooth_blocks: + if self.smooth_blocks > 0: log.info(f"Smoothing the forecast with {self.smooth_blocks} blocks") values_df["forecast_power_kw"] = ( values_df["forecast_power_kw"] .rolling(window=self.smooth_blocks, min_periods=1, center=True) .mean() - .astype(int) ) + # convert to int + values_df["forecast_power_kw"] = values_df["forecast_power_kw"].astype(int) + # remove any negative values values_df["forecast_power_kw"] = values_df["forecast_power_kw"].clip(lower=0.0)