From 53c718e6a3c1d53eaf0a7c80a99ab124808fef86 Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Tue, 9 May 2023 12:10:10 +0000
Subject: [PATCH 1/7] update for gsp multi-region merging fix

---
 tests/transform/xarray/gsp/test_remove_northern_gsp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/transform/xarray/gsp/test_remove_northern_gsp.py b/tests/transform/xarray/gsp/test_remove_northern_gsp.py
index f38c56628..83d15565f 100644
--- a/tests/transform/xarray/gsp/test_remove_northern_gsp.py
+++ b/tests/transform/xarray/gsp/test_remove_northern_gsp.py
@@ -16,10 +16,10 @@ def test_remove_northern_gsp_all(gsp_datapipe):
 
 
 def test_remove_northern_gsp_some(gsp_datapipe):
-    northern_y_osgb_limit = 180000
+    northern_y_osgb_limit = 180_000
 
     gsp_datapipe = RemoveNorthernGSP(gsp_datapipe, northern_y_osgb_limit=northern_y_osgb_limit)
     data = next(iter(gsp_datapipe))
 
-    assert len(data.gsp_id) == 5
+    assert len(data.gsp_id) == 6
     assert (data.y_osgb < northern_y_osgb_limit).all()

From 3fb18208071e709d1f4f2c847430149540e4c0dc Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Tue, 9 May 2023 12:12:14 +0000
Subject: [PATCH 2/7] remove explicit timezones and add missing import

---
 tests/config/test.yaml        |  4 ++--
 tests/conftest.py             | 18 +++++++++---------
 tests/load/pv/test_load_pv.py |  4 ++--
 tests/training/test_common.py |  1 +
 4 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/tests/config/test.yaml b/tests/config/test.yaml
index d4ae5799d..5589ff402 100644
--- a/tests/config/test.yaml
+++ b/tests/config/test.yaml
@@ -28,8 +28,8 @@ input_data:
     pv_image_size_meters_height: 10000000
     pv_image_size_meters_width: 10000000
     n_pv_systems_per_example: 32
-    start_datetime: "2010-01-01 00:00:00+00:00"
-    end_datetime: "2030-01-01 00:00:00+00:00"
+    start_datetime: "2010-01-01 00:00:00"
+    end_datetime: "2030-01-01 00:00:00"
   satellite:
     satellite_channels:
       - IR_016
diff --git a/tests/conftest.py b/tests/conftest.py
index 8093c9eeb..b3db2784d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -77,8 +77,8 @@ def passiv_datapipe():
     )
 
     pv = PV(
-        start_datetime=datetime(2018, 1, 1, tzinfo=timezone.utc),
-        end_datetime=datetime(2023, 1, 1, tzinfo=timezone.utc),
+        start_datetime=datetime(2018, 1, 1),
+        end_datetime=datetime(2023, 1, 1),
     )
     pv_file = PVFiles(
         pv_filename=str(filename),
@@ -110,8 +110,8 @@ def pvoutput_datapipe():
     )
 
     pv = PV(
-        start_datetime=datetime(2018, 1, 1, tzinfo=timezone.utc),
-        end_datetime=datetime(2023, 1, 1, tzinfo=timezone.utc),
+        start_datetime=datetime(2018, 1, 1),
+        end_datetime=datetime(2023, 1, 1),
     )
     pv_file = PVFiles(
         pv_filename=str(filename),
@@ -213,7 +213,7 @@ def pv_yields_and_systems(db_session):
     for hour in range(4, 10):
         for minute in range(0, 60, 5):
             pv_yield_1 = PVYield(
-                datetime_utc=datetime(2022, 1, 1, hour, minute, tzinfo=timezone.utc),
+                datetime_utc=datetime(2022, 1, 1, hour, minute),
                 solar_generation_kw=hour + minute / 100,
             ).to_orm()
             pv_yield_1.pv_system = pv_system_sql_1
@@ -229,7 +229,7 @@ def pv_yields_and_systems(db_session):
     # pv system with gaps every 5 mins
     for minutes in [0, 10, 20, 30]:
         pv_yield_4 = PVYield(
-            datetime_utc=datetime(2022, 1, 1, 4, tzinfo=timezone.utc) + timedelta(minutes=minutes),
+            datetime_utc=datetime(2022, 1, 1, 4) + timedelta(minutes=minutes),
             solar_generation_kw=4,
         ).to_orm()
         pv_yield_4.pv_system = pv_system_sql_2
@@ -237,7 +237,7 @@ def pv_yields_and_systems(db_session):
 
     # add a system with only on pv yield
     pv_yield_5 = PVYield(
-        datetime_utc=datetime(2022, 1, 1, 4, tzinfo=timezone.utc) + timedelta(minutes=minutes),
+        datetime_utc=datetime(2022, 1, 1, 4) + timedelta(minutes=minutes),
         solar_generation_kw=4,
     ).to_orm()
     pv_yield_5.pv_system = pv_system_sql_3
@@ -271,7 +271,7 @@ def gsp_yields(db_session):
         for hour in range(0, 8):
             for minute in range(0, 60, 30):
                 gsp_yield_1 = GSPYield(
-                    datetime_utc=datetime(2022, 1, 1, hour, minute, tzinfo=timezone.utc),
+                    datetime_utc=datetime(2022, 1, 1, hour, minute),
                     solar_generation_kw=hour + minute,
                 )
                 gsp_yield_1_sql = gsp_yield_1.to_orm()
@@ -298,7 +298,7 @@ def pv_parquet_file():
     - generation_wh
     """
 
-    date = datetime(2022, 9, 1, tzinfo=timezone.utc)
+    date = datetime(2022, 9, 1)
     ids = range(0, 10)
     days = 7
 
diff --git a/tests/load/pv/test_load_pv.py b/tests/load/pv/test_load_pv.py
index 2b8ea9df2..8d8f1dc8b 100644
--- a/tests/load/pv/test_load_pv.py
+++ b/tests/load/pv/test_load_pv.py
@@ -70,8 +70,8 @@ def test_open_both_from_nc():
 
 def test_load_parquet_file(pv_parquet_file):
     pv = PV(
-        start_datetime=datetime(2018, 1, 1, tzinfo=timezone.utc),
-        end_datetime=datetime(2023, 1, 1, tzinfo=timezone.utc),
+        start_datetime=datetime(2018, 1, 1),
+        end_datetime=datetime(2023, 1, 1),
     )
     pv_file = PVFiles(
         pv_filename=pv_parquet_file,
diff --git a/tests/training/test_common.py b/tests/training/test_common.py
index 6aaa4026b..9b53a0348 100644
--- a/tests/training/test_common.py
+++ b/tests/training/test_common.py
@@ -15,6 +15,7 @@
 from pyaml_env import parse_config
 
 import pandas as pd
+import numpy as np
 
 
 def test_open_and_return_datapipes():

From 438e3168e61c6b77378802e44c69bb20df35256d Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Tue, 9 May 2023 12:22:50 +0000
Subject: [PATCH 3/7] update indexer and remove explicit timezone - defaults to
 UTC time

---
 ocf_datapipes/select/select_live_t0_time.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ocf_datapipes/select/select_live_t0_time.py b/ocf_datapipes/select/select_live_t0_time.py
index d10e5b96b..726d255a7 100644
--- a/ocf_datapipes/select/select_live_t0_time.py
+++ b/ocf_datapipes/select/select_live_t0_time.py
@@ -24,8 +24,8 @@ def __iter__(self) -> pd.Timestamp:
         for xr_data in self.source_datapipe:
             # Get most recent time in data
             # Select the history that goes back that far
-            latest_time_idx = pd.DatetimeIndex(xr_data[self.dim_name].values).get_loc(
-                pd.Timestamp.utcnow(), method="pad"
-            )
+            latest_time_idx = pd.DatetimeIndex(xr_data[self.dim_name].values).get_indexer(
+                [pd.Timestamp.now(tz=None)], method="pad"
+            )[0]
             latest_time = xr_data[self.dim_name].values[latest_time_idx]
             yield latest_time

From de993bef4c3125acc3e4ac29d5a9715ca485c94f Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Tue, 9 May 2023 14:07:46 +0000
Subject: [PATCH 4/7] linting

---
 ocf_datapipes/select/select_loc_and_t0.py     |  26 +-
 ocf_datapipes/training/common.py              |   6 +-
 ocf_datapipes/transform/xarray/nwp_dropout.py |  28 +-
 .../transform/xarray/standard_dropout.py      |  30 +-
 ocf_datapipes/utils/eso_metadata.csv          | 318 ++++++++++++++++++
 5 files changed, 366 insertions(+), 42 deletions(-)
 create mode 100644 ocf_datapipes/utils/eso_metadata.csv

diff --git a/ocf_datapipes/select/select_loc_and_t0.py b/ocf_datapipes/select/select_loc_and_t0.py
index 379c9dac9..98da62d0d 100644
--- a/ocf_datapipes/select/select_loc_and_t0.py
+++ b/ocf_datapipes/select/select_loc_and_t0.py
@@ -14,18 +14,7 @@
 
 @functional_datapipe("select_loc_and_t0")
 class LocationT0PickerIterDataPipe(IterDataPipe):
-    """Datapipe to yield location-time pairs from the input data source.
-
-    Args:
-        source_datapipe: Datapipe emitting Xarray Dataset
-        return_all: Whether to return all t0-location pairs,
-            if True, also returns them in structured order
-        shuffle: If `return_all` sets whether the pairs are
-            shuffled before being returned.
-        x_dim_name: x dimension name, defaulted to 'x_osgb'
-        y_dim_name: y dimension name, defaulted to 'y_osgb'
-        time_dim_name: time dimension name, defaulted to 'time_utc'
-    """
+    """Datapipe to yield location-time pairs from the input data source."""
 
     def __init__(
         self,
@@ -36,6 +25,19 @@ def __init__(
         y_dim_name: Optional[str] = "y_osgb",
         time_dim_name: Optional[str] = "time_utc",
     ):
+        """   
+        Datapipe to yield location-time pairs from the input data source.
+
+        Args:
+            source_datapipe: Datapipe emitting Xarray Dataset
+            return_all: Whether to return all t0-location pairs,
+                if True, also returns them in structured order
+            shuffle: If `return_all` sets whether the pairs are
+                shuffled before being returned.
+            x_dim_name: x dimension name, defaulted to 'x_osgb'
+            y_dim_name: y dimension name, defaulted to 'y_osgb'
+            time_dim_name: time dimension name, defaulted to 'time_utc'
+        """
         super().__init__()
         self.source_datapipe = source_datapipe
         self.return_all = return_all
diff --git a/ocf_datapipes/training/common.py b/ocf_datapipes/training/common.py
index ce8c8c4f2..db30f92dd 100644
--- a/ocf_datapipes/training/common.py
+++ b/ocf_datapipes/training/common.py
@@ -319,9 +319,9 @@ def create_t0_and_loc_datapipes(
     shuffle: bool = True,
 ):
     """
-    Takes datapipes and returns datapipes of appropriate locations and times for which samples can
-    be constructed from the the input datapipe sources. The (location, t0) pairs are sampled without
-    replacement.
+    Takes source datapipes and returns datapipes of appropriate sample pairs of locations and times.
+    
+    The (location, t0) pairs are sampled without replacement.
 
     Args:
         datapipes_dict: Dictionary of datapipes of input sources for which we want to select
diff --git a/ocf_datapipes/transform/xarray/nwp_dropout.py b/ocf_datapipes/transform/xarray/nwp_dropout.py
index cc13e5187..2ee7c0a98 100644
--- a/ocf_datapipes/transform/xarray/nwp_dropout.py
+++ b/ocf_datapipes/transform/xarray/nwp_dropout.py
@@ -14,19 +14,7 @@
 
 @functional_datapipe("convert_to_nwp_target_time_with_dropout")
 class ConvertToNWPTargetTimeWithDropoutIterDataPipe(IterDataPipe):
-    """Convert NWP Xarray dataset to use target time as indexer
-
-    Args:
-        source_datapipe: Datapipe emitting a Xarray Dataset with step and init_time_utc indexers.
-        t0_datapipe: Datapipe emitting t0 times for indexing off of choosing the closest previous
-            init_time_utc.
-        sample_period_duration: How long the sampling period is.
-        history_duration: How long the history time should cover.
-        forecast_duration: How long the forecast time should cover.
-        dropout_timedeltas: List of timedeltas. We randonly select the delay for each NWP forecast
-            from this list. These should be negative timedeltas w.r.t time t0.
-        dropout_frac: Fraction of samples subject to dropout
-    """
+    """Convert NWP Xarray dataset to use target time as indexer"""
 
     def __init__(
         self,
@@ -38,6 +26,20 @@ def __init__(
         dropout_timedeltas: List[timedelta],
         dropout_frac: Optional[float] = 1,
     ):
+        """Convert NWP Xarray dataset to use target time as indexer
+
+        Args:
+            source_datapipe: Datapipe emitting an Xarray Dataset with step and init_time_utc 
+                indexers.
+            t0_datapipe: Datapipe emitting t0 times for indexing off of choosing the closest
+                previous init_time_utc.
+            sample_period_duration: How long the sampling period is.
+            history_duration: How long the history time should cover.
+            forecast_duration: How long the forecast time should cover.
+            dropout_timedeltas: List of timedeltas. We randonly select the delay for each NWP 
+                forecast from this list. These should be negative timedeltas w.r.t time t0.
+            dropout_frac: Fraction of samples subject to dropout
+        """
         self.source_datapipe = source_datapipe
         self.t0_datapipe = t0_datapipe
         self.sample_period_duration = sample_period_duration
diff --git a/ocf_datapipes/transform/xarray/standard_dropout.py b/ocf_datapipes/transform/xarray/standard_dropout.py
index 96be097e6..8bdf39f4b 100644
--- a/ocf_datapipes/transform/xarray/standard_dropout.py
+++ b/ocf_datapipes/transform/xarray/standard_dropout.py
@@ -11,14 +11,7 @@
 
 @functional_datapipe("select_dropout_time")
 class SelectDropoutTimeIterDataPipe(IterDataPipe):
-    """Generates dropout times. The times are absolute values, not timedeltas.
-
-    Args:
-        source_datapipe: Datapipe of t0 times
-        dropout_timedeltas: List of timedeltas. We randonly select the delay for each time from this
-            list. These should be negative timedeltas w.r.t time t0.
-        dropout_frac: Fraction of samples subject to dropout
-    """
+    """Generates dropout times. The times are absolute values, not timedeltas."""
 
     def __init__(
         self,
@@ -26,6 +19,14 @@ def __init__(
         dropout_timedeltas: List[timedelta],
         dropout_frac: Optional[float] = 0,
     ):
+        """Generates dropout times. The times are absolute values, not timedeltas.
+
+        Args:
+            source_datapipe: Datapipe of t0 times
+            dropout_timedeltas: List of timedeltas. We randonly select the delay for each time from 
+                this list. These should be negative timedeltas w.r.t time t0.
+            dropout_frac: Fraction of samples subject to dropout
+        """
         self.source_datapipe = source_datapipe
         self.dropout_timedeltas = dropout_timedeltas
         self.dropout_frac = dropout_frac
@@ -51,18 +52,19 @@ def __iter__(self):
 
 @functional_datapipe("apply_dropout_time")
 class ApplyDropoutTimeIterDataPipe(IterDataPipe):
-    """Masks an xarray object to replace values that come after the dropout time with NaN.
-
-    Args:
-        source_datapipe: Datapipe of Xarray objects
-        dropout_time_datapipe: Datapipe of dropout times
-    """
+    """Masks an xarray object to replace values that come after the dropout time with NaN."""
 
     def __init__(
         self,
         source_datapipe: IterDataPipe,
         dropout_time_datapipe: IterDataPipe,
     ):
+        """Masks an xarray object to replace values that come after the dropout time with NaN.
+
+        Args:
+            source_datapipe: Datapipe of Xarray objects
+            dropout_time_datapipe: Datapipe of dropout times
+        """
         self.source_datapipe = source_datapipe
         self.dropout_time_datapipe = dropout_time_datapipe
 
diff --git a/ocf_datapipes/utils/eso_metadata.csv b/ocf_datapipes/utils/eso_metadata.csv
new file mode 100644
index 000000000..525921b5f
--- /dev/null
+++ b/ocf_datapipes/utils/eso_metadata.csv
@@ -0,0 +1,318 @@
+,gsp_id,gsp_name,pes_id,region_id
+1,1,ABHA1,22,1
+2,2,ABNE_P,17,2
+3,3,ABTH_1,21,3
+4,4,ACTL_2|CBNK_H|GREE_H|PERI_H,20,4
+5,5,ACTL_C|WISD_1|WISD_6,12,5
+6,6,ALNE_P,17,6
+7,7,ALST_3|USKM_1,21,7
+8,8,ALVE1,22,8
+9,9,AMEM_1,10,9
+10,10,ARBR_P,17,10
+11,11,ARDK_P|CLAC_P,17,11
+12,12,ARMO_P,17,12
+13,13,AXMI1,22,13
+14,14,AYRR,18,14
+15,15,BAGA,18,15
+16,16,BAIN,18,16
+17,17,BARKC1|BARKW3,12,17
+18,18,BEAU_P|ORRI_P,17,18
+19,19,BEDDT1,12,19
+20,20,BEDD_1,19,20
+21,21,BERB_P|CAIF_P|DALL_P|GLEF_P|KEIT_P,17,21
+22,22,BERW,18,22
+23,23,BESW_1,11,23
+24,24,BICF_1,11,24
+25,25,BIRK_1,13,25
+26,26,BISW_1,14,26
+27,27,BLYTB1,15,27
+28,28,BLYTH132,16,28
+29,29,BOAG_P,17,29
+30,30,BOLN_1,19,30
+31,31,BONN,18,31
+32,32,BOTW_1,20,32
+33,33,BRAC_P,17,33
+34,34,BRAI_1,10,34
+35,35,BRAP,18,35
+36,36,BRAW_1,23,36
+37,37,BRED_1,16,37
+38,38,BRFO_1|CLT03,10,38
+39,39,BRID_P,17,39
+40,40,BRIM_1,10,40
+41,41,BRLE_1|FLEE_1,20,41
+42,42,BROA_P,17,42
+43,43,BROR_P,17,43
+44,44,BROX,18,44
+45,45,BRWA1,22,45
+46,46,BUMU_P,17,46
+47,47,BURM_1,10,47
+48,48,BUSH_1,14,48
+49,49,BUST_1,14,49
+50,50,CAAD_P,17,50
+51,51,CAFA|EAST|GLLE|KEOO|TONG,18,51
+52,52,CAMB_01,23,52
+53,53,CANTN1|RICH_J|RICH1,19,53
+54,54,CAPEA1,13,54
+55,55,CARE_1,21,55
+56,56,CARR_1,16,56
+57,57,CASS_P,17,57
+58,58,CATY,18,58
+59,59,CEAN_P,17,59
+60,60,CELL_1,14,60
+61,61,CHAP,18,61
+62,62,CHAR_P,17,62
+63,63,CHAS,18,63
+64,64,CHIC_1,20,64
+65,65,CHSI_1,19,65
+66,66,CHTE_1,11,66
+67,67,CITR_1,12,67
+68,68,CLAY_P,17,68
+69,69,CLYM,18,69
+70,70,COAT,18,70
+71,71,COCK,18,71
+72,72,CONQA1|SASA,13,72
+73,73,COUA_P,17,73
+74,74,COVE_1,11,74
+75,75,COWL_1|ECLA_H,20,75
+76,76,COYL,18,76
+77,77,CRAI_P,17,77
+78,78,CREB_1,23,78
+79,79,CROO,18,79
+80,80,CUMB,18,80
+81,81,CUPA,18,81
+82,82,CURR,18,82
+83,83,DALM3,18,83
+84,84,DEVM,18,84
+85,85,DEVO,18,85
+86,86,DEWP,18,86
+87,87,DOUN_P,17,87
+88,88,DRAK_1,11,88
+89,89,DRAX_1,23,89
+90,90,DRCR,18,90
+91,91,DRUM,18,91
+92,92,DUBE_P,17,92
+93,93,DUDH_P,17,93
+94,94,DUGR_P,17,94
+95,95,DUMF,18,95
+96,96,DUNB,18,96
+97,97,DUNF,18,97
+98,98,DUNO_P,17,98
+99,99,DYCE_P,17,99
+100,100,EALI_6,20,100
+101,101,EASO_1,10,101
+102,102,ECCL,18,102
+103,103,ECLA_1,11,103
+104,104,EERH,18,104
+105,105,EKIL,18,105
+106,106,EKIS,18,106
+107,107,ELDE,18,107
+108,108,ELGI_P,17,108
+109,109,ELLA_1,23,109
+110,110,ELST_1,10,110
+111,111,ENDE_1,11,111
+112,112,ERSK,18,112
+113,113,EXET1,22,113
+114,114,FASN_P,17,114
+115,115,FAUG_P,17,115
+116,116,FAWL_1,20,116
+117,117,FECK_6,14,117
+118,118,FERRA2,23,118
+119,119,FERRB_M|FERRB1,23,119
+120,120,FETT_P|KINT_P,17,120
+121,121,FIDD_P,17,121
+122,122,FIDF_1,13,122
+123,123,FINN,18,123
+124,124,FOUR_1,15,124
+125,125,FRAS_P,17,125
+126,126,FROD_1,13,126
+127,127,FWIL_P,17,127
+128,128,GALA,18,128
+129,129,GIFF,18,129
+130,130,GLLU,18,130
+131,131,GLNI,18,131
+132,132,GLRO,18,132
+133,133,GORG,18,133
+134,134,GOVA,18,134
+135,135,GREN_1,11,135
+136,136,GRIW_1,23,136
+137,137,GRMO,18,137
+138,138,GRUB_P,17,138
+139,139,G_EXTRA_12,17,139
+140,140,HACK_1|HACK_6,12,140
+141,141,HAGR,18,141
+142,142,HAMHC1,14,142
+143,143,HARK_1|HUTT_1|RRIG,16,143
+144,144,HARM_6,15,144
+145,145,HAWI,18,145
+146,146,HAWP_6,15,146
+147,147,HELE,18,147
+148,148,HEYS_1|HEYS1|ORMO,16,148
+149,149,HUER,18,149
+150,150,HURS_1,12,150
+151,151,IMPK_1,21,151
+152,152,INDQ1,22,152
+153,153,INKE,18,153
+154,154,INNE_P,17,154
+155,155,IROA1,22,155
+156,156,IRONB1,14,156
+157,157,ISLI_1|WHAM_1,12,157
+158,158,IVER_1|IVER_6,20,158
+159,159,JOHN,18,159
+160,160,JORD_3,23,160
+161,161,KAIM,18,161
+162,162,KEAD_1,23,162
+163,163,KEAR_1|KEAR_3,16,163
+164,164,KEMS_1,19,164
+165,165,KIBY_1,13,165
+166,166,KIBY_G|WASF_1,16,166
+167,167,KIER,18,167
+168,168,KIIN_P,17,168
+169,169,KILB,18,169
+170,170,KILS,18,170
+171,171,KILT,18,171
+172,172,KILW,18,172
+173,173,KINL_P,17,173
+174,174,KINO_1,19,174
+175,175,KIRKB1,23,175
+176,176,KITW_1,14,176
+177,177,KNAR,15,177
+178,178,LACK_6,15,178
+179,179,LAIR_P,17,179
+180,180,LALE1SG003,20,180
+181,181,LAND1,22,181
+182,182,LEGA_1,13,182
+183,183,LEVE,18,183
+184,184,LING,18,184
+185,185,LINM,18,185
+186,186,LISD_1,13,186
+187,187,LITT_C,12,187
+188,188,LITT_J,19,188
+189,189,LODR_6,12,189
+190,190,LOUD_H,20,190
+191,191,LOVE_1,20,191
+192,192,LUNA_P,17,192
+193,193,LYND_P,17,193
+194,194,MACC_3,16,194
+195,195,MACD_P,17,195
+196,196,MAGA_6,21,196
+197,197,MANN_1,20,197
+198,198,MAYB,18,198
+199,199,MELK_1,20,199
+200,200,MILC_P,17,200
+201,201,MILH_1,10,201
+202,202,MITY_1,20,202
+203,203,MYBS_P,17,203
+204,204,NAIR_P,17,204
+205,205,NEAR,18,205
+206,206,NECE_1,14,206
+207,207,NEEP_3,23,207
+208,208,NETS,18,208
+209,209,NEWX_6,12,209
+210,210,NFLE,19,210
+211,211,NHYD_6,20,211
+212,212,NINF_1,19,212
+213,213,NORL_3,23,213
+214,214,NORM_1|SALL1,10,214
+215,215,NORT_1,15,215
+216,216,NURS_1,20,216
+217,217,OCKH_1,14,217
+218,218,OFFE_3,15,218
+219,219,OLDB_1,14,219
+220,220,OSBA_1,15,220
+221,221,PADIB1,16,221
+222,222,PAIS,18,222
+223,223,PART,18,223
+224,224,PEHG_P,17,224
+225,225,PEHS_P|STRI_P,17,225
+226,226,PELH_1,10,226
+227,227,PEMB_1,21,227
+228,228,PENE_1,15,228
+229,229,PENN_1,14,229
+230,230,PENT_1,13,230
+231,231,PENW_1|STAH_1|WABO,16,231
+232,232,PERS_P,17,232
+233,233,PITS_3,23,233
+234,234,POOB,18,234
+235,235,POPP_3,15,235
+236,236,PORA_P,17,236
+237,237,PORD,18,237
+238,238,PYLE_1,21,238
+239,239,QUOI_P,17,239
+240,240,RAIN_1,13,240
+241,241,RANN_P,17,241
+242,242,RASS_1,21,242
+243,243,RATS_1,11,243
+244,244,RAVE|WISH,18,244
+245,245,RAYL_1,10,245
+246,246,REBR_3,12,246
+247,247,REDH,18,247
+248,248,REDM_P,17,248
+249,249,ROCH_1,16,249
+250,250,RUGEB1,14,250
+251,251,RYEH_1,10,251
+252,252,SACO,18,252
+253,253,SALH_1,15,253
+254,254,SALT_1,23,254
+255,255,SANX,18,255
+256,256,SBAR,11,256
+257,257,SEAB1,22,257
+258,258,SELL_1,19,258
+259,259,SFEG_P,17,259
+260,260,SFIL_P,17,260
+261,261,SHEC_3,23,261
+262,262,SHIN_P,17,262
+263,263,SHRU,18,263
+264,264,SHRW_1,14,264
+265,265,SIGH,18,265
+266,266,SJOW_1,12,266
+267,267,SKLGB1,23,267
+268,268,SLOY_P,17,268
+269,269,SMAN_1,16,269
+270,270,SPAV,18,270
+271,271,SPEN_1,15,271
+272,272,SSHI_3,15,272
+273,273,STAL_1,16,273
+274,274,STAY_1,11,274
+275,275,STEN_1,15,275
+276,276,STES_1,15,276
+277,277,STHA,18,277
+278,278,STIR,18,278
+279,279,STLE,18,279
+280,280,SUND_1,10,280
+281,281,SWAN_1,21,281
+282,282,TARL_P,17,282
+283,283,TAUN1,22,283
+284,284,TAYN_P,17,284
+285,285,TELR,18,285
+286,286,TEMP_3,23,286
+287,287,THOM_6,23,287
+288,288,THSO_P,17,288
+289,289,THUR_6,23,289
+290,290,TILBB_1,10,290
+291,291,TOTT_1,10,291
+292,292,TRAW_1,13,292
+293,293,TUMB_P,17,293
+294,294,TYNE_1|TYNE_2,15,294
+295,295,UPPB_1|UPPB_3,21,295
+296,296,WALH_1,14,296
+297,297,WALP_1,10,297
+298,298,WALP_B,11,298
+299,299,WARL_1,10,299
+300,300,WATFS_1,10,300
+301,301,WBOL_6,15,301
+302,302,WBUR_1,11,302
+303,303,WFIE,18,303
+304,304,WGEO,18,304
+305,305,WHGA_1,16,305
+306,306,WHHO,18,306
+307,307,WIBA_3,23,307
+308,308,WIEN_1,14,308
+309,309,WILL_1,11,309
+310,310,WIMBN1|WIMBS1,12,310
+311,311,WIOW_P,17,311
+312,312,WMEL_1,23,312
+313,313,WOHI_P,17,313
+314,314,WTHU31,10,314
+315,315,WWEY_1,19,315
+316,316,WYLF_1,13,316
+317,317,WYMOM_1,10,317

From 6cfd5e5299162c18cc9ca0ba32ad9bfafd471dd9 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 9 May 2023 14:08:45 +0000
Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 ocf_datapipes/select/select_loc_and_t0.py          | 2 +-
 ocf_datapipes/training/common.py                   | 2 +-
 ocf_datapipes/transform/xarray/nwp_dropout.py      | 4 ++--
 ocf_datapipes/transform/xarray/standard_dropout.py | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ocf_datapipes/select/select_loc_and_t0.py b/ocf_datapipes/select/select_loc_and_t0.py
index 98da62d0d..434ac0c55 100644
--- a/ocf_datapipes/select/select_loc_and_t0.py
+++ b/ocf_datapipes/select/select_loc_and_t0.py
@@ -25,7 +25,7 @@ def __init__(
         y_dim_name: Optional[str] = "y_osgb",
         time_dim_name: Optional[str] = "time_utc",
     ):
-        """   
+        """
         Datapipe to yield location-time pairs from the input data source.
 
         Args:
diff --git a/ocf_datapipes/training/common.py b/ocf_datapipes/training/common.py
index db30f92dd..d890f1231 100644
--- a/ocf_datapipes/training/common.py
+++ b/ocf_datapipes/training/common.py
@@ -320,7 +320,7 @@ def create_t0_and_loc_datapipes(
 ):
     """
     Takes source datapipes and returns datapipes of appropriate sample pairs of locations and times.
-    
+
     The (location, t0) pairs are sampled without replacement.
 
     Args:
diff --git a/ocf_datapipes/transform/xarray/nwp_dropout.py b/ocf_datapipes/transform/xarray/nwp_dropout.py
index 2ee7c0a98..fd4740320 100644
--- a/ocf_datapipes/transform/xarray/nwp_dropout.py
+++ b/ocf_datapipes/transform/xarray/nwp_dropout.py
@@ -29,14 +29,14 @@ def __init__(
         """Convert NWP Xarray dataset to use target time as indexer
 
         Args:
-            source_datapipe: Datapipe emitting an Xarray Dataset with step and init_time_utc 
+            source_datapipe: Datapipe emitting an Xarray Dataset with step and init_time_utc
                 indexers.
             t0_datapipe: Datapipe emitting t0 times for indexing off of choosing the closest
                 previous init_time_utc.
             sample_period_duration: How long the sampling period is.
             history_duration: How long the history time should cover.
             forecast_duration: How long the forecast time should cover.
-            dropout_timedeltas: List of timedeltas. We randonly select the delay for each NWP 
+            dropout_timedeltas: List of timedeltas. We randonly select the delay for each NWP
                 forecast from this list. These should be negative timedeltas w.r.t time t0.
             dropout_frac: Fraction of samples subject to dropout
         """
diff --git a/ocf_datapipes/transform/xarray/standard_dropout.py b/ocf_datapipes/transform/xarray/standard_dropout.py
index 8bdf39f4b..d65dde80e 100644
--- a/ocf_datapipes/transform/xarray/standard_dropout.py
+++ b/ocf_datapipes/transform/xarray/standard_dropout.py
@@ -23,7 +23,7 @@ def __init__(
 
         Args:
             source_datapipe: Datapipe of t0 times
-            dropout_timedeltas: List of timedeltas. We randonly select the delay for each time from 
+            dropout_timedeltas: List of timedeltas. We randonly select the delay for each time from
                 this list. These should be negative timedeltas w.r.t time t0.
             dropout_frac: Fraction of samples subject to dropout
         """

From 2994846aad94f734a82db111383aaea782d79182 Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Wed, 10 May 2023 10:10:54 +0000
Subject: [PATCH 6/7] remove eso metadata

---
 ocf_datapipes/utils/eso_metadata.csv | 318 ---------------------------
 1 file changed, 318 deletions(-)
 delete mode 100644 ocf_datapipes/utils/eso_metadata.csv

diff --git a/ocf_datapipes/utils/eso_metadata.csv b/ocf_datapipes/utils/eso_metadata.csv
deleted file mode 100644
index 525921b5f..000000000
--- a/ocf_datapipes/utils/eso_metadata.csv
+++ /dev/null
@@ -1,318 +0,0 @@
-,gsp_id,gsp_name,pes_id,region_id
-1,1,ABHA1,22,1
-2,2,ABNE_P,17,2
-3,3,ABTH_1,21,3
-4,4,ACTL_2|CBNK_H|GREE_H|PERI_H,20,4
-5,5,ACTL_C|WISD_1|WISD_6,12,5
-6,6,ALNE_P,17,6
-7,7,ALST_3|USKM_1,21,7
-8,8,ALVE1,22,8
-9,9,AMEM_1,10,9
-10,10,ARBR_P,17,10
-11,11,ARDK_P|CLAC_P,17,11
-12,12,ARMO_P,17,12
-13,13,AXMI1,22,13
-14,14,AYRR,18,14
-15,15,BAGA,18,15
-16,16,BAIN,18,16
-17,17,BARKC1|BARKW3,12,17
-18,18,BEAU_P|ORRI_P,17,18
-19,19,BEDDT1,12,19
-20,20,BEDD_1,19,20
-21,21,BERB_P|CAIF_P|DALL_P|GLEF_P|KEIT_P,17,21
-22,22,BERW,18,22
-23,23,BESW_1,11,23
-24,24,BICF_1,11,24
-25,25,BIRK_1,13,25
-26,26,BISW_1,14,26
-27,27,BLYTB1,15,27
-28,28,BLYTH132,16,28
-29,29,BOAG_P,17,29
-30,30,BOLN_1,19,30
-31,31,BONN,18,31
-32,32,BOTW_1,20,32
-33,33,BRAC_P,17,33
-34,34,BRAI_1,10,34
-35,35,BRAP,18,35
-36,36,BRAW_1,23,36
-37,37,BRED_1,16,37
-38,38,BRFO_1|CLT03,10,38
-39,39,BRID_P,17,39
-40,40,BRIM_1,10,40
-41,41,BRLE_1|FLEE_1,20,41
-42,42,BROA_P,17,42
-43,43,BROR_P,17,43
-44,44,BROX,18,44
-45,45,BRWA1,22,45
-46,46,BUMU_P,17,46
-47,47,BURM_1,10,47
-48,48,BUSH_1,14,48
-49,49,BUST_1,14,49
-50,50,CAAD_P,17,50
-51,51,CAFA|EAST|GLLE|KEOO|TONG,18,51
-52,52,CAMB_01,23,52
-53,53,CANTN1|RICH_J|RICH1,19,53
-54,54,CAPEA1,13,54
-55,55,CARE_1,21,55
-56,56,CARR_1,16,56
-57,57,CASS_P,17,57
-58,58,CATY,18,58
-59,59,CEAN_P,17,59
-60,60,CELL_1,14,60
-61,61,CHAP,18,61
-62,62,CHAR_P,17,62
-63,63,CHAS,18,63
-64,64,CHIC_1,20,64
-65,65,CHSI_1,19,65
-66,66,CHTE_1,11,66
-67,67,CITR_1,12,67
-68,68,CLAY_P,17,68
-69,69,CLYM,18,69
-70,70,COAT,18,70
-71,71,COCK,18,71
-72,72,CONQA1|SASA,13,72
-73,73,COUA_P,17,73
-74,74,COVE_1,11,74
-75,75,COWL_1|ECLA_H,20,75
-76,76,COYL,18,76
-77,77,CRAI_P,17,77
-78,78,CREB_1,23,78
-79,79,CROO,18,79
-80,80,CUMB,18,80
-81,81,CUPA,18,81
-82,82,CURR,18,82
-83,83,DALM3,18,83
-84,84,DEVM,18,84
-85,85,DEVO,18,85
-86,86,DEWP,18,86
-87,87,DOUN_P,17,87
-88,88,DRAK_1,11,88
-89,89,DRAX_1,23,89
-90,90,DRCR,18,90
-91,91,DRUM,18,91
-92,92,DUBE_P,17,92
-93,93,DUDH_P,17,93
-94,94,DUGR_P,17,94
-95,95,DUMF,18,95
-96,96,DUNB,18,96
-97,97,DUNF,18,97
-98,98,DUNO_P,17,98
-99,99,DYCE_P,17,99
-100,100,EALI_6,20,100
-101,101,EASO_1,10,101
-102,102,ECCL,18,102
-103,103,ECLA_1,11,103
-104,104,EERH,18,104
-105,105,EKIL,18,105
-106,106,EKIS,18,106
-107,107,ELDE,18,107
-108,108,ELGI_P,17,108
-109,109,ELLA_1,23,109
-110,110,ELST_1,10,110
-111,111,ENDE_1,11,111
-112,112,ERSK,18,112
-113,113,EXET1,22,113
-114,114,FASN_P,17,114
-115,115,FAUG_P,17,115
-116,116,FAWL_1,20,116
-117,117,FECK_6,14,117
-118,118,FERRA2,23,118
-119,119,FERRB_M|FERRB1,23,119
-120,120,FETT_P|KINT_P,17,120
-121,121,FIDD_P,17,121
-122,122,FIDF_1,13,122
-123,123,FINN,18,123
-124,124,FOUR_1,15,124
-125,125,FRAS_P,17,125
-126,126,FROD_1,13,126
-127,127,FWIL_P,17,127
-128,128,GALA,18,128
-129,129,GIFF,18,129
-130,130,GLLU,18,130
-131,131,GLNI,18,131
-132,132,GLRO,18,132
-133,133,GORG,18,133
-134,134,GOVA,18,134
-135,135,GREN_1,11,135
-136,136,GRIW_1,23,136
-137,137,GRMO,18,137
-138,138,GRUB_P,17,138
-139,139,G_EXTRA_12,17,139
-140,140,HACK_1|HACK_6,12,140
-141,141,HAGR,18,141
-142,142,HAMHC1,14,142
-143,143,HARK_1|HUTT_1|RRIG,16,143
-144,144,HARM_6,15,144
-145,145,HAWI,18,145
-146,146,HAWP_6,15,146
-147,147,HELE,18,147
-148,148,HEYS_1|HEYS1|ORMO,16,148
-149,149,HUER,18,149
-150,150,HURS_1,12,150
-151,151,IMPK_1,21,151
-152,152,INDQ1,22,152
-153,153,INKE,18,153
-154,154,INNE_P,17,154
-155,155,IROA1,22,155
-156,156,IRONB1,14,156
-157,157,ISLI_1|WHAM_1,12,157
-158,158,IVER_1|IVER_6,20,158
-159,159,JOHN,18,159
-160,160,JORD_3,23,160
-161,161,KAIM,18,161
-162,162,KEAD_1,23,162
-163,163,KEAR_1|KEAR_3,16,163
-164,164,KEMS_1,19,164
-165,165,KIBY_1,13,165
-166,166,KIBY_G|WASF_1,16,166
-167,167,KIER,18,167
-168,168,KIIN_P,17,168
-169,169,KILB,18,169
-170,170,KILS,18,170
-171,171,KILT,18,171
-172,172,KILW,18,172
-173,173,KINL_P,17,173
-174,174,KINO_1,19,174
-175,175,KIRKB1,23,175
-176,176,KITW_1,14,176
-177,177,KNAR,15,177
-178,178,LACK_6,15,178
-179,179,LAIR_P,17,179
-180,180,LALE1SG003,20,180
-181,181,LAND1,22,181
-182,182,LEGA_1,13,182
-183,183,LEVE,18,183
-184,184,LING,18,184
-185,185,LINM,18,185
-186,186,LISD_1,13,186
-187,187,LITT_C,12,187
-188,188,LITT_J,19,188
-189,189,LODR_6,12,189
-190,190,LOUD_H,20,190
-191,191,LOVE_1,20,191
-192,192,LUNA_P,17,192
-193,193,LYND_P,17,193
-194,194,MACC_3,16,194
-195,195,MACD_P,17,195
-196,196,MAGA_6,21,196
-197,197,MANN_1,20,197
-198,198,MAYB,18,198
-199,199,MELK_1,20,199
-200,200,MILC_P,17,200
-201,201,MILH_1,10,201
-202,202,MITY_1,20,202
-203,203,MYBS_P,17,203
-204,204,NAIR_P,17,204
-205,205,NEAR,18,205
-206,206,NECE_1,14,206
-207,207,NEEP_3,23,207
-208,208,NETS,18,208
-209,209,NEWX_6,12,209
-210,210,NFLE,19,210
-211,211,NHYD_6,20,211
-212,212,NINF_1,19,212
-213,213,NORL_3,23,213
-214,214,NORM_1|SALL1,10,214
-215,215,NORT_1,15,215
-216,216,NURS_1,20,216
-217,217,OCKH_1,14,217
-218,218,OFFE_3,15,218
-219,219,OLDB_1,14,219
-220,220,OSBA_1,15,220
-221,221,PADIB1,16,221
-222,222,PAIS,18,222
-223,223,PART,18,223
-224,224,PEHG_P,17,224
-225,225,PEHS_P|STRI_P,17,225
-226,226,PELH_1,10,226
-227,227,PEMB_1,21,227
-228,228,PENE_1,15,228
-229,229,PENN_1,14,229
-230,230,PENT_1,13,230
-231,231,PENW_1|STAH_1|WABO,16,231
-232,232,PERS_P,17,232
-233,233,PITS_3,23,233
-234,234,POOB,18,234
-235,235,POPP_3,15,235
-236,236,PORA_P,17,236
-237,237,PORD,18,237
-238,238,PYLE_1,21,238
-239,239,QUOI_P,17,239
-240,240,RAIN_1,13,240
-241,241,RANN_P,17,241
-242,242,RASS_1,21,242
-243,243,RATS_1,11,243
-244,244,RAVE|WISH,18,244
-245,245,RAYL_1,10,245
-246,246,REBR_3,12,246
-247,247,REDH,18,247
-248,248,REDM_P,17,248
-249,249,ROCH_1,16,249
-250,250,RUGEB1,14,250
-251,251,RYEH_1,10,251
-252,252,SACO,18,252
-253,253,SALH_1,15,253
-254,254,SALT_1,23,254
-255,255,SANX,18,255
-256,256,SBAR,11,256
-257,257,SEAB1,22,257
-258,258,SELL_1,19,258
-259,259,SFEG_P,17,259
-260,260,SFIL_P,17,260
-261,261,SHEC_3,23,261
-262,262,SHIN_P,17,262
-263,263,SHRU,18,263
-264,264,SHRW_1,14,264
-265,265,SIGH,18,265
-266,266,SJOW_1,12,266
-267,267,SKLGB1,23,267
-268,268,SLOY_P,17,268
-269,269,SMAN_1,16,269
-270,270,SPAV,18,270
-271,271,SPEN_1,15,271
-272,272,SSHI_3,15,272
-273,273,STAL_1,16,273
-274,274,STAY_1,11,274
-275,275,STEN_1,15,275
-276,276,STES_1,15,276
-277,277,STHA,18,277
-278,278,STIR,18,278
-279,279,STLE,18,279
-280,280,SUND_1,10,280
-281,281,SWAN_1,21,281
-282,282,TARL_P,17,282
-283,283,TAUN1,22,283
-284,284,TAYN_P,17,284
-285,285,TELR,18,285
-286,286,TEMP_3,23,286
-287,287,THOM_6,23,287
-288,288,THSO_P,17,288
-289,289,THUR_6,23,289
-290,290,TILBB_1,10,290
-291,291,TOTT_1,10,291
-292,292,TRAW_1,13,292
-293,293,TUMB_P,17,293
-294,294,TYNE_1|TYNE_2,15,294
-295,295,UPPB_1|UPPB_3,21,295
-296,296,WALH_1,14,296
-297,297,WALP_1,10,297
-298,298,WALP_B,11,298
-299,299,WARL_1,10,299
-300,300,WATFS_1,10,300
-301,301,WBOL_6,15,301
-302,302,WBUR_1,11,302
-303,303,WFIE,18,303
-304,304,WGEO,18,304
-305,305,WHGA_1,16,305
-306,306,WHHO,18,306
-307,307,WIBA_3,23,307
-308,308,WIEN_1,14,308
-309,309,WILL_1,11,309
-310,310,WIMBN1|WIMBS1,12,310
-311,311,WIOW_P,17,311
-312,312,WMEL_1,23,312
-313,313,WOHI_P,17,313
-314,314,WTHU31,10,314
-315,315,WWEY_1,19,315
-316,316,WYLF_1,13,316
-317,317,WYMOM_1,10,317

From c82257feda737dc290a586afcb179db6ac121468 Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Wed, 10 May 2023 10:14:39 +0000
Subject: [PATCH 7/7] update gitignore

---
 .gitignore | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index b1385cdf2..446a883ad 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
-# Byte-compiled / optimized / DLL files
+# Custom - ocf_datapipes
+ocf_datapipes/utils/eso_metadata.csv
+
+## Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class