-
-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #228 from openclimatefix/jacob/ecmwf-support
Add support for ECMWF
- Loading branch information
Showing
34 changed files
with
446 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| """ECMWF provider loaders""" | ||
| import pandas as pd | ||
| import xarray as xr | ||
|
|
||
| from ocf_datapipes.load.nwp.providers.utils import open_zarr_paths | ||
|
|
||
|
|
||
| def open_ifs(zarr_path) -> xr.DataArray: | ||
| """ | ||
| Opens the ECMWF IFS NWP data | ||
| Args: | ||
| zarr_path: Path to the zarr to open | ||
| Returns: | ||
| Xarray DataArray of the NWP data | ||
| """ | ||
| # Open the data | ||
| nwp = open_zarr_paths(zarr_path) | ||
| dataVars = list(nwp.data_vars.keys()) | ||
| if len(dataVars) > 1: | ||
| raise Exception("Too many TLDVs") | ||
| else: | ||
| dataVar = dataVars[0] | ||
| ifs: xr.Dataset = nwp[dataVar] | ||
| del nwp | ||
| ifs = ifs.transpose("init_time", "step", "variable", "latitude", "longitude") | ||
| ifs = ifs.rename( | ||
| { | ||
| "init_time": "init_time_utc", | ||
| "variable": "channel", | ||
| "latitude": "latitude", | ||
| "longitude": "longitude", | ||
| } | ||
| ) | ||
| # Sanity checks. | ||
| time = pd.DatetimeIndex(ifs.init_time_utc) | ||
| assert time.is_unique | ||
| assert time.is_monotonic_increasing | ||
| return ifs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| { | ||
| "zarr_format": 2 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,188 @@ | ||
| { | ||
| "metadata": { | ||
| ".zattrs": {}, | ||
| ".zgroup": { | ||
| "zarr_format": 2 | ||
| }, | ||
| "ECMWF_BLAH/.zarray": { | ||
| "chunks": [ | ||
| 17, | ||
| 1, | ||
| 49, | ||
| 110, | ||
| 110 | ||
| ], | ||
| "compressor": { | ||
| "clevel": 5, | ||
| "cname": "zstd", | ||
| "id": "blosc2" | ||
| }, | ||
| "dtype": "<f4", | ||
| "fill_value": "NaN", | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 2, | ||
| 1, | ||
| 2, | ||
| 221, | ||
| 221 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "ECMWF_BLAH/.zattrs": { | ||
| "Conventions": "CF-1.7", | ||
| "GRIB_centre": "ecmf", | ||
| "GRIB_centreDescription": "European Centre for Medium-Range Weather Forecasts", | ||
| "GRIB_subCentre": 0, | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "variable", | ||
| "init_time", | ||
| "step", | ||
| "latitude", | ||
| "longitude" | ||
| ], | ||
| "institution": "European Centre for Medium-Range Weather Forecasts" | ||
| }, | ||
| "init_time/.zarray": { | ||
| "chunks": [ | ||
| 1 | ||
| ], | ||
| "compressor": { | ||
| "blocksize": 0, | ||
| "clevel": 5, | ||
| "cname": "lz4", | ||
| "id": "blosc", | ||
| "shuffle": 1 | ||
| }, | ||
| "dtype": "<i8", | ||
| "fill_value": null, | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 1 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "init_time/.zattrs": { | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "init_time" | ||
| ], | ||
| "calendar": "proleptic_gregorian", | ||
| "long_name": "initial time of forecast", | ||
| "standard_name": "forecast_reference_time", | ||
| "units": "nanoseconds since 1970-01-01" | ||
| }, | ||
| "latitude/.zarray": { | ||
| "chunks": [ | ||
| 221 | ||
| ], | ||
| "compressor": { | ||
| "blocksize": 0, | ||
| "clevel": 5, | ||
| "cname": "lz4", | ||
| "id": "blosc", | ||
| "shuffle": 1 | ||
| }, | ||
| "dtype": "<f8", | ||
| "fill_value": "NaN", | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 221 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "latitude/.zattrs": { | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "latitude" | ||
| ], | ||
| "long_name": "latitude", | ||
| "standard_name": "latitude", | ||
| "stored_direction": "decreasing", | ||
| "units": "degrees_north" | ||
| }, | ||
| "longitude/.zarray": { | ||
| "chunks": [ | ||
| 221 | ||
| ], | ||
| "compressor": { | ||
| "blocksize": 0, | ||
| "clevel": 5, | ||
| "cname": "lz4", | ||
| "id": "blosc", | ||
| "shuffle": 1 | ||
| }, | ||
| "dtype": "<f8", | ||
| "fill_value": "NaN", | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 221 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "longitude/.zattrs": { | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "longitude" | ||
| ], | ||
| "long_name": "longitude", | ||
| "standard_name": "longitude", | ||
| "units": "degrees_east" | ||
| }, | ||
| "step/.zarray": { | ||
| "chunks": [ | ||
| 49 | ||
| ], | ||
| "compressor": { | ||
| "blocksize": 0, | ||
| "clevel": 5, | ||
| "cname": "lz4", | ||
| "id": "blosc", | ||
| "shuffle": 1 | ||
| }, | ||
| "dtype": "<f8", | ||
| "fill_value": "NaN", | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 2 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "step/.zattrs": { | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "step" | ||
| ], | ||
| "long_name": "time since forecast_reference_time", | ||
| "standard_name": "forecast_period", | ||
| "units": "hours" | ||
| }, | ||
| "variable/.zarray": { | ||
| "chunks": [ | ||
| 2 | ||
| ], | ||
| "compressor": { | ||
| "blocksize": 0, | ||
| "clevel": 5, | ||
| "cname": "lz4", | ||
| "id": "blosc", | ||
| "shuffle": 1 | ||
| }, | ||
| "dtype": "<U5", | ||
| "fill_value": null, | ||
| "filters": null, | ||
| "order": "C", | ||
| "shape": [ | ||
| 2 | ||
| ], | ||
| "zarr_format": 2 | ||
| }, | ||
| "variable/.zattrs": { | ||
| "_ARRAY_DIMENSIONS": [ | ||
| "variable" | ||
| ] | ||
| } | ||
| }, | ||
| "zarr_consolidated_format": 1 | ||
| } |
Oops, something went wrong.