From 7f0c7136b7ddbdbb87b6334a5be535eeadf0ba4f Mon Sep 17 00:00:00 2001 From: Ritesh Mehta Date: Fri, 12 May 2023 11:05:22 -0700 Subject: [PATCH 1/5] add new coordinate_system field and validators for coordinate_system, x and y --- ocf_datapipes/utils/consts.py | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/ocf_datapipes/utils/consts.py b/ocf_datapipes/utils/consts.py index 2acb71447..dd2f37062 100644 --- a/ocf_datapipes/utils/consts.py +++ b/ocf_datapipes/utils/consts.py @@ -147,7 +147,41 @@ class Location(BaseModel): x: float y: float id: Optional[int] - + coordinate_system: Optional[str] = "osgb" # ["osgb", "lat_lon"] + + @validator("coordinate_system") + def validate_coordinate_system(cls, v): + """Validate 'coordinate_system'""" + allowed_coordinate_systen = ["osgb", "lat_lon"] + if v not in allowed_coordinate_systen: + raise ValueError(f"coordinate_system = {v} is not in {allowed_coordinate_systen}") + return v + + @validator("x") + def validate_x(cls, v): + """Validate 'x'""" + min_x: float + max_x: float + if coordinate_system == "osgb": + min_x, max_x = -103976.3, 652897.98 + if coordinate_system == "lat_lon": + min_x, max_x = -180, 180 + if v < min_x or v > max_x: + raise ValueError(f"x = {v} must be within {[min_x, max_x]} for {coordinate_system} coordinate system") + return v + + @validator("y") + def validate_x(cls, v): + """Validate 'y'""" + min_y: float + max_y: float + if coordinate_system == "osgb": + min_y, max_y = -16703.87, 1199851.44 + if coordinate_system == "lat_lon": + min_y, max_y = -90, 90 + if v < min_y or v > max_y: + raise ValueError(f"y = {v} must be within {[min_y, max_y]} for {coordinate_system} coordinate system") + return v class BatchKey(Enum): """The names of the different elements of each batch. From 7cd1f7ba44d3fc0ecec544d59f5c3b81cdd56e90 Mon Sep 17 00:00:00 2001 From: Ritesh Mehta Date: Fri, 12 May 2023 11:37:41 -0700 Subject: [PATCH 2/5] update function name typo in validator for 'y' --- ocf_datapipes/utils/consts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocf_datapipes/utils/consts.py b/ocf_datapipes/utils/consts.py index dd2f37062..4ce8b2b3f 100644 --- a/ocf_datapipes/utils/consts.py +++ b/ocf_datapipes/utils/consts.py @@ -171,7 +171,7 @@ def validate_x(cls, v): return v @validator("y") - def validate_x(cls, v): + def validate_y(cls, v): """Validate 'y'""" min_y: float max_y: float From afe78a3f96409f3165b9a696f6c8f1906369b523 Mon Sep 17 00:00:00 2001 From: Ritesh Mehta Date: Wed, 31 May 2023 16:15:08 -0700 Subject: [PATCH 3/5] add validation to the Location object in consts.py, add corresponding tests in tests/utils/test_constants.py --- ocf_datapipes/utils/consts.py | 30 ++++++++++++-------- tests/utils/test_constants.py | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 tests/utils/test_constants.py diff --git a/ocf_datapipes/utils/consts.py b/ocf_datapipes/utils/consts.py index 4ce8b2b3f..128e34413 100644 --- a/ocf_datapipes/utils/consts.py +++ b/ocf_datapipes/utils/consts.py @@ -5,7 +5,7 @@ import numpy as np import xarray as xr -from pydantic import BaseModel +from pydantic import BaseModel, validator PV_TIME_AXIS = 1 PV_SYSTEM_AXIS = 2 @@ -144,12 +144,12 @@ class Location(BaseModel): """Represent a spatial location.""" + coordinate_system: Optional[str] = "osgb" # ["osgb", "lat_lon"] x: float y: float id: Optional[int] - coordinate_system: Optional[str] = "osgb" # ["osgb", "lat_lon"] - @validator("coordinate_system") + @validator("coordinate_system", pre=True, always=True) def validate_coordinate_system(cls, v): """Validate 'coordinate_system'""" allowed_coordinate_systen = ["osgb", "lat_lon"] @@ -157,30 +157,36 @@ def validate_coordinate_system(cls, v): raise ValueError(f"coordinate_system = {v} is not in {allowed_coordinate_systen}") return v - @validator("x") - def validate_x(cls, v): + @validator('x') + def validate_x(cls, v, values): """Validate 'x'""" min_x: float max_x: float - if coordinate_system == "osgb": + if "coordinate_system" not in values: + raise ValueError(f"coordinate_system is incorrect") + co = values["coordinate_system"] + if co == "osgb": min_x, max_x = -103976.3, 652897.98 - if coordinate_system == "lat_lon": + if co == "lat_lon": min_x, max_x = -180, 180 if v < min_x or v > max_x: - raise ValueError(f"x = {v} must be within {[min_x, max_x]} for {coordinate_system} coordinate system") + raise ValueError(f"x = {v} must be within {[min_x, max_x]} for {co} coordinate system") return v @validator("y") - def validate_y(cls, v): + def validate_y(cls, v, values): """Validate 'y'""" min_y: float max_y: float - if coordinate_system == "osgb": + if "coordinate_system" not in values: + raise ValueError(f"coordinate_system is incorrect") + co = values["coordinate_system"] + if co == "osgb": min_y, max_y = -16703.87, 1199851.44 - if coordinate_system == "lat_lon": + if co == "lat_lon": min_y, max_y = -90, 90 if v < min_y or v > max_y: - raise ValueError(f"y = {v} must be within {[min_y, max_y]} for {coordinate_system} coordinate system") + raise ValueError(f"y = {v} must be within {[min_y, max_y]} for {co} coordinate system") return v class BatchKey(Enum): diff --git a/tests/utils/test_constants.py b/tests/utils/test_constants.py new file mode 100644 index 000000000..8eb74ef6c --- /dev/null +++ b/tests/utils/test_constants.py @@ -0,0 +1,53 @@ +from ocf_datapipes.utils.consts import Location +import pytest + +def test_make_valid_location_object_with_default_coordinate_system(): + x, y = -1000.5, 50000 + location = Location(x=x, y=y) + assert location.x == x, "location.x value not set correctly" + assert location.y == y, "location.x value not set correctly" + assert location.coordinate_system == "osgb", "location.coordinate_system value not set correctly" + +def test_make_valid_location_object_with_osgb_coordinate_system(): + x, y, coordinate_system = 1.2, 22.9, "osgb" + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert location.x == x, "location.x value not set correctly" + assert location.y == y, "location.x value not set correctly" + assert location.coordinate_system == coordinate_system, "location.coordinate_system value not set correctly" + +def test_make_valid_location_object_with_lat_lon_coordinate_system(): + x, y, coordinate_system = 1.2, 1.2, "lat_lon" + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert location.x == x, "location.x value not set correctly" + assert location.y == y, "location.x value not set correctly" + assert location.coordinate_system == coordinate_system, "location.coordinate_system value not set correctly" + +def test_make_invalid_location_object_with_invalid_osgb_x(): + x, y, coordinate_system = 10000000, 1.2, "osgb" + with pytest.raises(ValueError) as err: + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert err.typename == "ValidationError" + +def test_make_invalid_location_object_with_invalid_osgb_y(): + x, y, coordinate_system = 2.5, 10000000, "osgb" + with pytest.raises(ValueError) as err: + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert err.typename == "ValidationError" + +def test_make_invalid_location_object_with_invalid_lat_lon_x(): + x, y, coordinate_system = 200, 1.2, "lat_lon" + with pytest.raises(ValueError) as err: + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert err.typename == "ValidationError" + +def test_make_invalid_location_object_with_invalid_lat_lon_y(): + x, y, coordinate_system = 2.5, -200, "lat_lon" + with pytest.raises(ValueError) as err: + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert err.typename == "ValidationError" + +def test_make_invalid_location_object_with_invalid_coordinate_system(): + x, y, coordinate_system = 2.5, 1000, "abcd" + with pytest.raises(ValueError) as err: + location = Location(x=x, y=y, coordinate_system=coordinate_system) + assert err.typename == "ValidationError" \ No newline at end of file From 2c1dc199e11b5ebf50a63292a67709abb4eb14ef Mon Sep 17 00:00:00 2001 From: Ritesh Mehta Date: Wed, 31 May 2023 16:32:38 -0700 Subject: [PATCH 4/5] update indentation in tests/utils/test_constants.py --- tests/utils/test_constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/utils/test_constants.py b/tests/utils/test_constants.py index 8eb74ef6c..5735cf143 100644 --- a/tests/utils/test_constants.py +++ b/tests/utils/test_constants.py @@ -2,11 +2,11 @@ import pytest def test_make_valid_location_object_with_default_coordinate_system(): - x, y = -1000.5, 50000 - location = Location(x=x, y=y) - assert location.x == x, "location.x value not set correctly" - assert location.y == y, "location.x value not set correctly" - assert location.coordinate_system == "osgb", "location.coordinate_system value not set correctly" + x, y = -1000.5, 50000 + location = Location(x=x, y=y) + assert location.x == x, "location.x value not set correctly" + assert location.y == y, "location.x value not set correctly" + assert location.coordinate_system == "osgb", "location.coordinate_system value not set correctly" def test_make_valid_location_object_with_osgb_coordinate_system(): x, y, coordinate_system = 1.2, 22.9, "osgb" From cd4c52f23b91b3a048c8818b4d1a458a0a875b97 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 23:51:10 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ocf_datapipes/utils/consts.py | 7 ++++--- tests/utils/test_constants.py | 22 ++++++++++++++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ocf_datapipes/utils/consts.py b/ocf_datapipes/utils/consts.py index 128e34413..e2887860f 100644 --- a/ocf_datapipes/utils/consts.py +++ b/ocf_datapipes/utils/consts.py @@ -157,13 +157,13 @@ def validate_coordinate_system(cls, v): raise ValueError(f"coordinate_system = {v} is not in {allowed_coordinate_systen}") return v - @validator('x') + @validator("x") def validate_x(cls, v, values): """Validate 'x'""" min_x: float max_x: float if "coordinate_system" not in values: - raise ValueError(f"coordinate_system is incorrect") + raise ValueError("coordinate_system is incorrect") co = values["coordinate_system"] if co == "osgb": min_x, max_x = -103976.3, 652897.98 @@ -179,7 +179,7 @@ def validate_y(cls, v, values): min_y: float max_y: float if "coordinate_system" not in values: - raise ValueError(f"coordinate_system is incorrect") + raise ValueError("coordinate_system is incorrect") co = values["coordinate_system"] if co == "osgb": min_y, max_y = -16703.87, 1199851.44 @@ -189,6 +189,7 @@ def validate_y(cls, v, values): raise ValueError(f"y = {v} must be within {[min_y, max_y]} for {co} coordinate system") return v + class BatchKey(Enum): """The names of the different elements of each batch. diff --git a/tests/utils/test_constants.py b/tests/utils/test_constants.py index 5735cf143..189b3dc80 100644 --- a/tests/utils/test_constants.py +++ b/tests/utils/test_constants.py @@ -1,26 +1,36 @@ from ocf_datapipes.utils.consts import Location import pytest + def test_make_valid_location_object_with_default_coordinate_system(): x, y = -1000.5, 50000 location = Location(x=x, y=y) assert location.x == x, "location.x value not set correctly" assert location.y == y, "location.x value not set correctly" - assert location.coordinate_system == "osgb", "location.coordinate_system value not set correctly" + assert ( + location.coordinate_system == "osgb" + ), "location.coordinate_system value not set correctly" + def test_make_valid_location_object_with_osgb_coordinate_system(): x, y, coordinate_system = 1.2, 22.9, "osgb" location = Location(x=x, y=y, coordinate_system=coordinate_system) assert location.x == x, "location.x value not set correctly" assert location.y == y, "location.x value not set correctly" - assert location.coordinate_system == coordinate_system, "location.coordinate_system value not set correctly" + assert ( + location.coordinate_system == coordinate_system + ), "location.coordinate_system value not set correctly" + def test_make_valid_location_object_with_lat_lon_coordinate_system(): x, y, coordinate_system = 1.2, 1.2, "lat_lon" location = Location(x=x, y=y, coordinate_system=coordinate_system) assert location.x == x, "location.x value not set correctly" assert location.y == y, "location.x value not set correctly" - assert location.coordinate_system == coordinate_system, "location.coordinate_system value not set correctly" + assert ( + location.coordinate_system == coordinate_system + ), "location.coordinate_system value not set correctly" + def test_make_invalid_location_object_with_invalid_osgb_x(): x, y, coordinate_system = 10000000, 1.2, "osgb" @@ -28,26 +38,30 @@ def test_make_invalid_location_object_with_invalid_osgb_x(): location = Location(x=x, y=y, coordinate_system=coordinate_system) assert err.typename == "ValidationError" + def test_make_invalid_location_object_with_invalid_osgb_y(): x, y, coordinate_system = 2.5, 10000000, "osgb" with pytest.raises(ValueError) as err: location = Location(x=x, y=y, coordinate_system=coordinate_system) assert err.typename == "ValidationError" + def test_make_invalid_location_object_with_invalid_lat_lon_x(): x, y, coordinate_system = 200, 1.2, "lat_lon" with pytest.raises(ValueError) as err: location = Location(x=x, y=y, coordinate_system=coordinate_system) assert err.typename == "ValidationError" + def test_make_invalid_location_object_with_invalid_lat_lon_y(): x, y, coordinate_system = 2.5, -200, "lat_lon" with pytest.raises(ValueError) as err: location = Location(x=x, y=y, coordinate_system=coordinate_system) assert err.typename == "ValidationError" + def test_make_invalid_location_object_with_invalid_coordinate_system(): x, y, coordinate_system = 2.5, 1000, "abcd" with pytest.raises(ValueError) as err: location = Location(x=x, y=y, coordinate_system=coordinate_system) - assert err.typename == "ValidationError" \ No newline at end of file + assert err.typename == "ValidationError"