From 4509d3f9fdee38dd263959f1917b4c9615dcfd07 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 18 Dec 2024 11:29:17 +0100 Subject: [PATCH 1/8] Implement validation of directional regions --- nomenclature/codelist.py | 12 ++++++++++++ .../directional_non-existing_component/region.yaml | 4 ++++ 2 files changed, 16 insertions(+) create mode 100644 tests/data/codelist/region_codelist/directional_non-existing_component/region.yaml diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 685bb483..34984dc1 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -788,6 +788,18 @@ def from_directory( ) ) mapping[code.name] = code + + for code in list(mapping): + if ">" in code: + origin, destination = code.split(">") + for region in [origin, destination]: + if region not in mapping: + errors.append( + ValueError( + f"Region '{region}' not defined for '{code}'" + ) + ) + if errors: raise ValueError(errors) return cls(name=name, mapping=mapping) diff --git a/tests/data/codelist/region_codelist/directional_non-existing_component/region.yaml b/tests/data/codelist/region_codelist/directional_non-existing_component/region.yaml new file mode 100644 index 00000000..941e5f3a --- /dev/null +++ b/tests/data/codelist/region_codelist/directional_non-existing_component/region.yaml @@ -0,0 +1,4 @@ +- countries: + - Austria +- directional: + - Austria>Germany From 37a368c05da786dd9872f19a75d4ea1967d46543 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 18 Dec 2024 11:30:33 +0100 Subject: [PATCH 2/8] Add a test --- .../codelist/region_codelist/simple/region.yaml | 12 +++++++----- tests/test_codelist.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/data/codelist/region_codelist/simple/region.yaml b/tests/data/codelist/region_codelist/simple/region.yaml index f069d7bb..58288897 100644 --- a/tests/data/codelist/region_codelist/simple/region.yaml +++ b/tests/data/codelist/region_codelist/simple/region.yaml @@ -1,7 +1,9 @@ - common: - - World: - definition: The entire world + - World: + definition: The entire world - countries: - - Some Country: - iso2: XY - iso3: XYZ + - Some Country: + iso2: XY + iso3: XYZ +- directional: + - Some Country>World diff --git a/tests/test_codelist.py b/tests/test_codelist.py index f0b56955..466d4a34 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -193,6 +193,9 @@ def test_region_codelist(): assert code["Some Country"].hierarchy == "countries" assert code["Some Country"].iso2 == "XY" + assert "Some Country>World" in code + assert code["Some Country>World"].hierarchy == "directional" + def test_region_codelist_nonexisting_country_name(): """Check that countries are validated against `nomenclature.countries`""" @@ -205,6 +208,17 @@ def test_region_codelist_nonexisting_country_name(): ) +def test_directional_region_codelist_nonexisting_country_name(): + """Check that directional regions have defined origin and destination""" + with pytest.raises(ValueError, match="Region 'Germany' not .* 'Austria>Germany'"): + RegionCodeList.from_directory( + "region", + MODULE_TEST_DATA_DIR + / "region_codelist" + / "directional_non-existing_component", + ) + + def test_region_codelist_str_country_name(): """Check that country name as string is validated against `nomenclature.countries`""" code = RegionCodeList.from_directory( From b40da8f7dc0578737272c1c0a502b4390ac03e63 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 18 Dec 2024 11:31:01 +0100 Subject: [PATCH 3/8] Make ruff --- nomenclature/codelist.py | 4 +--- tests/test_codelist.py | 1 - tests/test_model_registration_parser.py | 6 ++---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 34984dc1..a86422e6 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -795,9 +795,7 @@ def from_directory( for region in [origin, destination]: if region not in mapping: errors.append( - ValueError( - f"Region '{region}' not defined for '{code}'" - ) + ValueError(f"Region '{region}' not defined for '{code}'") ) if errors: diff --git a/tests/test_codelist.py b/tests/test_codelist.py index 466d4a34..9a574e6c 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -394,7 +394,6 @@ def test_RegionCodeList_filter(): def test_RegionCodeList_hierarchy(): """Verifies that the hierarchy method returns a list""" - rcl = RegionCodeList.from_directory( "Region", MODULE_TEST_DATA_DIR / "region_to_filter_codelist" ) diff --git a/tests/test_model_registration_parser.py b/tests/test_model_registration_parser.py index c9142525..b2525363 100644 --- a/tests/test_model_registration_parser.py +++ b/tests/test_model_registration_parser.py @@ -15,8 +15,7 @@ def test_parse_model_registration(tmp_path): ) # Test model mapping - with open(tmp_path / "Model 1.1_mapping.yaml", "r", encoding="utf-8") \ - as file: + with open(tmp_path / "Model 1.1_mapping.yaml", "r", encoding="utf-8") as file: obs_model_mapping = yaml.safe_load(file) with open( TEST_DATA_DIR @@ -30,8 +29,7 @@ def test_parse_model_registration(tmp_path): assert obs_model_mapping == exp_model_mapping # Test model regions - with open(tmp_path / "Model 1.1_regions.yaml", "r", encoding="utf-8") \ - as file: + with open(tmp_path / "Model 1.1_regions.yaml", "r", encoding="utf-8") as file: obs_model_regions = yaml.safe_load(file) exp_model_regions = [ {"Model 1.1": ["Model 1.1|Region 1", "Region 2", "Model 1.1|Region 3"]} From 117012d8a734440fd6dd9b6f66af273f5e3acd72 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 18 Dec 2024 11:42:04 +0100 Subject: [PATCH 4/8] Add directional regions to the docs --- docs/user_guide/region.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/user_guide/region.rst b/docs/user_guide/region.rst index 0ba316e2..0a323921 100644 --- a/docs/user_guide/region.rst +++ b/docs/user_guide/region.rst @@ -22,6 +22,20 @@ Regions can have attributes, for example a description or ISO3-codes. If the att `iso3_codes` is provided, the item(s) are validated against a list of valid codes taken from the `pycountry `_ package. +Directional data +---------------- + +For reporting of directional data (e.g., trade flows), "directional regions" can be +defined using a *>* separator. The region before the separator is the *origin*, +the region after the separator is the *destination*. + +.. code:: yaml + + - Trade Connections: + - China>Europe + +Both the origin and destination regions must be defined in the region codelist. + Common regions -------------- From 4b378d9d54d0d9bfd103deffa0781ded4f77981e Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:23:35 +0100 Subject: [PATCH 5/8] Add directional properties --- nomenclature/code.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nomenclature/code.py b/nomenclature/code.py index 9c06bf36..036ad86d 100644 --- a/nomenclature/code.py +++ b/nomenclature/code.py @@ -309,6 +309,22 @@ def check_iso3_codes(cls, v: list[str], info: ValidationInfo) -> list[str]: raise ValueError(errors) return v + @property + def is_directional(self) -> bool: + return ">" in self.name + + @property + def destination(self) -> str: + if not self.is_directional: + raise ValueError("Non directional region does not have a destination") + return self.name.split(">")[1] + + @property + def origin(self) -> str: + if not self.is_directional: + raise ValueError("Non directional region does not have an origin") + return self.name.split(">")[0] + class MetaCode(Code): """Code object with allowed values list From df70e9a71df2708f0d6babda2ce8ac97e59aeec7 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:23:54 +0100 Subject: [PATCH 6/8] Move directional validation into validator --- nomenclature/codelist.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index a86422e6..167d7ad4 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -789,19 +789,28 @@ def from_directory( ) mapping[code.name] = code - for code in list(mapping): - if ">" in code: - origin, destination = code.split(">") - for region in [origin, destination]: - if region not in mapping: - errors.append( - ValueError(f"Region '{region}' not defined for '{code}'") - ) - if errors: raise ValueError(errors) return cls(name=name, mapping=mapping) + @field_validator("mapping") + @classmethod + def check_directional_regions(cls, v: dict[str, RegionCode]): + missing_regions = [] + for region in v.values(): + if region.is_directional: + if region.origin not in v: + missing_regions.append( + f"Region '{region.origin}' not defined for '{region.name}'" + ) + if region.destination not in v: + missing_regions.append( + f"Region '{region.destination}' not defined for '{region.name}'" + ) + if missing_regions: + raise ValueError("\n".join(missing_regions)) + return v + @property def hierarchy(self) -> list[str]: """Return the hierarchies defined in the RegionCodeList From 51244d5df5c1871486257225c65ea8f050b9df56 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:56:04 +0100 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Daniel Huppmann --- nomenclature/codelist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 167d7ad4..496a5411 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -801,11 +801,11 @@ def check_directional_regions(cls, v: dict[str, RegionCode]): if region.is_directional: if region.origin not in v: missing_regions.append( - f"Region '{region.origin}' not defined for '{region.name}'" + f"Origin '{region.origin}' not defined for '{region.name}'" ) if region.destination not in v: missing_regions.append( - f"Region '{region.destination}' not defined for '{region.name}'" + f"Destination '{region.destination}' not defined for '{region.name}'" ) if missing_regions: raise ValueError("\n".join(missing_regions)) From f3da9a606ce9036c8eed80de05b1a1f8165bbcca Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 19 Dec 2024 11:59:45 +0100 Subject: [PATCH 8/8] Fix the failing test --- tests/test_codelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_codelist.py b/tests/test_codelist.py index 9a574e6c..0a833c02 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -210,7 +210,7 @@ def test_region_codelist_nonexisting_country_name(): def test_directional_region_codelist_nonexisting_country_name(): """Check that directional regions have defined origin and destination""" - with pytest.raises(ValueError, match="Region 'Germany' not .* 'Austria>Germany'"): + with pytest.raises(ValueError, match="Destination 'Germany' .* 'Austria>Germany'"): RegionCodeList.from_directory( "region", MODULE_TEST_DATA_DIR