From 16087b9932f45a4c0106a3520da9e74ec26789c7 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 8 Apr 2024 18:07:48 +0200 Subject: [PATCH 1/8] Implement a test against using `|Index` or `|Share` --- tests/test_reserved_terms.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/test_reserved_terms.py diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py new file mode 100644 index 00000000..e23cad6d --- /dev/null +++ b/tests/test_reserved_terms.py @@ -0,0 +1,19 @@ +from nomenclature import DataStructureDefinition + + +def test_legacy_variables(): + # Check that variables use square brackets for operations + # https://github.com/IAMconsortium/common-definitions/issues/55 + + dsd = DataStructureDefinition("definitions/", dimensions=["variable"]) + + error = [] + for variable, attrs in dsd.variable.items(): + if reserved_terms := [r for r in ["Index", "Share"] if "|" + r in variable]: + error.append( + f"Variable '{variable}' -> '[{''.join(reserved_terms)}]'" + ) + if error: + raise ValueError( + f"Found reserved terms in the following variables:\n{'\n - '.join(error)}" + ) From aa168f710b615335a15a83e87aacb24d5395e1f7 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 8 Apr 2024 18:08:10 +0200 Subject: [PATCH 2/8] Rename price-index variable in line with new naming convention --- definitions/variable/sdg-indicators/sdg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/definitions/variable/sdg-indicators/sdg.yaml b/definitions/variable/sdg-indicators/sdg.yaml index 31fdd96e..4621c50b 100644 --- a/definitions/variable/sdg-indicators/sdg.yaml +++ b/definitions/variable/sdg-indicators/sdg.yaml @@ -98,12 +98,12 @@ sdg: 2 unit: kcal/cap/day weight: Population -- Price|Agriculture|Livestock|Index: +- Price|Agriculture|Livestock [Index]: description: Weighted average price index of livestock sdg: 2 unit: Index (2020 = 1) skip-region-aggregation: true -# weight: Agricultural Production|Non-Energy + shape: Price|Agriculture|Livestock|Index - Health|Premature Deaths|PM2.5: description: Number of premature deaths associated with increased health risks from exposure to air pollution of fine particles with a diameter of 2.5 μm or less (PM2.5) From 7eaaf244f78041045280033848b79875e3276153 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 8 Apr 2024 18:09:49 +0200 Subject: [PATCH 3/8] Make ruff --- tests/test_reserved_terms.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py index e23cad6d..189ca090 100644 --- a/tests/test_reserved_terms.py +++ b/tests/test_reserved_terms.py @@ -10,9 +10,7 @@ def test_legacy_variables(): error = [] for variable, attrs in dsd.variable.items(): if reserved_terms := [r for r in ["Index", "Share"] if "|" + r in variable]: - error.append( - f"Variable '{variable}' -> '[{''.join(reserved_terms)}]'" - ) + error.append(f"Variable '{variable}' -> '[{''.join(reserved_terms)}]'") if error: raise ValueError( f"Found reserved terms in the following variables:\n{'\n - '.join(error)}" From dda77fcf37a47bc80ab14d9e14c26fc418957d9e Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 8 Apr 2024 18:12:31 +0200 Subject: [PATCH 4/8] Rename test name --- tests/test_reserved_terms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py index 189ca090..04c5e1ef 100644 --- a/tests/test_reserved_terms.py +++ b/tests/test_reserved_terms.py @@ -1,7 +1,7 @@ from nomenclature import DataStructureDefinition -def test_legacy_variables(): +def test_variable_ops_as_square_brackets(): # Check that variables use square brackets for operations # https://github.com/IAMconsortium/common-definitions/issues/55 From 7852a988db76b1983e3ab12602eb5f0fa8dbd356 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Tue, 9 Apr 2024 10:04:09 +0200 Subject: [PATCH 5/8] Simplify test --- tests/test_reserved_terms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py index 04c5e1ef..4d19bca7 100644 --- a/tests/test_reserved_terms.py +++ b/tests/test_reserved_terms.py @@ -8,9 +8,9 @@ def test_variable_ops_as_square_brackets(): dsd = DataStructureDefinition("definitions/", dimensions=["variable"]) error = [] - for variable, attrs in dsd.variable.items(): + for variable in dsd.variable: if reserved_terms := [r for r in ["Index", "Share"] if "|" + r in variable]: - error.append(f"Variable '{variable}' -> '[{''.join(reserved_terms)}]'") + error.append(f"Variable '{variable}' -> '... [{''.join(reserved_terms)}]'") if error: raise ValueError( f"Found reserved terms in the following variables:\n{'\n - '.join(error)}" From 54bb2df54037d41e68bfaa86a1068f17a5a7f8fe Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 10 Apr 2024 18:18:09 +0200 Subject: [PATCH 6/8] Guard against more terms --- tests/test_reserved_terms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py index 4d19bca7..544ea283 100644 --- a/tests/test_reserved_terms.py +++ b/tests/test_reserved_terms.py @@ -1,5 +1,7 @@ from nomenclature import DataStructureDefinition +RESERVED_TERMS = ["Index", "Share", "Value", "Volume"] + def test_variable_ops_as_square_brackets(): # Check that variables use square brackets for operations @@ -9,7 +11,7 @@ def test_variable_ops_as_square_brackets(): error = [] for variable in dsd.variable: - if reserved_terms := [r for r in ["Index", "Share"] if "|" + r in variable]: + if reserved_terms := [r for r in RESERVED_TERMS if "|" + r in variable]: error.append(f"Variable '{variable}' -> '... [{''.join(reserved_terms)}]'") if error: raise ValueError( From 25c8edaf475f7a84b3f419cfe9d47e19a9a2e84f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 11 Apr 2024 10:15:00 +0200 Subject: [PATCH 7/8] Add a reference to a nomenclature issue --- tests/test_reserved_terms.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_reserved_terms.py b/tests/test_reserved_terms.py index 544ea283..f2c26759 100644 --- a/tests/test_reserved_terms.py +++ b/tests/test_reserved_terms.py @@ -3,6 +3,10 @@ RESERVED_TERMS = ["Index", "Share", "Value", "Volume"] +# TODO: Move this test to the nomenclature `validate-project` utility +# see https://github.com/IAMconsortium/nomenclature/issues/341 + + def test_variable_ops_as_square_brackets(): # Check that variables use square brackets for operations # https://github.com/IAMconsortium/common-definitions/issues/55 From 69948d34032e84324f90bd4e3b8f79fdde37b96c Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 11 Apr 2024 10:19:06 +0200 Subject: [PATCH 8/8] Add SHAPE to list of legacy projects for duplicate-validation --- tests/test_legacy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_legacy.py b/tests/test_legacy.py index 5c458e90..eecb39cd 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -1,5 +1,7 @@ from nomenclature import DataStructureDefinition +LEGACY_PROJECTS = ["navigate", "engage", "shape"] + def test_legacy_variables(): # Check that (new) variables are not referenced as deprecated legacy variables @@ -8,7 +10,7 @@ def test_legacy_variables(): legacy_variables = {} for code, attrs in dsd.variable.items(): - for project in ["navigate", "engage"]: + for project in LEGACY_PROJECTS: if project in attrs.extra_attributes: legacy_var = attrs.__getattr__(project) if legacy_var in existing_variables: