From 12b2421a3bec86189e775fac8de84d8068ac86d3 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Fri, 2 Aug 2024 13:39:26 +0200 Subject: [PATCH 1/6] Fix and test indexset list and tabulate for specific runs --- ixmp4/core/optimization/indexset.py | 8 +++++-- tests/core/test_indexset.py | 23 +++++++++++++++---- tests/data/test_optimization_indexset.py | 29 ++++++++++++++++++++---- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/ixmp4/core/optimization/indexset.py b/ixmp4/core/optimization/indexset.py index 163e82ee..9e768a7a 100644 --- a/ixmp4/core/optimization/indexset.py +++ b/ixmp4/core/optimization/indexset.py @@ -94,7 +94,9 @@ def get(self, name: str) -> IndexSet: return IndexSet(_backend=self.backend, _model=indexset) def list(self, name: str | None = None) -> list[IndexSet]: - indexsets = self.backend.optimization.indexsets.list(name=name) + indexsets = self.backend.optimization.indexsets.list( + run_id=self._run.id, name=name + ) return [ IndexSet( _backend=self.backend, @@ -104,4 +106,6 @@ def list(self, name: str | None = None) -> list[IndexSet]: ] def tabulate(self, name: str | None = None) -> pd.DataFrame: - return self.backend.optimization.indexsets.tabulate(name=name) + return self.backend.optimization.indexsets.tabulate( + run_id=self._run.id, name=name + ) diff --git a/tests/core/test_indexset.py b/tests/core/test_indexset.py index 0260f191..4748fdcc 100644 --- a/tests/core/test_indexset.py +++ b/tests/core/test_indexset.py @@ -51,6 +51,7 @@ def test_create_indexset(self, test_mp, request): def test_get_indexset(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.runs.create("Model", "Scenario") + run.set_as_default() _ = run.optimization.indexsets.create("IndexSet 1") indexset = run.optimization.indexsets.get("IndexSet 1") assert indexset.id == 1 @@ -90,8 +91,6 @@ def test_add_elements(self, test_mp, request): def test_list_indexsets(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.runs.create("Model", "Scenario") - # Per default, list() lists only `default` version runs: - run.set_as_default() indexset_1 = run.optimization.indexsets.create("Indexset 1") indexset_2 = run.optimization.indexsets.create("Indexset 2") expected_ids = [indexset_1.id, indexset_2.id] @@ -106,11 +105,17 @@ def test_list_indexsets(self, test_mp, request): ] assert not (set(expected_id) ^ set(list_id)) + # Test that only indexsets belonging to this Run are listed + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 1") + indexset_4 = run_2.optimization.indexsets.create("Indexset 2") + expected_ids = [indexset_3.id, indexset_4.id] + list_ids = [indexset.id for indexset in run_2.optimization.indexsets.list()] + assert not (set(expected_ids) ^ set(list_ids)) + def test_tabulate_indexsets(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.runs.create("Model", "Scenario") - # Per default, tabulate() lists only `default` version runs: - run.set_as_default() indexset_1 = run.optimization.indexsets.create("Indexset 1") indexset_2 = run.optimization.indexsets.create("Indexset 2") expected = df_from_list(indexsets=[indexset_1, indexset_2]) @@ -123,6 +128,16 @@ def test_tabulate_indexsets(self, test_mp, request): result = run.optimization.indexsets.tabulate(name="Indexset 2") pdt.assert_frame_equal(expected, result) + # Test that only IndexSets belonging to this Run are tabulated + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 1") + indexset_4 = run_2.optimization.indexsets.create("Indexset 2") + expected = df_from_list(indexsets=[indexset_3, indexset_4]) + result = run_2.optimization.indexsets.tabulate() + # utils.assert_unordered_equality doesn't like lists, so make sure the order in + # df_from_list() is correct! + pdt.assert_frame_equal(expected, result) + def test_indexset_docs(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.runs.create("Model", "Scenario") diff --git a/tests/data/test_optimization_indexset.py b/tests/data/test_optimization_indexset.py index 5c99a2d9..6f6ee96c 100644 --- a/tests/data/test_optimization_indexset.py +++ b/tests/data/test_optimization_indexset.py @@ -71,8 +71,6 @@ def test_get_indexset(self, test_mp, request): def test_list_indexsets(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, list() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) indexset_1 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset 1" ) @@ -84,11 +82,21 @@ def test_list_indexsets(self, test_mp, request): ) assert [indexset_1, indexset_2] == test_mp.backend.optimization.indexsets.list() + # Test only indexsets belonging to this Run are listed when run_id is provided + run_2 = test_mp.backend.runs.create("Model", "Scenario") + indexset_3 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 1" + ) + indexset_4 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 2" + ) + assert [indexset_3, indexset_4] == test_mp.backend.optimization.indexsets.list( + run_id=run_2.id + ) + def test_tabulate_indexsets(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, tabulate() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) indexset_1 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset 1" ) @@ -118,6 +126,19 @@ def test_tabulate_indexsets(self, test_mp, request): expected, test_mp.backend.optimization.indexsets.tabulate(name="Indexset 1") ) + # Test only indexsets belonging to this Run are tabulated if run_id is provided + run_2 = test_mp.backend.runs.create("Model", "Scenario") + indexset_3 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 1" + ) + indexset_4 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 2" + ) + expected = df_from_list(indexsets=[indexset_3, indexset_4]) + pdt.assert_frame_equal( + expected, test_mp.backend.optimization.indexsets.tabulate(run_id=run_2.id) + ) + def test_add_elements(self, test_mp, request): test_mp = request.getfixturevalue(test_mp) test_elements = ["foo", "bar"] From c306efc6bf38a98f6f5f59aac5f73b54c2095b0f Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Fri, 2 Aug 2024 14:01:55 +0200 Subject: [PATCH 2/6] Fix and test scalar list and tabulate for specific runs --- ixmp4/core/optimization/scalar.py | 4 ++- tests/core/test_scalar.py | 42 +++++++++++++++++++------- tests/data/test_optimization_scalar.py | 41 +++++++++++++++++++------ 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/ixmp4/core/optimization/scalar.py b/ixmp4/core/optimization/scalar.py index 6c8b36d7..b4262d80 100644 --- a/ixmp4/core/optimization/scalar.py +++ b/ixmp4/core/optimization/scalar.py @@ -137,4 +137,6 @@ def list(self, name: str | None = None) -> Iterable[Scalar]: ] def tabulate(self, name: str | None = None) -> pd.DataFrame: - return self.backend.optimization.scalars.tabulate(name=name) + return self.backend.optimization.scalars.tabulate( + run_id=self._run.id, name=name + ) diff --git a/tests/core/test_scalar.py b/tests/core/test_scalar.py index a964ccf5..0ecbd6c2 100644 --- a/tests/core/test_scalar.py +++ b/tests/core/test_scalar.py @@ -1,7 +1,7 @@ import pandas as pd import pytest -from ixmp4 import Scalar +from ixmp4.core import Platform, Scalar from ..utils import all_platforms, assert_unordered_equality @@ -35,7 +35,7 @@ def df_from_list(scalars: list[Scalar]): @all_platforms class TestCoreScalar: def test_create_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") unit = test_mp.units.create("Test Unit") scalar_1 = run.optimization.scalars.create( @@ -63,7 +63,7 @@ def test_create_scalar(self, test_mp, request): assert scalar_3.unit.name == "" def test_get_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") unit = test_mp.units.create("Test Unit") scalar = run.optimization.scalars.create("Scalar", value=10, unit=unit.name) @@ -77,7 +77,7 @@ def test_get_scalar(self, test_mp, request): _ = run.optimization.scalars.get("Foo") def test_update_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") unit = test_mp.units.create("Test Unit") unit2 = test_mp.units.create("Test Unit 2") @@ -100,10 +100,8 @@ def test_update_scalar(self, test_mp, request): assert scalar.unit.id == result.unit.id == 1 def test_list_scalars(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - # Per default, list() lists only `default` version runs: - run.set_as_default() unit = test_mp.units.create("Test Unit") scalar_1 = run.optimization.scalars.create( "Scalar 1", value=1, unit="Test Unit" @@ -120,11 +118,21 @@ def test_list_scalars(self, test_mp, request): ] assert not (set(expected_id) ^ set(list_id)) + # Test that only Scalars belonging to this Run are listed + run_2 = test_mp.runs.create("Model", "Scenario") + scalar_3 = run_2.optimization.scalars.create( + "Scalar 1", value=1, unit="Test Unit" + ) + scalar_4 = run_2.optimization.scalars.create( + "Scalar 2", value=2, unit=unit.name + ) + expected_ids = [scalar_3.id, scalar_4.id] + list_ids = [scalar.id for scalar in run_2.optimization.scalars.list()] + assert not (set(expected_ids) ^ set(list_ids)) + def test_tabulate_scalars(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - # Per default, tabulate() lists only `default` version runs: - run.set_as_default() unit = test_mp.units.create("Test Unit") scalar_1 = run.optimization.scalars.create("Scalar 1", value=1, unit=unit.name) scalar_2 = run.optimization.scalars.create("Scalar 2", value=2, unit=unit.name) @@ -136,8 +144,20 @@ def test_tabulate_scalars(self, test_mp, request): result = run.optimization.scalars.tabulate(name="Scalar 2") assert_unordered_equality(expected, result, check_dtype=False) + # Test that only Scalars belonging to this Run are tabulated + run_2 = test_mp.runs.create("Model", "Scenario") + scalar_3 = run_2.optimization.scalars.create( + "Scalar 1", value=1, unit=unit.name + ) + scalar_4 = run_2.optimization.scalars.create( + "Scalar 2", value=2, unit=unit.name + ) + expected = df_from_list(scalars=[scalar_3, scalar_4]) + result = run_2.optimization.scalars.tabulate() + assert_unordered_equality(expected, result, check_dtype=False) + def test_scalar_docs(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") unit = test_mp.units.create("Test Unit") scalar = run.optimization.scalars.create("Scalar 1", value=4, unit=unit.name) diff --git a/tests/data/test_optimization_scalar.py b/tests/data/test_optimization_scalar.py index 1f9491f1..10c73554 100644 --- a/tests/data/test_optimization_scalar.py +++ b/tests/data/test_optimization_scalar.py @@ -2,7 +2,7 @@ import pandas.testing as pdt import pytest -from ixmp4 import Scalar +from ixmp4.core import Platform, Scalar from ..utils import all_platforms @@ -36,7 +36,7 @@ def df_from_list(scalars: list): @all_platforms class TestDataOptimizationScalar: def test_create_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") unit = test_mp.backend.units.create("Unit") unit2 = test_mp.backend.units.create("Unit 2") @@ -54,7 +54,7 @@ def test_create_scalar(self, test_mp, request): ) def test_get_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") unit = test_mp.backend.units.create("Unit") scalar = test_mp.backend.optimization.scalars.create( @@ -68,7 +68,7 @@ def test_get_scalar(self, test_mp, request): _ = test_mp.backend.optimization.scalars.get(run_id=run.id, name="Scalar 2") def test_update_scalar(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") unit = test_mp.backend.units.create("Unit") unit2 = test_mp.backend.units.create("Unit 2") @@ -87,10 +87,8 @@ def test_update_scalar(self, test_mp, request): assert ret.value == 20 def test_list_scalars(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, list() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) unit = test_mp.backend.units.create("Unit") unit2 = test_mp.backend.units.create("Unit 2") scalar_1 = test_mp.backend.optimization.scalars.create( @@ -102,11 +100,21 @@ def test_list_scalars(self, test_mp, request): assert [scalar_1] == test_mp.backend.optimization.scalars.list(name="Scalar") assert [scalar_1, scalar_2] == test_mp.backend.optimization.scalars.list() + # Test listing of scalars of particular run only + run_2 = test_mp.backend.runs.create("Model", "Scenario") + scalar_3 = test_mp.backend.optimization.scalars.create( + run_id=run_2.id, name="Scalar", value=1, unit_name=unit.name + ) + scalar_4 = test_mp.backend.optimization.scalars.create( + run_id=run_2.id, name="Scalar 2", value=2, unit_name=unit2.name + ) + assert [scalar_3, scalar_4] == test_mp.backend.optimization.scalars.list( + run_id=run_2.id + ) + def test_tabulate_scalars(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, tabulate() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) unit = test_mp.backend.units.create("Unit") unit2 = test_mp.backend.units.create("Unit 2") scalar_1 = test_mp.backend.optimization.scalars.create( @@ -124,3 +132,16 @@ def test_tabulate_scalars(self, test_mp, request): pdt.assert_frame_equal( expected, test_mp.backend.optimization.scalars.tabulate(name="Scalar") ) + + # Test tabulation of scalars of particular run only + run_2 = test_mp.backend.runs.create("Model", "Scenario") + scalar_3 = test_mp.backend.optimization.scalars.create( + run_id=run_2.id, name="Scalar", value=1, unit_name=unit.name + ) + scalar_4 = test_mp.backend.optimization.scalars.create( + run_id=run_2.id, name="Scalar 2", value=2, unit_name=unit2.name + ) + expected = df_from_list(scalars=[scalar_3, scalar_4]) + pdt.assert_frame_equal( + expected, test_mp.backend.optimization.scalars.tabulate(run_id=run_2.id) + ) From 35ec82d908fcf9402ce3f96d2052caef19778d4d Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 5 Aug 2024 09:54:55 +0200 Subject: [PATCH 3/6] Fix and test table list and tabulate for specific runs --- ixmp4/core/optimization/table.py | 2 +- tests/core/test_table.py | 35 +++++++++++++++++--- tests/data/test_optimization_table.py | 47 +++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/ixmp4/core/optimization/table.py b/ixmp4/core/optimization/table.py index 065d6793..dad74e6d 100644 --- a/ixmp4/core/optimization/table.py +++ b/ixmp4/core/optimization/table.py @@ -116,4 +116,4 @@ def list(self, name: str | None = None) -> Iterable[Table]: ] def tabulate(self, name: str | None = None) -> pd.DataFrame: - return self.backend.optimization.tables.tabulate(name=name) + return self.backend.optimization.tables.tabulate(run_id=self._run.id, name=name) diff --git a/tests/core/test_table.py b/tests/core/test_table.py index ac923f4d..16661857 100644 --- a/tests/core/test_table.py +++ b/tests/core/test_table.py @@ -1,7 +1,7 @@ import pandas as pd import pytest -from ixmp4 import Platform, Table +from ixmp4.core import Platform, Table from ..utils import all_platforms @@ -216,8 +216,6 @@ def test_table_add_data(self, test_mp, request): def test_list_tables(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - # Per default, list() lists scalars for `default` version runs: - run.set_as_default() _ = run.optimization.indexsets.create("Indexset") _ = run.optimization.indexsets.create("Indexset 2") table = run.optimization.tables.create( @@ -235,11 +233,23 @@ def test_list_tables(self, test_mp, request): list_id = [table.id for table in run.optimization.tables.list(name="Table")] assert not (set(expected_id) ^ set(list_id)) + # Test that only Tables belonging to this Run are listed + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 3") + indexset_4 = run_2.optimization.indexsets.create("Indexset 4") + table_3 = run_2.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset_3.name, indexset_4.name] + ) + table_4 = run_2.optimization.tables.create( + "Table 2", constrained_to_indexsets=[indexset_3.name, indexset_4.name] + ) + expected_ids = [table_3.id, table_4.id] + list_ids = [table.id for table in run_2.optimization.tables.list()] + assert not (set(expected_ids) ^ set(list_ids)) + def test_tabulate_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - # Per default, tabulate() lists scalars for `default` version runs: - run.set_as_default() indexset = run.optimization.indexsets.create("Indexset") indexset_2 = run.optimization.indexsets.create("Indexset 2") table = run.optimization.tables.create( @@ -266,6 +276,21 @@ def test_tabulate_table(self, test_mp, request): run.optimization.tables.tabulate(), ) + # Test that only Tables belonging to this Run are listed + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 3") + indexset_4 = run_2.optimization.indexsets.create("Indexset 4") + table_3 = run_2.optimization.tables.create( + "Table", constrained_to_indexsets=[indexset_3.name, indexset_4.name] + ) + table_4 = run_2.optimization.tables.create( + "Table 2", constrained_to_indexsets=[indexset_3.name, indexset_4.name] + ) + pd.testing.assert_frame_equal( + df_from_list([table_3, table_4]), + run_2.optimization.tables.tabulate(), + ) + def test_table_docs(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") diff --git a/tests/data/test_optimization_table.py b/tests/data/test_optimization_table.py index d2b3c49c..c25f31d8 100644 --- a/tests/data/test_optimization_table.py +++ b/tests/data/test_optimization_table.py @@ -1,7 +1,7 @@ import pandas as pd import pytest -from ixmp4 import Platform, Table +from ixmp4.core import Platform, Table from ..utils import all_platforms @@ -101,7 +101,7 @@ def test_create_table(self, test_mp, request): assert table_3.columns[1].dtype == "int64" def test_get_table(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") _ = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset" @@ -267,10 +267,8 @@ def test_table_add_data(self, test_mp, request): assert table_5.data == test_data_5 def test_list_table(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, list() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) _ = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset" ) @@ -287,11 +285,27 @@ def test_list_table(self, test_mp, request): assert [table] == test_mp.backend.optimization.tables.list(name="Table") + # Test listing of Tables when specifying a Run + run_2 = test_mp.backend.runs.create("Model", "Scenario") + indexset_3 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 3" + ) + indexset_4 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 4" + ) + table_3 = test_mp.backend.optimization.tables.create( + run_id=run_2.id, name="Table", constrained_to_indexsets=[indexset_3.name] + ) + table_4 = test_mp.backend.optimization.tables.create( + run_id=run_2.id, name="Table 2", constrained_to_indexsets=[indexset_4.name] + ) + assert [table_3, table_4] == test_mp.backend.optimization.tables.list( + run_id=run_2.id + ) + def test_tabulate_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - # Per default, tabulate() lists scalars for `default` version runs: - test_mp.backend.runs.set_as_default_version(run.id) indexset = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset" ) @@ -334,3 +348,22 @@ def test_tabulate_table(self, test_mp, request): df_from_list([table, table_2]), test_mp.backend.optimization.tables.tabulate(), ) + + # Test tabulation of Tables when specifying a Run + run_2 = test_mp.backend.runs.create("Model", "Scenario") + indexset_3 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 3" + ) + indexset_4 = test_mp.backend.optimization.indexsets.create( + run_id=run_2.id, name="Indexset 4" + ) + table_3 = test_mp.backend.optimization.tables.create( + run_id=run_2.id, name="Table", constrained_to_indexsets=[indexset_3.name] + ) + table_4 = test_mp.backend.optimization.tables.create( + run_id=run_2.id, name="Table 2", constrained_to_indexsets=[indexset_4.name] + ) + pd.testing.assert_frame_equal( + df_from_list([table_3, table_4]), + test_mp.backend.optimization.tables.tabulate(run_id=run_2.id), + ) From 85c5debc1cd40c296c40dd23d20e20e5f04bc816 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 8 Aug 2024 08:42:59 +0200 Subject: [PATCH 4/6] Make new tests more efficient --- tests/core/test_indexset.py | 42 ++++++++++------------- tests/core/test_scalar.py | 34 ++++++------------- tests/core/test_table.py | 43 ++++++++---------------- tests/data/test_optimization_indexset.py | 12 +++---- 4 files changed, 47 insertions(+), 84 deletions(-) diff --git a/tests/core/test_indexset.py b/tests/core/test_indexset.py index 4748fdcc..d485f6dd 100644 --- a/tests/core/test_indexset.py +++ b/tests/core/test_indexset.py @@ -2,7 +2,7 @@ import pandas.testing as pdt import pytest -from ixmp4 import IndexSet +from ixmp4.core import IndexSet, Platform from ..utils import all_platforms @@ -36,7 +36,7 @@ def df_from_list(indexsets: list[IndexSet]): @all_platforms class TestCoreIndexSet: def test_create_indexset(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") indexset_1 = run.optimization.indexsets.create("IndexSet 1") assert indexset_1.id == 1 @@ -49,7 +49,7 @@ def test_create_indexset(self, test_mp, request): _ = run.optimization.indexsets.create("IndexSet 1") def test_get_indexset(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") run.set_as_default() _ = run.optimization.indexsets.create("IndexSet 1") @@ -61,7 +61,7 @@ def test_get_indexset(self, test_mp, request): _ = run.optimization.indexsets.get("Foo") def test_add_elements(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") test_elements = ["foo", "bar"] indexset_1 = run.optimization.indexsets.create("IndexSet 1") @@ -89,10 +89,15 @@ def test_add_elements(self, test_mp, request): assert indexset_5.elements == test_elements_2 def test_list_indexsets(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") indexset_1 = run.optimization.indexsets.create("Indexset 1") indexset_2 = run.optimization.indexsets.create("Indexset 2") + # Create indexset in another run to test listing indexsets for specific run + test_mp.runs.create("Model", "Scenario").optimization.indexsets.create( + "Indexset 1" + ) + expected_ids = [indexset_1.id, indexset_2.id] list_ids = [indexset.id for indexset in run.optimization.indexsets.list()] assert not (set(expected_ids) ^ set(list_ids)) @@ -105,19 +110,16 @@ def test_list_indexsets(self, test_mp, request): ] assert not (set(expected_id) ^ set(list_id)) - # Test that only indexsets belonging to this Run are listed - run_2 = test_mp.runs.create("Model", "Scenario") - indexset_3 = run_2.optimization.indexsets.create("Indexset 1") - indexset_4 = run_2.optimization.indexsets.create("Indexset 2") - expected_ids = [indexset_3.id, indexset_4.id] - list_ids = [indexset.id for indexset in run_2.optimization.indexsets.list()] - assert not (set(expected_ids) ^ set(list_ids)) - def test_tabulate_indexsets(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") indexset_1 = run.optimization.indexsets.create("Indexset 1") indexset_2 = run.optimization.indexsets.create("Indexset 2") + # Create indexset in another run to test tabulating indexsets for specific run + test_mp.runs.create("Model", "Scenario").optimization.indexsets.create( + "Indexset 1" + ) + expected = df_from_list(indexsets=[indexset_1, indexset_2]) result = run.optimization.indexsets.tabulate() # utils.assert_unordered_equality doesn't like lists, so make sure the order in @@ -128,18 +130,8 @@ def test_tabulate_indexsets(self, test_mp, request): result = run.optimization.indexsets.tabulate(name="Indexset 2") pdt.assert_frame_equal(expected, result) - # Test that only IndexSets belonging to this Run are tabulated - run_2 = test_mp.runs.create("Model", "Scenario") - indexset_3 = run_2.optimization.indexsets.create("Indexset 1") - indexset_4 = run_2.optimization.indexsets.create("Indexset 2") - expected = df_from_list(indexsets=[indexset_3, indexset_4]) - result = run_2.optimization.indexsets.tabulate() - # utils.assert_unordered_equality doesn't like lists, so make sure the order in - # df_from_list() is correct! - pdt.assert_frame_equal(expected, result) - def test_indexset_docs(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") indexset_1 = run.optimization.indexsets.create("IndexSet 1") docs = "Documentation of IndexSet 1" diff --git a/tests/core/test_scalar.py b/tests/core/test_scalar.py index 0ecbd6c2..58cee876 100644 --- a/tests/core/test_scalar.py +++ b/tests/core/test_scalar.py @@ -107,6 +107,11 @@ def test_list_scalars(self, test_mp, request): "Scalar 1", value=1, unit="Test Unit" ) scalar_2 = run.optimization.scalars.create("Scalar 2", value=2, unit=unit.name) + # Create scalar in another run to test listing scalars for specific run + test_mp.runs.create("Model", "Scenario").optimization.scalars.create( + "Scalar 1", value=1, unit=unit + ) + expected_ids = [scalar_1.id, scalar_2.id] list_ids = [scalar.id for scalar in run.optimization.scalars.list()] assert not (set(expected_ids) ^ set(list_ids)) @@ -118,24 +123,17 @@ def test_list_scalars(self, test_mp, request): ] assert not (set(expected_id) ^ set(list_id)) - # Test that only Scalars belonging to this Run are listed - run_2 = test_mp.runs.create("Model", "Scenario") - scalar_3 = run_2.optimization.scalars.create( - "Scalar 1", value=1, unit="Test Unit" - ) - scalar_4 = run_2.optimization.scalars.create( - "Scalar 2", value=2, unit=unit.name - ) - expected_ids = [scalar_3.id, scalar_4.id] - list_ids = [scalar.id for scalar in run_2.optimization.scalars.list()] - assert not (set(expected_ids) ^ set(list_ids)) - def test_tabulate_scalars(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") unit = test_mp.units.create("Test Unit") scalar_1 = run.optimization.scalars.create("Scalar 1", value=1, unit=unit.name) scalar_2 = run.optimization.scalars.create("Scalar 2", value=2, unit=unit.name) + # Create scalar in another run to test tabulating scalars for specific run + test_mp.runs.create("Model", "Scenario").optimization.scalars.create( + "Scalar 1", value=1, unit=unit + ) + expected = df_from_list(scalars=[scalar_1, scalar_2]) result = run.optimization.scalars.tabulate() assert_unordered_equality(expected, result, check_dtype=False) @@ -144,18 +142,6 @@ def test_tabulate_scalars(self, test_mp, request): result = run.optimization.scalars.tabulate(name="Scalar 2") assert_unordered_equality(expected, result, check_dtype=False) - # Test that only Scalars belonging to this Run are tabulated - run_2 = test_mp.runs.create("Model", "Scenario") - scalar_3 = run_2.optimization.scalars.create( - "Scalar 1", value=1, unit=unit.name - ) - scalar_4 = run_2.optimization.scalars.create( - "Scalar 2", value=2, unit=unit.name - ) - expected = df_from_list(scalars=[scalar_3, scalar_4]) - result = run_2.optimization.scalars.tabulate() - assert_unordered_equality(expected, result, check_dtype=False) - def test_scalar_docs(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") diff --git a/tests/core/test_table.py b/tests/core/test_table.py index 16661857..3691a017 100644 --- a/tests/core/test_table.py +++ b/tests/core/test_table.py @@ -224,6 +224,13 @@ def test_list_tables(self, test_mp, request): table_2 = run.optimization.tables.create( "Table 2", constrained_to_indexsets=["Indexset 2"] ) + # Create table in another run to test listing tables for specific run + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 3") + run_2.optimization.tables.create( + "Table 1", constrained_to_indexsets=[indexset_3.name] + ) + expected_ids = [table.id, table_2.id] list_ids = [table.id for table in run.optimization.tables.list()] assert not (set(expected_ids) ^ set(list_ids)) @@ -233,20 +240,6 @@ def test_list_tables(self, test_mp, request): list_id = [table.id for table in run.optimization.tables.list(name="Table")] assert not (set(expected_id) ^ set(list_id)) - # Test that only Tables belonging to this Run are listed - run_2 = test_mp.runs.create("Model", "Scenario") - indexset_3 = run_2.optimization.indexsets.create("Indexset 3") - indexset_4 = run_2.optimization.indexsets.create("Indexset 4") - table_3 = run_2.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset_3.name, indexset_4.name] - ) - table_4 = run_2.optimization.tables.create( - "Table 2", constrained_to_indexsets=[indexset_3.name, indexset_4.name] - ) - expected_ids = [table_3.id, table_4.id] - list_ids = [table.id for table in run_2.optimization.tables.list()] - assert not (set(expected_ids) ^ set(list_ids)) - def test_tabulate_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") @@ -260,6 +253,13 @@ def test_tabulate_table(self, test_mp, request): name="Table 2", constrained_to_indexsets=["Indexset", "Indexset 2"], ) + # Create table in another run to test listing tables for specific run + run_2 = test_mp.runs.create("Model", "Scenario") + indexset_3 = run_2.optimization.indexsets.create("Indexset 3") + run_2.optimization.tables.create( + "Table 1", constrained_to_indexsets=[indexset_3.name] + ) + pd.testing.assert_frame_equal( df_from_list([table_2]), run.optimization.tables.tabulate(name="Table 2"), @@ -276,21 +276,6 @@ def test_tabulate_table(self, test_mp, request): run.optimization.tables.tabulate(), ) - # Test that only Tables belonging to this Run are listed - run_2 = test_mp.runs.create("Model", "Scenario") - indexset_3 = run_2.optimization.indexsets.create("Indexset 3") - indexset_4 = run_2.optimization.indexsets.create("Indexset 4") - table_3 = run_2.optimization.tables.create( - "Table", constrained_to_indexsets=[indexset_3.name, indexset_4.name] - ) - table_4 = run_2.optimization.tables.create( - "Table 2", constrained_to_indexsets=[indexset_3.name, indexset_4.name] - ) - pd.testing.assert_frame_equal( - df_from_list([table_3, table_4]), - run_2.optimization.tables.tabulate(), - ) - def test_table_docs(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") diff --git a/tests/data/test_optimization_indexset.py b/tests/data/test_optimization_indexset.py index 6f6ee96c..21613062 100644 --- a/tests/data/test_optimization_indexset.py +++ b/tests/data/test_optimization_indexset.py @@ -2,7 +2,7 @@ import pandas.testing as pdt import pytest -from ixmp4 import IndexSet +from ixmp4.core import IndexSet, Platform from ..utils import all_platforms @@ -36,7 +36,7 @@ def df_from_list(indexsets: list): @all_platforms class TestDataOptimizationIndexSet: def test_create_indexset(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") indexset_1 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset" @@ -51,7 +51,7 @@ def test_create_indexset(self, test_mp, request): ) def test_get_indexset(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") _ = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset" @@ -69,7 +69,7 @@ def test_get_indexset(self, test_mp, request): ) def test_list_indexsets(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") indexset_1 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset 1" @@ -95,7 +95,7 @@ def test_list_indexsets(self, test_mp, request): ) def test_tabulate_indexsets(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") indexset_1 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset 1" @@ -140,7 +140,7 @@ def test_tabulate_indexsets(self, test_mp, request): ) def test_add_elements(self, test_mp, request): - test_mp = request.getfixturevalue(test_mp) + test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore test_elements = ["foo", "bar"] run = test_mp.backend.runs.create("Model", "Scenario") indexset_1 = test_mp.backend.optimization.indexsets.create( From 9592abdffc0fb563823f697c308a0d88716ed63b Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Thu, 8 Aug 2024 11:03:09 +0200 Subject: [PATCH 5/6] Make indexset-creation a test utility --- tests/core/test_indexset.py | 49 ++++++++------ tests/core/test_table.py | 70 ++++++++++++-------- tests/data/test_optimization_indexset.py | 63 +++++++----------- tests/data/test_optimization_scalar.py | 3 +- tests/data/test_optimization_table.py | 84 ++++++++++-------------- tests/utils.py | 15 +++++ 6 files changed, 145 insertions(+), 139 deletions(-) diff --git a/tests/core/test_indexset.py b/tests/core/test_indexset.py index d485f6dd..a887730b 100644 --- a/tests/core/test_indexset.py +++ b/tests/core/test_indexset.py @@ -4,7 +4,7 @@ from ixmp4.core import IndexSet, Platform -from ..utils import all_platforms +from ..utils import all_platforms, create_indexsets_for_run def df_from_list(indexsets: list[IndexSet]): @@ -38,24 +38,24 @@ class TestCoreIndexSet: def test_create_indexset(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset_1 = run.optimization.indexsets.create("IndexSet 1") + indexset_1 = run.optimization.indexsets.create("Indexset 1") assert indexset_1.id == 1 - assert indexset_1.name == "IndexSet 1" + assert indexset_1.name == "Indexset 1" - indexset_2 = run.optimization.indexsets.create("IndexSet 2") + indexset_2 = run.optimization.indexsets.create("Indexset 2") assert indexset_1.id != indexset_2.id with pytest.raises(IndexSet.NotUnique): - _ = run.optimization.indexsets.create("IndexSet 1") + _ = run.optimization.indexsets.create("Indexset 1") def test_get_indexset(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") run.set_as_default() - _ = run.optimization.indexsets.create("IndexSet 1") - indexset = run.optimization.indexsets.get("IndexSet 1") + create_indexsets_for_run(platform=test_mp, run_id=run.id, amount=1) + indexset = run.optimization.indexsets.get("Indexset 1") assert indexset.id == 1 - assert indexset.name == "IndexSet 1" + assert indexset.name == "Indexset 1" with pytest.raises(IndexSet.NotFound): _ = run.optimization.indexsets.get("Foo") @@ -64,10 +64,10 @@ def test_add_elements(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") test_elements = ["foo", "bar"] - indexset_1 = run.optimization.indexsets.create("IndexSet 1") + indexset_1 = run.optimization.indexsets.create("Indexset 1") indexset_1.add(test_elements) - run.optimization.indexsets.create("IndexSet 2").add(test_elements) - indexset_2 = run.optimization.indexsets.get("IndexSet 2") + run.optimization.indexsets.create("Indexset 2").add(test_elements) + indexset_2 = run.optimization.indexsets.get("Indexset 2") assert indexset_1.elements == indexset_2.elements with pytest.raises(ValueError): @@ -77,22 +77,23 @@ def test_add_elements(self, test_mp, request): indexset_2.add(["baz", "baz"]) indexset_1.add(1) - indexset_3 = run.optimization.indexsets.get("IndexSet 1") + indexset_3 = run.optimization.indexsets.get("Indexset 1") indexset_2.add("1") - indexset_4 = run.optimization.indexsets.get("IndexSet 2") + indexset_4 = run.optimization.indexsets.get("Indexset 2") assert indexset_3.elements != indexset_4.elements assert len(indexset_3.elements) == len(indexset_4.elements) test_elements_2 = ["One", 2, 3.141] - indexset_5 = run.optimization.indexsets.create("IndexSet 5") + indexset_5 = run.optimization.indexsets.create("Indexset 5") indexset_5.add(test_elements_2) assert indexset_5.elements == test_elements_2 def test_list_indexsets(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset_1 = run.optimization.indexsets.create("Indexset 1") - indexset_2 = run.optimization.indexsets.create("Indexset 2") + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id + ) # Create indexset in another run to test listing indexsets for specific run test_mp.runs.create("Model", "Scenario").optimization.indexsets.create( "Indexset 1" @@ -113,8 +114,11 @@ def test_list_indexsets(self, test_mp, request): def test_tabulate_indexsets(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset_1 = run.optimization.indexsets.create("Indexset 1") - indexset_2 = run.optimization.indexsets.create("Indexset 2") + # Convert to core.indexset; util uses db.indexset for efficiency + indexset_1, indexset_2 = tuple( + IndexSet(_backend=test_mp.backend, _model=model) + for model in create_indexsets_for_run(platform=test_mp, run_id=run.id) + ) # Create indexset in another run to test tabulating indexsets for specific run test_mp.runs.create("Model", "Scenario").optimization.indexsets.create( "Indexset 1" @@ -133,8 +137,13 @@ def test_tabulate_indexsets(self, test_mp, request): def test_indexset_docs(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset_1 = run.optimization.indexsets.create("IndexSet 1") - docs = "Documentation of IndexSet 1" + (indexset_1,) = tuple( + IndexSet(_backend=test_mp.backend, _model=model) + for model in create_indexsets_for_run( + platform=test_mp, run_id=run.id, amount=1 + ) + ) + docs = "Documentation of Indexset 1" indexset_1.docs = docs assert indexset_1.docs == docs diff --git a/tests/core/test_table.py b/tests/core/test_table.py index 3691a017..6d9b54f7 100644 --- a/tests/core/test_table.py +++ b/tests/core/test_table.py @@ -1,9 +1,9 @@ import pandas as pd import pytest -from ixmp4.core import Platform, Table +from ixmp4.core import IndexSet, Platform, Table -from ..utils import all_platforms +from ..utils import all_platforms, create_indexsets_for_run def df_from_list(tables: list[Table]): @@ -39,7 +39,10 @@ def test_create_table(self, test_mp, request): run = test_mp.runs.create("Model", "Scenario") # Test normal creation - indexset = run.optimization.indexsets.create("Indexset") + indexset, indexset_2 = tuple( + IndexSet(_backend=test_mp.backend, _model=model) + for model in create_indexsets_for_run(platform=test_mp, run_id=run.id) + ) table = run.optimization.tables.create( "Table 1", constrained_to_indexsets=[indexset.name], @@ -48,20 +51,20 @@ def test_create_table(self, test_mp, request): assert table.id == 1 assert table.name == "Table 1" assert table.data == {} - assert table.columns[0].name == "Indexset" + assert table.columns[0].name == indexset.name assert table.constrained_to_indexsets == [indexset.name] # Test duplicate name raises with pytest.raises(Table.NotUnique): _ = run.optimization.tables.create( - "Table 1", constrained_to_indexsets=["Indexset"] + "Table 1", constrained_to_indexsets=[indexset.name] ) # Test mismatch in constrained_to_indexsets and column_names raises with pytest.raises(ValueError, match="not equal in length"): _ = run.optimization.tables.create( name="Table 2", - constrained_to_indexsets=["Indexset"], + constrained_to_indexsets=[indexset.name], column_names=["Dimension 1", "Dimension 2"], ) @@ -82,11 +85,10 @@ def test_create_table(self, test_mp, request): ) # Test column.dtype is registered correctly - indexset_2 = run.optimization.indexsets.create("Indexset 2") indexset_2.add(elements=2024) table_3 = run.optimization.tables.create( "Table 5", - constrained_to_indexsets=["Indexset", indexset_2.name], + constrained_to_indexsets=[indexset.name, indexset_2.name], ) # If indexset doesn't have elements, a generic dtype is registered assert table_3.columns[0].dtype == "object" @@ -95,9 +97,11 @@ def test_create_table(self, test_mp, request): def test_get_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset = run.optimization.indexsets.create(name="Indexset") + (indexset,) = create_indexsets_for_run( + platform=test_mp, run_id=run.id, amount=1 + ) _ = run.optimization.tables.create( - name="Table", constrained_to_indexsets=["Indexset"] + name="Table", constrained_to_indexsets=[indexset.name] ) table = run.optimization.tables.get("Table") assert table.run_id == run.id @@ -113,16 +117,18 @@ def test_get_table(self, test_mp, request): def test_table_add_data(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset = run.optimization.indexsets.create("Indexset") + indexset, indexset_2 = tuple( + IndexSet(_backend=test_mp.backend, _model=model) + for model in create_indexsets_for_run(platform=test_mp, run_id=run.id) + ) indexset.add(elements=["foo", "bar", ""]) - indexset_2 = run.optimization.indexsets.create("Indexset 2") indexset_2.add([1, 2, 3]) # pandas can only convert dicts to dataframes if the values are lists # or if index is given. But maybe using read_json instead of from_dict # can remedy this. Or maybe we want to catch the resulting # "ValueError: If using all scalar values, you must pass an index" and # reraise a custom informative error? - test_data_1 = {"Indexset": ["foo"], "Indexset 2": [1]} + test_data_1 = {indexset.name: ["foo"], indexset_2.name: [1]} table = run.optimization.tables.create( "Table", constrained_to_indexsets=[indexset.name, indexset_2.name], @@ -137,22 +143,22 @@ def test_table_add_data(self, test_mp, request): with pytest.raises(ValueError, match="missing values"): table_2.add( - pd.DataFrame({"Indexset": [None], "Indexset 2": [2]}), + pd.DataFrame({indexset.name: [None], indexset_2.name: [2]}), # empty string is allowed for now, but None or NaN raise ) with pytest.raises(ValueError, match="contains duplicate rows"): table_2.add( - data={"Indexset": ["foo", "foo"], "Indexset 2": [2, 2]}, + data={indexset.name: ["foo", "foo"], indexset_2.name: [2, 2]}, ) # Test raising on unrecognised data.values() with pytest.raises(ValueError, match="contains values that are not allowed"): table_2.add( - data={"Indexset": ["foo"], "Indexset 2": [0]}, + data={indexset.name: ["foo"], indexset_2.name: [0]}, ) - test_data_2 = {"Indexset": [""], "Indexset 2": [3]} + test_data_2 = {indexset.name: [""], indexset_2.name: [3]} table_2.add(data=test_data_2) assert table_2.data == test_data_2 @@ -195,12 +201,15 @@ def test_table_add_data(self, test_mp, request): # This doesn't seem to test a distinct case compared to the above with pytest.raises(ValueError, match="Trying to add data to unknown Columns!"): table_4.add( - data={"Column 1": ["bar"], "Column 2": [3], "Indexset": ["foo"]}, + data={"Column 1": ["bar"], "Column 2": [3], indexset.name: ["foo"]}, ) # Test various data types - test_data_5 = {"Indexset": ["foo", "foo", "bar"], "Indexset 3": [1, "2", 3.14]} indexset_3 = run.optimization.indexsets.create(name="Indexset 3") + test_data_5 = { + indexset.name: ["foo", "foo", "bar"], + indexset_3.name: [1, "2", 3.14], + } indexset_3.add(elements=[1, "2", 3.14]) table_5 = run.optimization.tables.create( name="Table 5", @@ -216,10 +225,9 @@ def test_table_add_data(self, test_mp, request): def test_list_tables(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - _ = run.optimization.indexsets.create("Indexset") - _ = run.optimization.indexsets.create("Indexset 2") + create_indexsets_for_run(platform=test_mp, run_id=run.id) table = run.optimization.tables.create( - "Table", constrained_to_indexsets=["Indexset"] + "Table", constrained_to_indexsets=["Indexset 1"] ) table_2 = run.optimization.tables.create( "Table 2", constrained_to_indexsets=["Indexset 2"] @@ -243,15 +251,17 @@ def test_list_tables(self, test_mp, request): def test_tabulate_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset = run.optimization.indexsets.create("Indexset") - indexset_2 = run.optimization.indexsets.create("Indexset 2") + indexset, indexset_2 = tuple( + IndexSet(_backend=test_mp.backend, _model=model) + for model in create_indexsets_for_run(platform=test_mp, run_id=run.id) + ) table = run.optimization.tables.create( name="Table", - constrained_to_indexsets=["Indexset", "Indexset 2"], + constrained_to_indexsets=[indexset.name, indexset_2.name], ) table_2 = run.optimization.tables.create( name="Table 2", - constrained_to_indexsets=["Indexset", "Indexset 2"], + constrained_to_indexsets=[indexset.name, indexset_2.name], ) # Create table in another run to test listing tables for specific run run_2 = test_mp.runs.create("Model", "Scenario") @@ -267,9 +277,9 @@ def test_tabulate_table(self, test_mp, request): indexset.add(["foo", "bar"]) indexset_2.add([1, 2, 3]) - test_data_1 = {"Indexset": ["foo"], "Indexset 2": [1]} + test_data_1 = {indexset.name: ["foo"], indexset_2.name: [1]} table.add(test_data_1) - test_data_2 = {"Indexset 2": [2, 3], "Indexset": ["foo", "bar"]} + test_data_2 = {indexset_2.name: [2, 3], indexset.name: ["foo", "bar"]} table_2.add(test_data_2) pd.testing.assert_frame_equal( df_from_list([table, table_2]), @@ -279,7 +289,9 @@ def test_tabulate_table(self, test_mp, request): def test_table_docs(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.runs.create("Model", "Scenario") - indexset = run.optimization.indexsets.create("Indexset") + (indexset,) = create_indexsets_for_run( + platform=test_mp, run_id=run.id, amount=1 + ) table_1 = run.optimization.tables.create( "Table 1", constrained_to_indexsets=[indexset.name] ) diff --git a/tests/data/test_optimization_indexset.py b/tests/data/test_optimization_indexset.py index 21613062..78bc2441 100644 --- a/tests/data/test_optimization_indexset.py +++ b/tests/data/test_optimization_indexset.py @@ -2,9 +2,10 @@ import pandas.testing as pdt import pytest -from ixmp4.core import IndexSet, Platform +from ixmp4.core import Platform +from ixmp4.data.abstract import IndexSet -from ..utils import all_platforms +from ..utils import all_platforms, create_indexsets_for_run def df_from_list(indexsets: list): @@ -53,15 +54,13 @@ def test_create_indexset(self, test_mp, request): def test_get_indexset(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - _ = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset" - ) + create_indexsets_for_run(platform=test_mp, run_id=run.id, amount=1) indexset = test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="Indexset" + run_id=run.id, name="Indexset 1" ) assert indexset.id == 1 assert indexset.run__id == 1 - assert indexset.name == "Indexset" + assert indexset.name == "Indexset 1" with pytest.raises(IndexSet.NotFound): _ = test_mp.backend.optimization.indexsets.get( @@ -71,24 +70,18 @@ def test_get_indexset(self, test_mp, request): def test_list_indexsets(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - indexset_1 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 1" - ) - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id ) assert [indexset_1] == test_mp.backend.optimization.indexsets.list( - name="Indexset 1" + name=indexset_1.name ) assert [indexset_1, indexset_2] == test_mp.backend.optimization.indexsets.list() # Test only indexsets belonging to this Run are listed when run_id is provided run_2 = test_mp.backend.runs.create("Model", "Scenario") - indexset_3 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 1" - ) - indexset_4 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 2" + indexset_3, indexset_4 = create_indexsets_for_run( + platform=test_mp, run_id=run_2.id, offset=2 ) assert [indexset_3, indexset_4] == test_mp.backend.optimization.indexsets.list( run_id=run_2.id @@ -97,11 +90,8 @@ def test_list_indexsets(self, test_mp, request): def test_tabulate_indexsets(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - indexset_1 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 1" - ) - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_1.id, elements="foo" @@ -128,11 +118,8 @@ def test_tabulate_indexsets(self, test_mp, request): # Test only indexsets belonging to this Run are tabulated if run_id is provided run_2 = test_mp.backend.runs.create("Model", "Scenario") - indexset_3 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 1" - ) - indexset_4 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 2" + indexset_3, indexset_4 = create_indexsets_for_run( + platform=test_mp, run_id=run_2.id, offset=2 ) expected = df_from_list(indexsets=[indexset_3, indexset_4]) pdt.assert_frame_equal( @@ -143,19 +130,17 @@ def test_add_elements(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore test_elements = ["foo", "bar"] run = test_mp.backend.runs.create("Model", "Scenario") - indexset_1 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="IndexSet 1" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id ) + test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_1.id, elements=test_elements ) indexset_1 = test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="IndexSet 1" + run_id=run.id, name=indexset_1.name ) - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="IndexSet 2" - ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_2.id, elements=test_elements ) @@ -163,7 +148,7 @@ def test_add_elements(self, test_mp, request): assert ( indexset_1.elements == test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="IndexSet 2" + run_id=run.id, name=indexset_2.name ).elements ) @@ -181,25 +166,25 @@ def test_add_elements(self, test_mp, request): indexset_id=indexset_1.id, elements=1 ) indexset_3 = test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="IndexSet 1" + run_id=run.id, name=indexset_1.name ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_2.id, elements="1" ) indexset_4 = test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="IndexSet 2" + run_id=run.id, name=indexset_2.name ) assert indexset_3.elements != indexset_4.elements assert len(indexset_3.elements) == len(indexset_4.elements) test_elements_2 = [1, "2", 3.14] indexset_5 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="IndexSet 5" + run_id=run.id, name="Indexset 5" ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_5.id, elements=test_elements_2 ) indexset_5 = test_mp.backend.optimization.indexsets.get( - run_id=run.id, name="IndexSet 5" + run_id=run.id, name="Indexset 5" ) assert indexset_5.elements == test_elements_2 diff --git a/tests/data/test_optimization_scalar.py b/tests/data/test_optimization_scalar.py index 10c73554..bb8c827a 100644 --- a/tests/data/test_optimization_scalar.py +++ b/tests/data/test_optimization_scalar.py @@ -2,7 +2,8 @@ import pandas.testing as pdt import pytest -from ixmp4.core import Platform, Scalar +from ixmp4.core import Platform +from ixmp4.data.abstract import Scalar from ..utils import all_platforms diff --git a/tests/data/test_optimization_table.py b/tests/data/test_optimization_table.py index c25f31d8..fc26e2d7 100644 --- a/tests/data/test_optimization_table.py +++ b/tests/data/test_optimization_table.py @@ -1,9 +1,10 @@ import pandas as pd import pytest -from ixmp4.core import Platform, Table +from ixmp4.core import Platform +from ixmp4.data.abstract import Table -from ..utils import all_platforms +from ..utils import all_platforms, create_indexsets_for_run def df_from_list(tables: list): @@ -37,23 +38,23 @@ def test_create_table(self, test_mp, request): run = test_mp.backend.runs.create("Model", "Scenario") # Test normal creation - indexset_1 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id ) table = test_mp.backend.optimization.tables.create( - run_id=run.id, name="Table", constrained_to_indexsets=["Indexset"] + run_id=run.id, name="Table", constrained_to_indexsets=[indexset_1.name] ) assert table.run__id == run.id assert table.name == "Table" assert table.data == {} # JsonDict type currently requires a dict, not None - assert table.columns[0].name == "Indexset" + assert table.columns[0].name == indexset_1.name assert table.columns[0].constrained_to_indexset == indexset_1.id # Test duplicate name raises with pytest.raises(Table.NotUnique): _ = test_mp.backend.optimization.tables.create( - run_id=run.id, name="Table", constrained_to_indexsets=["Indexset"] + run_id=run.id, name="Table", constrained_to_indexsets=[indexset_1.name] ) # Test mismatch in constrained_to_indexsets and column_names raises @@ -61,7 +62,7 @@ def test_create_table(self, test_mp, request): _ = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table 2", - constrained_to_indexsets=["Indexset"], + constrained_to_indexsets=[indexset_1.name], column_names=["Dimension 1", "Dimension 2"], ) @@ -84,9 +85,6 @@ def test_create_table(self, test_mp, request): ) # Test column.dtype is registered correctly - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" - ) test_mp.backend.optimization.indexsets.add_elements( indexset_2.id, elements=2024 ) @@ -94,7 +92,7 @@ def test_create_table(self, test_mp, request): table_3 = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table 5", - constrained_to_indexsets=["Indexset", indexset_2.name], + constrained_to_indexsets=[indexset_1.name, indexset_2.name], ) # If indexset doesn't have elements, a generic dtype is registered assert table_3.columns[0].dtype == "object" @@ -119,15 +117,12 @@ def test_get_table(self, test_mp, request): def test_table_add_data(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - indexset_1 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_1.id, elements=["foo", "bar", ""] ) - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" - ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_2.id, elements=[1, 2, 3] ) @@ -136,7 +131,7 @@ def test_table_add_data(self, test_mp, request): # can remedy this. Or maybe we want to catch the resulting # "ValueError: If using all scalar values, you must pass an index" and # reraise a custom informative error? - test_data_1 = {"Indexset": ["foo"], "Indexset 2": [1]} + test_data_1 = {indexset_1.name: ["foo"], indexset_2.name: [1]} table = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table", @@ -158,24 +153,24 @@ def test_table_add_data(self, test_mp, request): with pytest.raises(ValueError, match="missing values"): test_mp.backend.optimization.tables.add_data( table_id=table_2.id, - data=pd.DataFrame({"Indexset": [None], "Indexset 2": [2]}), + data=pd.DataFrame({indexset_1.name: [None], indexset_2.name: [2]}), # empty string is allowed for now (see below), but None or NaN raise ) with pytest.raises(ValueError, match="contains duplicate rows"): test_mp.backend.optimization.tables.add_data( table_id=table_2.id, - data={"Indexset": ["foo", "foo"], "Indexset 2": [2, 2]}, + data={indexset_1.name: ["foo", "foo"], indexset_2.name: [2, 2]}, ) # Test raising on unrecognised data.values() with pytest.raises(ValueError, match="contains values that are not allowed"): test_mp.backend.optimization.tables.add_data( table_id=table_2.id, - data={"Indexset": ["foo"], "Indexset 2": [0]}, + data={indexset_1.name: ["foo"], indexset_2.name: [0]}, ) - test_data_2 = {"Indexset": [""], "Indexset 2": [3]} + test_data_2 = {indexset_1.name: [""], indexset_2.name: [3]} test_mp.backend.optimization.tables.add_data( table_id=table_2.id, data=test_data_2 ) @@ -243,10 +238,13 @@ def test_table_add_data(self, test_mp, request): ) # Test various data types - test_data_5 = {"Indexset": ["foo", "foo", "bar"], "Indexset 3": [1, "2", 3.14]} indexset_3 = test_mp.backend.optimization.indexsets.create( run_id=run.id, name="Indexset 3" ) + test_data_5 = { + indexset_1.name: ["foo", "foo", "bar"], + indexset_3.name: [1, "2", 3.14], + } test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_3.id, elements=[1, "2", 3.14] ) @@ -269,14 +267,9 @@ def test_table_add_data(self, test_mp, request): def test_list_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - _ = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset" - ) - _ = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" - ) + create_indexsets_for_run(platform=test_mp, run_id=run.id) table = test_mp.backend.optimization.tables.create( - run_id=run.id, name="Table", constrained_to_indexsets=["Indexset"] + run_id=run.id, name="Table", constrained_to_indexsets=["Indexset 1"] ) table_2 = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table 2", constrained_to_indexsets=["Indexset 2"] @@ -287,11 +280,8 @@ def test_list_table(self, test_mp, request): # Test listing of Tables when specifying a Run run_2 = test_mp.backend.runs.create("Model", "Scenario") - indexset_3 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 3" - ) - indexset_4 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 4" + indexset_3, indexset_4 = create_indexsets_for_run( + platform=test_mp, run_id=run_2.id, offset=2 ) table_3 = test_mp.backend.optimization.tables.create( run_id=run_2.id, name="Table", constrained_to_indexsets=[indexset_3.name] @@ -306,21 +296,18 @@ def test_list_table(self, test_mp, request): def test_tabulate_table(self, test_mp, request): test_mp: Platform = request.getfixturevalue(test_mp) # type: ignore run = test_mp.backend.runs.create("Model", "Scenario") - indexset = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset" - ) - indexset_2 = test_mp.backend.optimization.indexsets.create( - run_id=run.id, name="Indexset 2" + indexset_1, indexset_2 = create_indexsets_for_run( + platform=test_mp, run_id=run.id, offset=2 ) table = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table", - constrained_to_indexsets=["Indexset", "Indexset 2"], + constrained_to_indexsets=[indexset_1.name, indexset_2.name], ) table_2 = test_mp.backend.optimization.tables.create( run_id=run.id, name="Table 2", - constrained_to_indexsets=["Indexset", "Indexset 2"], + constrained_to_indexsets=[indexset_1.name, indexset_2.name], ) pd.testing.assert_frame_equal( df_from_list([table_2]), @@ -328,18 +315,18 @@ def test_tabulate_table(self, test_mp, request): ) test_mp.backend.optimization.indexsets.add_elements( - indexset_id=indexset.id, elements=["foo", "bar"] + indexset_id=indexset_1.id, elements=["foo", "bar"] ) test_mp.backend.optimization.indexsets.add_elements( indexset_id=indexset_2.id, elements=[1, 2, 3] ) - test_data_1 = {"Indexset": ["foo"], "Indexset 2": [1]} + test_data_1 = {indexset_1.name: ["foo"], indexset_2.name: [1]} test_mp.backend.optimization.tables.add_data( table_id=table.id, data=test_data_1 ) table = test_mp.backend.optimization.tables.get(run_id=run.id, name="Table") - test_data_2 = {"Indexset 2": [2, 3], "Indexset": ["foo", "bar"]} + test_data_2 = {indexset_2.name: [2, 3], indexset_1.name: ["foo", "bar"]} test_mp.backend.optimization.tables.add_data( table_id=table_2.id, data=test_data_2 ) @@ -351,11 +338,8 @@ def test_tabulate_table(self, test_mp, request): # Test tabulation of Tables when specifying a Run run_2 = test_mp.backend.runs.create("Model", "Scenario") - indexset_3 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 3" - ) - indexset_4 = test_mp.backend.optimization.indexsets.create( - run_id=run_2.id, name="Indexset 4" + indexset_3, indexset_4 = create_indexsets_for_run( + platform=test_mp, run_id=run_2.id, offset=2 ) table_3 = test_mp.backend.optimization.tables.create( run_id=run_2.id, name="Table", constrained_to_indexsets=[indexset_3.name] diff --git a/tests/utils.py b/tests/utils.py index 1dc74f8c..e6da592f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,6 +3,8 @@ import pytest from ixmp4 import DataPoint +from ixmp4.core import Platform +from ixmp4.data.abstract import IndexSet from .conftest import SKIP_PGSQL_TESTS @@ -198,3 +200,16 @@ def create_iamc_query_test_data(test_mp): run2.meta = {"run": 2, "test": "string", "bool": False} return [r1, r2, r3], units + + +def create_indexsets_for_run( + platform: Platform, run_id: int, amount: int = 2, offset: int = 0 +) -> tuple[IndexSet, ...]: + """Create `amount` indexsets called `Indexset n` for `run` (n in (offset, + offset+amount]).""" + return tuple( + platform.backend.optimization.indexsets.create( + run_id=run_id, name=f"Indexset {i+1}" + ) + for i in range(offset, offset + amount) + ) From b2c1debdf49d32e3aa2ab73cb171b5ded969238f Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Fri, 9 Aug 2024 09:04:57 +0200 Subject: [PATCH 6/6] Include final suggestions for new util function --- tests/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index e6da592f..79c8d826 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -203,13 +203,12 @@ def create_iamc_query_test_data(test_mp): def create_indexsets_for_run( - platform: Platform, run_id: int, amount: int = 2, offset: int = 0 + platform: Platform, run_id: int, amount: int = 2, offset: int = 1 ) -> tuple[IndexSet, ...]: - """Create `amount` indexsets called `Indexset n` for `run` (n in (offset, - offset+amount]).""" + """Create `amount` indexsets called `Indexset n` for `run`.""" return tuple( platform.backend.optimization.indexsets.create( - run_id=run_id, name=f"Indexset {i+1}" + run_id=run_id, name=f"Indexset {i}" ) for i in range(offset, offset + amount) )