more performance refactoring

tests/conftest.py

@@ -11,11 +11,12 @@
 from ixmp4.data.backend import RestTestBackend, SqliteTestBackend
 from ixmp4.data.backend.db import PostgresTestBackend
 
-from .fixtures import MediumIamcDataset
+from .fixtures import BigIamcDataset
 
 backend_choices = ("sqlite", "postgres", "rest-sqlite", "rest-postgres")
 backend_fixtures = {
-    "ro_platform_med": ["sqlite", "postgres", "rest-sqlite", "rest-postgres"],
+    "platform_big": ["sqlite", "postgres", "rest-sqlite", "rest-postgres"],
+    "db_platform_big": ["sqlite", "postgres"],
     "platform": ["sqlite", "postgres", "rest-sqlite", "rest-postgres"],
     "db_platform": ["sqlite", "postgres"],
     "rest_platform": ["rest-sqlite", "rest-postgres"],
@@ -105,22 +106,26 @@ def platform_fixture(request):
 sqlite_platform = pytest.fixture(platform_fixture, name="sqlite_platform")
 platform = pytest.fixture(platform_fixture, name="platform")
 
-medium = MediumIamcDataset()
+big = BigIamcDataset()
 
 
-@pytest.fixture(scope="module")
-def ro_platform_med(request):
+def platform_td_big(request):
     type = request.param
     postgres_dsn = request.config.option.postgres_dsn
     bctx = get_backend_context(type, postgres_dsn)
 
     with bctx as backend:
         backend.reset()
         platform = Platform(_backend=backend)
-        medium.load_dataset(platform)
+        big.load_dataset(platform)
         yield platform
 
 
+db_platform_big = pytest.fixture(
+    platform_td_big, scope="session", name="db_platform_big"
+)
+
+
 def pytest_generate_tests(metafunc):
     # This is called for every test. Only get/set command line arguments
     # if the argument is specified in the list of test "fixturenames".

tests/core/test_iamc.py

@@ -7,14 +7,15 @@
 from ixmp4.conf import settings
 from ixmp4.core.exceptions import SchemaError
 
-from ..fixtures import SmallIamcDataset
+from ..fixtures import FilterIamcDataset, MediumIamcDataset, SmallIamcDataset
 from ..utils import (
     assert_unordered_equality,
 )
 
 
 class TestCoreIamc:
     small = SmallIamcDataset()
+    filter = FilterIamcDataset()
 
     def test_run_annual_datapoints_raw(self, platform: ixmp4.Platform):
         self.do_run_datapoints(
@@ -163,62 +164,68 @@ def do_run_datapoints(self, platform: ixmp4.Platform, data, raw=True, _type=None
         ret = run.iamc.tabulate(raw=raw)
         assert ret.empty
 
-
-class TestCoreIamcReadOnly:
     @pytest.mark.parametrize(
         "filters, run, exp_len",
         (
-            (dict(variable={"name": "Variable 4"}), ("Model 1", "Scenario 1", 1), 50),
+            (dict(variable={"name": "Variable 4"}), ("Model 1", "Scenario 1", 1), 1),
             (
                 dict(
                     variable={"name": "Variable 4"},
-                    unit={"name": "Unit 4"},
+                    unit={"name": "Unit 2"},
                 ),
                 ("Model 1", "Scenario 1", 1),
-                50,
+                1,
             ),
             (
                 dict(
                     variable={"name__like": "Variable *"},
                     unit={"name__like": "Unit *"},
                 ),
                 ("Model 2", "Scenario 2", 1),
-                280,
+                4,
             ),
             (
-                dict(variable={"name__in": ["Variable 4", "Variable 24"]}),
+                dict(variable={"name__in": ["Variable 3", "Variable 4"]}),
                 ("Model 1", "Scenario 1", 1),
-                50,
+                2,
             ),
-            (dict(variable="Variable 24"), ("Model 1", "Scenario 1", 1), 0),
+            (dict(variable="Variable 1"), ("Model 1", "Scenario 1", 1), 2),
             (
-                dict(variable="Variable 54", unit="Unit 54"),
-                ("Model 8", "Scenario 0", 1),
-                50,
+                dict(variable="Variable 1", unit="Unit 2"),
+                ("Model 1", "Scenario 1", 1),
+                1,
             ),
             (
-                dict(variable=["Variable 54", "Variable 25", "Variable 4"]),
+                dict(variable=["Variable 1", "Variable 2", "Variable 3", "Variable 4"]),
                 ("Model 1", "Scenario 1", 1),
-                50,
+                4,
             ),
         ),
     )
     def test_run_tabulate_with_filter_raw(
-        self, ro_platform_med: ixmp4.Platform, filters, run, exp_len
+        self, platform: ixmp4.Platform, filters, run, exp_len
     ):
-        run = ro_platform_med.runs.get(*run)
+        self.filter.load_dataset(platform)
+        run = platform.runs.get(*run)
         obs = run.iamc.tabulate(raw=True, **filters)
         assert len(obs) == exp_len
 
-    def test_mp_tabulate_big_async(self, ro_platform_med: ixmp4.Platform):
+
+class TestCoreIamcReadOnly:
+    medium = MediumIamcDataset()
+
+    def test_mp_tabulate_big_async(self, platform: ixmp4.Platform):
         """Tests if big tabulations work in async contexts."""
+        self.medium.load_dataset(platform)
 
         async def tabulate():
-            return ro_platform_med.iamc.tabulate(raw=True, run={"default_only": False})
+            return platform.iamc.tabulate(raw=True, run={"default_only": False})
 
         res = asyncio.run(tabulate())
         assert len(res) > settings.default_page_size
 
-    def test_mp_tabulate_big(self, ro_platform_med: ixmp4.Platform):
-        res = ro_platform_med.iamc.tabulate(raw=True, run={"default_only": False})
+    def test_mp_tabulate_big(self, platform: ixmp4.Platform):
+        self.medium.load_dataset(platform)
+
+        res = platform.iamc.tabulate(raw=True, run={"default_only": False})
         assert len(res) > settings.default_page_size

tests/data/test_count.py

@@ -79,7 +79,6 @@ def deepgetattr(obj, attr):
         ],
     ],
 )
-def test_count(db_platform: ixmp4.Platform, repo_name, filters):
-    big.load_dataset(db_platform)
-    repo = deepgetattr(db_platform.backend, repo_name)
+def test_count(db_platform_big: ixmp4.Platform, repo_name, filters):
+    repo = deepgetattr(db_platform_big.backend, repo_name)
     assert len(repo.list(**filters)) == repo.count(**filters)