Improve IAMC endpoint query performance

ixmp4/data/db/filters/model.py

@@ -1,6 +1,6 @@
 from typing import ClassVar
 
-from ixmp4.db import filters
+from ixmp4.db import filters, utils
 
 from .. import Model, Run
 
@@ -12,4 +12,5 @@ class ModelFilter(filters.BaseFilter, metaclass=filters.FilterMeta):
     sqla_model: ClassVar[type] = Model
 
     def join(self, exc, **kwargs):
-        return exc.join(Model, Run.model)
+        if not utils.is_joined(exc, Model):
+            return exc.join(Model, Run.model)

ixmp4/data/db/filters/scenario.py

@@ -1,6 +1,6 @@
 from typing import ClassVar
 
-from ixmp4.db import filters
+from ixmp4.db import filters, utils
 
 from .. import Run, Scenario
 
@@ -12,4 +12,5 @@ class ScenarioFilter(filters.BaseFilter, metaclass=filters.FilterMeta):
     sqla_model: ClassVar[type] = Scenario
 
     def join(self, exc, **kwargs):
-        return exc.join(Scenario, Run.scenario)
+        if not utils.is_joined(exc, Scenario):
+            return exc.join(Scenario, Run.scenario)

ixmp4/data/db/iamc/datapoint/model.py

@@ -22,7 +22,10 @@ class DataPoint(base.BaseModel):
     @declared_attr
     def time_series__id(cls):
         return db.Column(
-            db.Integer, db.ForeignKey("iamc_timeseries.id"), nullable=False, index=True
+            db.Integer,
+            db.ForeignKey("iamc_timeseries.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
         )
 
     value = db.Column(db.Float)

ixmp4/data/db/iamc/variable/filter.py

@@ -1,5 +1,4 @@
 from ixmp4.data.db import filters as base
-from ixmp4.data.db.iamc.datapoint import get_datapoint_model
 from ixmp4.data.db.iamc.timeseries import TimeSeries
 from ixmp4.db import filters, utils
 
@@ -22,8 +21,4 @@ def join(self, exc, session=None):
             exc = exc.join(
                 TimeSeries, onclause=TimeSeries.measurand__id == Measurand.id
             )
-
-        model = get_datapoint_model(session)
-        if not utils.is_joined(exc, model):
-            exc = exc.join(model, onclause=model.time_series__id == TimeSeries.id)
         return exc

ixmp4/data/db/model/filter.py

@@ -1,6 +1,5 @@
 from ixmp4 import db
 from ixmp4.data.db import filters as base
-from ixmp4.data.db.iamc.datapoint import get_datapoint_model
 from ixmp4.data.db.iamc.timeseries import TimeSeries
 from ixmp4.data.db.run.model import Run
 from ixmp4.db import filters, utils
@@ -16,9 +15,6 @@ def join_datapoints(self, exc: db.sql.Select, session=None):
         if not utils.is_joined(exc, TimeSeries):
             exc = exc.join(TimeSeries, onclause=TimeSeries.run__id == Run.id)
 
-        model = get_datapoint_model(session)
-        if not utils.is_joined(exc, model):
-            exc = exc.join(model, onclause=model.time_series__id == TimeSeries.id)
         return exc
 
 

ixmp4/data/db/region/filter.py

@@ -1,6 +1,5 @@
 from ixmp4 import db
 from ixmp4.data.db import filters as base
-from ixmp4.data.db.iamc.datapoint import get_datapoint_model
 from ixmp4.data.db.iamc.timeseries import TimeSeries
 from ixmp4.db import filters, utils
 
@@ -11,10 +10,6 @@ class BaseIamcFilter(filters.BaseFilter, metaclass=filters.FilterMeta):
     def join_datapoints(self, exc: db.sql.Select, session=None):
         if not utils.is_joined(exc, TimeSeries):
             exc = exc.join(TimeSeries, onclause=TimeSeries.region__id == Region.id)
-
-        model = get_datapoint_model(session)
-        if not utils.is_joined(exc, model):
-            exc = exc.join(model, onclause=model.time_series__id == TimeSeries.id)
         return exc
 
 

ixmp4/data/db/scenario/filter.py

@@ -1,6 +1,5 @@
 from ixmp4 import db
 from ixmp4.data.db import filters as base
-from ixmp4.data.db.iamc.datapoint import get_datapoint_model
 from ixmp4.data.db.iamc.timeseries import TimeSeries
 from ixmp4.data.db.run.model import Run
 from ixmp4.db import filters, utils
@@ -16,9 +15,6 @@ def join_datapoints(self, exc: db.sql.Select, session=None):
         if not utils.is_joined(exc, TimeSeries):
             exc = exc.join(TimeSeries, onclause=TimeSeries.run__id == Run.id)
 
-        model = get_datapoint_model(session)
-        if not utils.is_joined(exc, model):
-            exc = exc.join(model, onclause=model.time_series__id == TimeSeries.id)
         return exc
 
 

ixmp4/data/db/unit/filter.py

@@ -1,6 +1,5 @@
 from ixmp4 import db
 from ixmp4.data.db import filters as base
-from ixmp4.data.db.iamc.datapoint import get_datapoint_model
 from ixmp4.data.db.iamc.measurand import Measurand
 from ixmp4.data.db.iamc.timeseries import TimeSeries
 from ixmp4.db import filters, utils
@@ -17,10 +16,6 @@ def join_datapoints(self, exc, session=None):
             exc = exc.join(
                 TimeSeries, onclause=TimeSeries.measurand__id == Measurand.id
             )
-
-        model = get_datapoint_model(session)
-        if not utils.is_joined(exc, model):
-            exc = exc.join(model, onclause=model.time_series__id == TimeSeries.id)
         return exc
 
 

ixmp4/db/migrations/versions/07c169c7792d_add_on_delete_cascade_to_timeseries_.py

@@ -0,0 +1,50 @@
+# type: ignore
+"""add ON DELETE CASCADE to timeseries table
+
+Revision ID: 07c169c7792d
+Revises: 97ba231770e2
+Create Date: 2024-03-06 09:45:00.725366
+
+"""
+from alembic import op
+
+# Revision identifiers, used by Alembic.
+revision = "07c169c7792d"
+down_revision = "97ba231770e2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("iamc_datapoint_universal", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_iamc_datapoint_universal_time_series__id_iamc_timeseries",
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_iamc_datapoint_universal_time_series__id_iamc_timeseries"),
+            "iamc_timeseries",
+            ["time_series__id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("iamc_datapoint_universal", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_iamc_datapoint_universal_time_series__id_iamc_timeseries"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_iamc_datapoint_universal_time_series__id_iamc_timeseries",
+            "iamc_timeseries",
+            ["time_series__id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###

tests/data/test_count.py

@@ -0,0 +1,81 @@
+from functools import reduce
+
+import pytest
+
+from ..utils import generated_db_platforms
+
+
+def deepgetattr(obj, attr):
+    return reduce(getattr, attr.split("."), obj)
+
+
+@generated_db_platforms
+@pytest.mark.parametrize(
+    "repo_name,filters",
+    [
+        [
+            "iamc.datapoints",
+            {
+                "model": {"name": "Model 0"},
+                "scenario": {"name": "Scenario 0"},
+                "run": {"default_only": False},
+            },
+        ],
+        [
+            "iamc.datapoints",
+            {
+                "scenario": {"name__like": "Scenario *"},
+                "run": {"default_only": False},
+            },
+        ],
+        [
+            "iamc.datapoints",
+            {
+                "model": {"name__like": "Model *"},
+                "unit": {"name__in": [f"Unit {i}" for i in range(10)]},
+                "variable": {"name__like": "Variable 1*"},
+                "region": {"name__in": [f"Region {i}" for i in range(10)]},
+                "run": {"default_only": False},
+            },
+        ],
+        [
+            "regions",
+            {
+                "name__like": "Region 1*",
+                "iamc": {
+                    "run": {"default_only": False},
+                },
+            },
+        ],
+        [
+            "models",
+            {
+                "name__like": "Model *",
+                "iamc": True,
+            },
+        ],
+        [
+            "scenarios",
+            {
+                "name__like": "Scenario *",
+                "iamc": {
+                    "variable": {"name__like": "Variable *"},
+                    "run": {"default_only": False},
+                },
+            },
+        ],
+        [
+            "units",
+            {
+                "iamc": {
+                    "variable": {"name__like": "Variable *"},
+                    "run": {"default_only": False},
+                },
+            },
+        ],
+    ],
+)
+def test_count(generated_mp, repo_name, filters, request):
+    generated_mp = request.getfixturevalue(generated_mp)
+    repo = deepgetattr(generated_mp.backend, repo_name)
+    assert len(repo.list(**filters)) == repo.count(**filters)

tests/utils.py

@@ -69,6 +69,20 @@ def assert_unordered_equality(df1, df2, **kwargs):
     ],
 )
 
+generated_db_platforms = pytest.mark.parametrize(
+    "generated_mp",
+    [
+        "test_sqlite_mp_generated",
+        pytest.param(
+            "test_pgsql_mp_generated",
+            marks=pytest.mark.skipif(
+                SKIP_PGSQL_TESTS,
+                reason="Cannot connect to PostgreSQL database service, skipping test",
+            ),
+        ),
+    ],
+)
+
 
 generated_api_platforms = pytest.mark.parametrize(
     "generated_mp",