Merge branch 'main' into include/optimization-table

iiasa · Mar 5, 2024 · f76b38e · f76b38e
2 parents 922f958 + c85d987
commit f76b38e
Show file tree

Hide file tree

Showing 10 changed files with 421 additions and 406 deletions.
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -8,10 +8,14 @@ jobs:
     strategy:
       matrix:
         python-version:
-        - "3.10"  # Earliest version supported by ixmp4
-        - "3.11"
-        - "3.12"  # Latest version supported by ixmp4
-
+          - "3.10" # Earliest version supported by ixmp4
+          - "3.11"
+          - "3.12" # Latest version supported by ixmp4
+        with-pyarrow:
+          - false
+        include:
+          - python-version: "3.12"
+            with-pyarrow: true
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -65,6 +69,10 @@ jobs:
       #----------------------------------------------
       # install your root project, if required
       #----------------------------------------------
+      - name: Install PyArrow
+        if: ${{ matrix.with-pyarrow }}
+        run: pip install pyarrow
+
       - name: Install library
         run: poetry install --no-interaction
       #----------------------------------------------

diff --git a/dev.env b/dev.env
@@ -0,0 +1,4 @@
+IXMP4_MODE=debug
+IXMP4_STORAGE_DIRECTORY=./run/dev/ixmp4/
+IXMP4_MANAGED=false
+IXMP4_MANAGER_URL=https://api.dev.manager.ece.iiasa.ac.at/v1/
diff --git a/ixmp4/conf/logging/server.conf b/ixmp4/conf/logging/server.conf
@@ -1,6 +1,6 @@
 
 [loggers]
-keys = root,sqlalchemy,fastapi,httpx,uvicorn,access
+keys = root,sqlalchemy,fastapi,httpx,uvicorn,access,watchfiles
 
 [handlers]
 keys = access,debug,error,console
@@ -13,6 +13,11 @@ level = NOTSET
 handlers = debug,error
 qualname = sqlalchemy
 
+[logger_watchfiles]
+level = ERROR
+handlers = error,debug
+qualname = watchfiles.main
+
 [logger_fastapi]
 level = NOTSET
 handlers = debug,error,console

diff --git a/ixmp4/data/db/base.py b/ixmp4/data/db/base.py
@@ -269,9 +269,7 @@ def count(
         **kwargs,
     ) -> int:
         _exc = self.select(
-            _exc=db.select(db.func.count(self.model_class.id.distinct())).select_from(
-                self.model_class
-            ),
+            _exc=db.select(db.func.count(self.model_class.id.distinct())),
             **kwargs,
         )
         return self.session.execute(_exc).scalar_one()
@@ -391,7 +389,14 @@ def bulk_upsert_chunk(self, df: pd.DataFrame) -> None:
             for col in self.model_class.updateable_columns:
                 updated_col = col + self.merge_suffix
                 if updated_col in df.columns:
-                    cond.append(df[col] != df[updated_col])
+                    # coerce to same type so the inequality
+                    # operation works with pyarrow installed
+                    df[updated_col] = df[updated_col].astype(df[col].dtype)
+                    are_not_equal = df[col] != df[updated_col]
+                    # extra check if both values are NA because NA == NA = NA
+                    # in pandas with pyarrow
+                    both_are_na = pd.isna(df[col]) & pd.isna(df[updated_col])
+                    cond.append(~both_are_na | are_not_equal)
 
             df["differs"] = np.where(np.logical_or.reduce(cond), True, False)
 

diff --git a/ixmp4/data/db/iamc/datapoint/filter.py b/ixmp4/data/db/iamc/datapoint/filter.py
@@ -88,7 +88,7 @@ def join(self, exc, session):
         if not utils.is_joined(exc, Run):
             exc = exc.join(Run, TimeSeries.run)
         if not utils.is_joined(exc, Scenario):
-            exc = exc.join(Scenario, onclause=Run.model__id == Scenario.id)
+            exc = exc.join(Scenario, onclause=Run.scenario__id == Scenario.id)
         return exc
 
 

diff --git a/ixmp4/data/db/iamc/datapoint/repository.py b/ixmp4/data/db/iamc/datapoint/repository.py
@@ -4,8 +4,6 @@
 import pandas as pd
 import pandera as pa
 from pandera.typing import DataFrame, Series
-from sqlalchemy import select
-from sqlalchemy.orm import Bundle
 
 from ixmp4 import db
 from ixmp4.core.decorators import check_types
@@ -109,36 +107,27 @@ def join_auth(self, exc: db.sql.Select) -> db.sql.Select:
         return exc
 
     def select_joined_parameters(self, join_runs=False):
-        _bundle = []
+        bundle = []
         if join_runs:
-            _bundle.extend(
+            bundle.extend(
                 [
-                    Bundle("Model", Model.name.label("model")),
-                    Bundle("Scenario", Scenario.name.label("scenario")),
-                    Bundle("Run", Run.version),
+                    Model.name.label("model"),
+                    Scenario.name.label("scenario"),
+                    Run.version.label("version"),
                 ]
             )
 
-        _bundle.extend(
+        bundle.extend(
             [
-                Bundle(
-                    "Region",
-                    Region.name.label("region"),
-                ),
-                Bundle(
-                    "Unit",
-                    Unit.name.label("unit"),
-                ),
-                Bundle(
-                    "Variable",
-                    Variable.name.label("variable"),
-                ),
+                Region.name.label("region"),
+                Unit.name.label("unit"),
+                Variable.name.label("variable"),
                 self.bundle,
             ]
         )
 
         _exc = (
-            select(*_bundle)
+            db.select(*bundle)
             .join(
                 TimeSeries, onclause=self.model_class.time_series__id == TimeSeries.id
             )

diff --git a/ixmp4/server/rest/__init__.py b/ixmp4/server/rest/__init__.py
@@ -19,7 +19,7 @@
 from .middleware import RequestSizeLoggerMiddleware, RequestTimeLoggerMiddleware
 from .optimization import indexset, scalar, table
 
-v1 = FastAPI()
+v1 = FastAPI(servers=[{"url": "/v1", "description": "v1"}])
 
 if settings.mode == "debug":
     v1.add_middleware(RequestSizeLoggerMiddleware)