Merge pull request #207 from iiasa/migrate/transport

Migrate `message_data.model.transport`

.github/workflows/pytest.yaml

@@ -2,7 +2,7 @@ name: Test
 
 on:
   pull_request:
-    branches: [ main , "migrate-*"]
+    branches: [ main , "migrate**"]
   schedule:
   # 05:00 UTC = 06:00 CET = 07:00 CEST
   - cron: "0 5 * * *"
@@ -123,7 +123,7 @@ jobs:
     - name: Run test suite using pytest
       run: |
         pytest message_ix_models \
-          -m "not snapshot" \
+          -m "not (ece_db or snapshot)" \
           -rA --verbose --color=yes --durations=20 \
           --cov-report=term-missing --cov-report=xml \
           --numprocesses=auto \

.github/workflows/transport.yaml

@@ -0,0 +1,212 @@
+name: MESSAGEix-Transport
+
+env:
+  # Starting point of the workflow.
+  # Use this value to build from a certain scenario:
+  # base: --url="ixmp://ixmp-dev/MESSAGEix-GLOBIOM 1.1-R12/baseline_DEFAULT#21"
+  # Use this value to allow the workflow to determine model & scenario names
+  # and versions:
+  base: --platform=ixmp-dev
+
+  # Set this to a particular step to truncate the workflow
+  from-step: ""
+
+on:
+  # Uncomment these lines for debugging, but leave them commented on 'main'/'dev'
+  # pull_request:
+  #   branches: [ main, dev ]
+  # push:
+  #   branches: [ main, dev ]
+  schedule:
+  - cron: "0 1 * * *"  # 01:00 UTC = 02:00 CET = 03:00 CEST
+  workflow_dispatch: {}
+
+# Cancel previous runs that have not completed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  report:
+    name: MESSAGEix-Transport ${{ matrix.SSP }}${{ matrix.policy }}
+    # Distribute jobs across any available self-hosted runner(s)
+    # runs-on: self-hosted
+    # Use one or a few specific runner(s)
+    runs-on: [ self-hosted, iiasa-self-hosted-l2 ]
+
+    strategy:
+      matrix:
+        # These correspond to short labels used for workflow step names
+        SSP:
+        - SSP1
+        - SSP2
+        - SSP3
+        - SSP4
+        - SSP5
+        policy:
+        - " policy"
+        - ""
+
+    steps:
+    - name: Clean workspace
+      run: |
+        rm -rf ${{ github.workspace }}/report/*
+
+    - uses: webfactory/[email protected]
+      with:
+        ssh-private-key: |
+          ${{ secrets.MESSAGE_BUILDINGS_PRIVATE_KEY }}
+          ${{ secrets.MESSAGE_TRADE_PRIVATE_KEY }}
+          ${{ secrets.NAVIGATE_WORKFLOW_PRIVATE_KEY }}
+
+    - uses: actions/checkout@v4
+      with:
+        path: message_data
+        lfs: true
+        # On GitHub Actions
+        # submodules: true
+        # On self-hosted runners
+        submodules: false
+
+    # NB Needed for self-hosted runner, not for GitHub
+    - name: Check out Git LFS files
+      working-directory: message_data
+      run: git lfs checkout
+
+    # Other repositories required for testing
+    - name: Check out message-static-data
+      uses: actions/checkout@v4
+      with:
+        repository: iiasa/message-static-data
+        token: ${{ secrets.MESSAGE_STATIC_DATA_PAT }}
+        path: message-static-data
+        lfs: true
+        # Only check out the following files to limit bandwidth usage.
+        # Respectively
+        # 1. IEA Extended World Energy Balances, OECD source, 2022 edition
+        sparse-checkout: |
+          iea/372f7e29-en.zip
+        sparse-checkout-cone-mode: false
+
+    - name: Connect static data to message_data
+      run: cp -rsv $(realpath message-static-data)/* message_data/data/
+
+    - uses: actions/checkout@v4
+      with:
+        repository: iiasa/MESSAGE_Buildings
+        path: buildings
+        ssh-key: ${{ secrets.MESSAGE_BUILDINGS_PRIVATE_KEY }}
+
+    # NB Needed for self-hosted runner, not for GitHub
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: "17"
+
+    - uses: actions/setup-python@v5
+      with:
+        # Latest version testable on GitHub Actions
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: "**/pyproject.toml"
+
+    - uses: iiasa/actions/setup-gams@main
+      with:
+        version: 35.2.0
+        license: ${{ secrets.GAMS_LICENSE }}
+
+    # NB the IIASA-hosted runner does not have permissions to do "sudo apt-get",
+    #    so the following doesn't work. Rather, the necessary packages
+    #    (graphviz libgraphviz-dev pkg-config) are manually installed on the
+    #    host system
+    # - uses: ts-graphviz/setup-graphviz@v2
+
+    # By default, the next step installs dependencies from PyPI.
+    # To run against unreleased code (on `main`, or other branches for open
+    # PRs), uncomment or modify the lines below as appropriate.
+    # - genno and message-ix-models are released frequently and *should* be
+    #   installed from PyPI, with only temporary exceptions.
+    # - ixmp and message-ix are released less frequently and *may* be installed
+    #   from `main`
+    - name: Force upgrade to unreleased dependencies
+      run: |
+        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
+        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
+        pip install --upgrade "message_ix @ git+https://github.com/iiasa/message_ix.git@main"
+        # Temporary, pending a release that includes iiasa/message-ix-models#161, #166
+        pip install --upgrade "message-ix-models[iea-web] @ git+https://github.com/iiasa/message-ix-models.git@main"
+
+        # Upgrade matplotlib
+        # pyam pins matplotlib < 3.7.1, which conflicts with plotnine per
+        # https://github.com/has2k1/plotnine/pull/751. Override.
+        pip install --upgrade matplotlib
+
+    - name: Install package; upgrade dependencies
+      working-directory: message_data
+      run: pip install --upgrade --upgrade-strategy=eager --editable .[dl,scgen,tests,transport]
+
+    - name: Configure
+      run: |
+        from pathlib import Path
+
+        # Ensure keys are defined
+        import message_data.model.buildings
+        from ixmp import config
+
+        config.add_platform(
+            "ixmp-dev",
+            "jdbc",
+            "oracle",
+            "x8oda.iiasa.ac.at:1521/PIXMP2.iiasa.ac.at",
+            "${{ secrets.IXMP_DEV_USER }}",
+            "${{ secrets.IXMP_DEV_PASS }}",
+            jvmargs="-Xmx16G",
+        )
+
+        p = Path("${{ github.workspace }}")
+        config.set("message local data", p / "report")
+        config.set("message solve options", {"iis": 1, "tilim": 60 * 30})
+        config.set("message buildings dir", p / "buildings")
+        config.save()
+
+        # Equivalent to "(ixmp|message-ix|mix-models) config show"
+        print(config.path.read_text())
+      shell: python
+
+    - name: Fetch data
+      run: |
+        mix-models fetch "SSP-Update-3.0"
+        mix-models fetch "SSP-Update-3.0.1"
+
+    - name: Run MESSAGEix-Transport workflow "${{ env.from-step }}" → "${{ matrix.SSP }}${{ matrix.policy }} reported"
+      run: |
+        mix-models \
+          ${{ env.base }} \
+          transport run \
+          --base=auto --nodes=R12 --model-extra="ci nightly" \
+          --from="${{ env.from-step }}" \
+          "${{ matrix.SSP }}${{ matrix.policy }} reported" \
+          --go
+      continue-on-error: true
+
+    - name: Upload results as a build artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: MESSAGEix-Transport ${{ matrix.SSP }}${{ matrix.policy }}
+        path: |
+          ${{ github.workspace }}/report/
+          !${{ github.workspace }}/report/cache/
+        if-no-files-found: error
+      continue-on-error: true
+
+  collate:
+    name: Collate outputs
+    needs: report
+    runs-on: self-hosted
+
+    steps:
+    - name: Merge artifacts
+      uses: actions/upload-artifact/merge@v4
+      with:
+        name: MESSAGEix-Transport
+        delete-merged: true

.readthedocs.yaml

@@ -13,6 +13,11 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.10"
+  jobs:
+    post_install:
+    # Remove requests-cache, which appears to cause segfaults on RTD workers
+    - pip uninstall --yes requests-cache
+
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

conftest.py

@@ -1,2 +1,6 @@
 # Modules containing fixtures to use in testing
-pytest_plugins = ["ixmp.testing", "message_ix_models.testing"]
+pytest_plugins = [
+    "genno.testing",
+    "ixmp.testing",
+    "message_ix_models.testing",
+]

doc/api/report/index.rst

@@ -13,7 +13,7 @@ Elsewhere:
 - Reporting for specific model variants:
 
   - :mod:`.water.reporting`
-  - (Private) :doc:`Reporting of message_data.model.transport <m-data:reference/model/transport/report>`
+  - :doc:`transport/output` of :mod:`.model.transport`
 
 - :doc:`‘Legacy’ reporting <legacy>`.
 

doc/api/tools-messagev.rst

@@ -0,0 +1,54 @@
+MESSAGE V input files
+*********************
+
+This document describes some of the file formats for the pre-:mod:`.ixmp` MESSAGE model, a.k.a. **MESSAGE V**, and code in :mod:`message_ix_models.tools.messagev` that reads these formats.
+
+.. note:: See also the earlier :doc:`import_from_msgV_sqlite` for similar code/descriptions.
+
+.. contents::
+   :local:
+
+``easemps3.free``: soft dynamic constraints
+-------------------------------------------
+
+Each constraint is specified by a single row with the following, space-separated entries:
+
+1. **Constraint type.** `mpa` (constraint on activity) or `mpc` (constraint on capacity).
+2. **Technology name.** Four-letter internal name/code of a technology, e.g. `uHap`.
+3. **Lower/upper bound.** Either `LO` (decline constraint) or `UP` (growth constraint).
+4. **Cost type.** One of:
+
+   - `lev`: levelized costs.
+   - `abs`: absolute costs.
+   - `var`: variable costs.
+
+5. **Growth rate for step 1.** Percentage points of growth/decline at which the constraint becomes active.
+
+6. **Additional cost for step 1.** Additional cost applied to activity/capacity growth or decline beyond the rate in #5. Depending on #4, specified:
+
+   - `lev`: in percentage points of the levelized cost of the technology.
+   - `abs`, `var`: in absolute monetary value.
+
+7. **Up to 4 additional pairs of 5 and 6.** Growth rates for successive constraints are cumulative.
+
+An example:
+
+.. code::
+
+   mpa uEAp UP lev 5 50 15 300000
+
+Here the constraint relates to a growth constraint (UP) for activities (mpa) and the technology for which the constraint is to be extended is uEAp.
+The allowed rate of growth is increased by 5 %-points and each additional unit of output that can be produced costs 50 % of the levelized costs additional on top of the normal costs (i.e. the costs that result from building and using the additional capacity required for the additional production).
+
+The second step increases the maximum growth rate further, by 15 %-points, but the costs are prohibitive (300000).
+
+Soft constraints can be set for each technology individually. This can be done globally ("regions = all -glb") or for each region separately ("regions = cpa").
+
+
+API reference
+-------------
+
+.. currentmodule:: message_ix_models.tools.messagev
+
+.. automodule:: message_ix_models.tools.messagev
+   :members:

doc/global/energy/enduse/index.rst

@@ -1,6 +1,6 @@
 Energy end-use
 =================
-MESSAGEix distinguishes three energy end-use sectors, i.e. transport, residential/commercial (also referred to as the buildings sector) and industry. Given the long-term nature of the scenarios, the model version used for the SSPs, represents these end-use sectors in a stylized way. For more detailed short-term analysis, a model version with a more detailed transport sector module that distinguishes different transport modes, vehicle classes and consumer types exists (McCollum et al., 2016 :cite:`mccollum_transport_2016`).
+MESSAGEix distinguishes three energy end-use sectors, i.e. transport, residential/commercial (also referred to as the buildings sector) and industry. Given the long-term nature of the scenarios, the model version used for the SSPs, represents these end-use sectors in a stylized way. For more detailed short-term analysis, a model version with a more detailed transport sector module that distinguishes different transport modes, vehicle classes and consumer types exists (McCollum et al., 2017 :cite:`mccollum-2016`).
 
 .. toctree::
    :maxdepth: 1

doc/index.rst

@@ -69,6 +69,7 @@ Commonly used classes may be imported directly from :mod:`message_ix_models`.
    api/report/index
    api/tools
    api/tools-costs
+   api/tools-messagev
    api/data-sources
    api/util
    api/testing
@@ -80,6 +81,7 @@ Commonly used classes may be imported directly from :mod:`message_ix_models`.
 
    global/index
    material/index
+   transport/index
    water/index
 
 .. toctree::

doc/main.bib

@@ -1496,12 +1496,29 @@ @techreport{iea_investment_2014
   year = {2014}
 }
 
-@article{mccollum_transport_2016,
-  author = {McCollum, David L. and Wilson, Charlie and Pettifor, Hazel and Ramea, Kalai and Krey, Volker and Riahi, Keywan and Bertram, Christoph and Lin, Zhenhong and Edelenbosch, Oreane Y. and Fujisawa, Sei},
-  journal = {Transportation Research Part D: Transport and Environment},
-  title = {Improving the behavioral realism of global integrated assessment models: An application to consumers’ vehicle choices},
-  type = {Journal Article},
-  year = {2016}
+@article{mccollum-2017,
+	author = {McCollum, David L. and Wilson, Charlie and Pettifor, Hazel and Ramea, Kalai and Krey, Volker and Riahi, Keywan and Bertram, Christoph and Lin, Zhenhong and Edelenbosch, Oreane Y. and Fujisawa, Sei},
+	doi = {10.1016/j.trd.2016.04.003},
+	issn = {13619209},
+	journal = {Transportation Research Part D: Transport and Environment},
+	keywords = {Climate change mitigation; Consumer choice; Human behavior; Light-duty vehicles; Transport},
+	pages = {322--342},
+	publisher = {Elsevier Ltd},
+	title = {{Improving the behavioral realism of global integrated assessment models: An application to consumers' vehicle choices}},
+	volume = {55},
+	year = {2017}
+}
+
+@article{mccollum-2018,
+	author = {McCollum, David L. and Wilson, Charlie and Bevione, Michela and Carrara, Samuel and Edelenbosch, Oreane Y. and Emmerling, Johannes and Guivarch, C{\'e}line and Karkatsoulis, Panagiotis and Keppo, Ilkka and Krey, Volker and Lin, Zhenhong and Broin, Eoin {\'O} and Paroussos, Leonidas and Pettifor, Hazel and Ramea, Kalai and Riahi, Keywan and Sano, Fuminori and Rodriguez, Baltazar Solano and van Vuuren, Detlef P.},
+	doi = {10.1038/s41560-018-0195-z},
+	issn = {2058-7546},
+	journal = {Nature Energy},
+	number = {8},
+	pages = {664–673},
+	title = {Interaction of consumer preferences and climate policies in the global transition to low-carbon vehicles},
+	volume = {3},
+	year = {2018},
 }
 
 @article{mcjeon_gas_2014,