From 2fc258888c0301673630e3474b3270c487b04dbe Mon Sep 17 00:00:00 2001 From: Florian Maczek Date: Thu, 21 Nov 2024 18:38:13 +0100 Subject: [PATCH] Fix methanol final energy allocation bug --- .../reporting/fe_reporting_aggregates.yaml | 33 ++++++---- .../model/material/report/reporter_utils.py | 66 ++++++++++++------- .../model/material/report/run_reporting.py | 35 ++++++++-- 3 files changed, 96 insertions(+), 38 deletions(-) diff --git a/message_ix_models/data/material/reporting/fe_reporting_aggregates.yaml b/message_ix_models/data/material/reporting/fe_reporting_aggregates.yaml index 5feff5bf20..96b1eef4bc 100644 --- a/message_ix_models/data/material/reporting/fe_reporting_aggregates.yaml +++ b/message_ix_models/data/material/reporting/fe_reporting_aggregates.yaml @@ -95,30 +95,39 @@ level_2: fe_pe_chem_other_h2 ] level_1: + + # some Chemicals|| variables are not specified here + # because they are generated later in the workflow, after the + # Methanol/HVC/Ammonia subcomponents are generated. + # The function run_reporting.add_chemicals_to_final_energy_variables + # does this aggregation and updates all related aggregates. + # Only the ones not generated there are mapped here + + Chemicals|Liquids|Other: + short: fe_pe_chem_oth + components: + [ fe_pe_hvc_oth ] + Chemicals|Liquids|Biomass: + short: fe_pe_chem_oth + components: + [ fe_pe_hvc_bio_eth ] + Chemicals|High-Value Chemicals|Electricity: short: fe_pe_hvc_el components: [fe_pe_hvc_el_ht, fe_pe_hvc_el_mto, fe_pe_hvc_el_sc] - Chemicals|High-Value Chemicals|Liquids|Oil: - short: fe_pe_hvc_oil - components: - [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h ] Chemicals|High-Value Chemicals|Liquids: short: fe_pe_hvc_liq components: [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h, fe_pe_hvc_bio_eth, fe_pe_hvc_oth] + Chemicals|High-Value Chemicals|Liquids|Oil: + short: fe_pe_hvc_oil + components: + [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h ] Chemicals|High-Value Chemicals|Solids: short: fe_pe_hvc_solids components: [ fe_pe_hvc_bio, fe_pe_hvc_coal, fe_pe_hvc_coke] - Chemicals|Liquids|Other: - short: fe_pe_chem_oth - components: - [ fe_pe_hvc_oth ] - Chemicals|Liquids|Gases: - short: fe_pe_hvc_gases - components: - [ fe_pe_hvc_gas ] Chemicals|Other Sector|Electricity: short: fe_pe_chem_other_el diff --git a/message_ix_models/model/material/report/reporter_utils.py b/message_ix_models/model/material/report/reporter_utils.py index 6466366399..33d096d156 100644 --- a/message_ix_models/model/material/report/reporter_utils.py +++ b/message_ix_models/model/material/report/reporter_utils.py @@ -106,18 +106,33 @@ def add_methanol_share_calculations(rep: message_ix.Reporter, mode: str = "feeds if mode == "fuel": t_filter2.update({"l": ["primary"]}) - rep.add("out::methanol", "select", "out:nl-t-ya-m-c-l", t_filter2) - rep.add("out::methanol-by-tec", "group_sum", "out::methanol", group="t", sum="c") + rep.add("out::methanol-prod", "select", "out:nl-t-ya-m-c-l", t_filter2) rep.add( - "out::methanol-total", "group_sum", "out::methanol", group=["nl", "ya"], sum="t" + "out::methanol-prod-by-tec", + "group_sum", + "out::methanol-prod", + group="t", + sum="c", + ) + rep.add( + "out::methanol-prod-total", + "group_sum", + "out::methanol-prod", + group=["nl", "ya"], + sum="t", + ) + rep.add( + "share::methanol-prod-by-tec", + "div", + "out::methanol-prod-by-tec", + "out::methanol-prod-total", ) - rep.add("share::methanol", "div", "out::methanol-by-tec", "out::methanol-total") for comm, tecs in comm_tec_map.items(): rep.add( - f"share::{comm}methanol", + f"share::{comm}-methanol-prod", "aggregate", - "share::methanol", - groups={"t": {f"{comm}methanol": tecs}}, + "share::methanol-prod-by-tec", + groups={"t": {f"{comm}-methanol": tecs}}, keep=False, ) @@ -136,10 +151,8 @@ def add_meth_export_calculations(rep: message_ix.Reporter, mode: str = "feedstoc ---------- rep """ + add_methanol_share_calculations(rep, mode=mode) for comm in comm_tec_map.keys(): - if f"share::{comm}methanol" not in rep.keys(): - add_methanol_share_calculations(rep, mode=mode) - t_filter2 = {"t": "meth_exp", "m": mode} rep.add("out::methanol-export", "select", "out:nl-t-ya-m", t_filter2) rep.add( @@ -153,7 +166,7 @@ def add_meth_export_calculations(rep: message_ix.Reporter, mode: str = "feedstoc f"out::{comm}methanol-export", "mul", "out::methanol-export", - f"share::{comm}methanol", + f"share::{comm}-methanol-prod", ) @@ -174,10 +187,9 @@ def add_meth_import_calculations(rep: message_ix.Reporter, mode: str = "feedstoc ---------- rep """ - for comm in comm_tec_map.keys(): - if "out::{comm}methanol-export" not in rep.keys(): - add_meth_export_calculations(rep, mode=mode) + add_meth_export_calculations(rep, mode=mode) + for comm in comm_tec_map.keys(): t_filter2 = {"t": "meth_imp", "m": mode} rep.add("out::methanol-import", "select", "out:nl-t-ya-m", t_filter2) @@ -205,7 +217,7 @@ def add_meth_import_calculations(rep: message_ix.Reporter, mode: str = "feedstoc f"share::{comm}methanol-import", "div", f"out::{comm}methanol-import", - "out::methanol-total", + "out::methanol-prod-total", ) @@ -220,17 +232,27 @@ def add_biometh_final_share(rep: message_ix.Reporter, mode: str = "feedstock"): ---------- rep """ + add_meth_import_calculations(rep, mode=mode) + if mode == "feedstock": + t_filter2 = { + "t": ["meth_t_d"], + "m": [mode], + } + else: + t_filter2 = { + "t": ["meth_t_d", "furnace_methanol_refining"], + "m": [mode, "high_temp"], + } for comm in comm_tec_map.keys(): - if f"out::{comm}methanol-import" not in rep.keys(): - add_meth_import_calculations(rep, mode=mode) - - t_filter2 = {"t": "meth_t_d", "m": mode} - rep.add("in::methanol-final", "select", "in:nl-t-ya-m", t_filter2) + rep.add("in::methanol-final0", "select", "in:nl-t-ya-m", t_filter2) + rep.add( + "in::methanol-final", "sum", "in::methanol-final0", dimensions=["t", "m"] + ) rep.add( f"out::{comm}methanol-prod", "mul", - "out::methanol-total", - f"share::{comm}methanol", + "out::methanol-prod-total", + f"share::{comm}-methanol-prod", ) rep.add( f"out::{comm}methanol-final", diff --git a/message_ix_models/model/material/report/run_reporting.py b/message_ix_models/model/material/report/run_reporting.py index 320b5f1099..4a905a40a3 100644 --- a/message_ix_models/model/material/report/run_reporting.py +++ b/message_ix_models/model/material/report/run_reporting.py @@ -266,7 +266,7 @@ def run_fe_reporting(rep: message_ix.Reporter, model: str, scenario: str): py_df_all = add_chemicals_to_final_energy_variables(dfs, rep, model, scenario) - py_df_all = update_liquids_subvariables(rep, py_df_all, model, scenario) + py_df_all = split_fe_other(rep, py_df_all, model, scenario) df_final = ( py_df_all.filter(unit="dimensionless", keep=False) .convert_unit("GWa", "EJ") @@ -340,13 +340,13 @@ def add_chemicals_to_final_energy_variables( updated_fe_totals = [] for comm in [ "Electricity", + "Gases", + "Gases|Gas", "Liquids", "Liquids|Oil", "Solids", "Solids|Biomass", "Solids|Coal", - "Gases", - "Gases|Gas", "Hydrogen", ]: updated_fe_totals.append( @@ -363,10 +363,36 @@ def add_chemicals_to_final_energy_variables( return py_df_all -def update_liquids_subvariables( +def split_fe_other( rep: message_ix.Reporter, py_df_all: pyam.IamDataFrame, model: str, scenario: str ): + """This function takes the Final Energy|Industry|*|Liquids|Other values + and reallocates it to Liquids|Biomass/Coal/Oil/Gas based on the methanol + feedstock shares. + 1) calculates the feedstock shares of methanol production with message_ix.Reporter + 2) append the shares as temporary iamc variables them to the existing reporting + pyam object + 3) Uses pyam multiply feature to calculate shares with each "Liquids|Other" + timeseries + 4) Uses pyam aggregate to sum existing Liquids|Biomass/Coal/Oil/Gas with new + variables and store in separate pyam object + 5) Filters out existing (outdated) Liquids|Biomass/Coal/Oil/Gas from reporting + pyam object + 6) Concats the updated variables with the full reporting + + Parameters + ---------- + rep + py_df_all + model + scenario + + Returns + ------- + + """ add_biometh_final_share(rep, mode="fuel") + # set temporary filter on Reporter to speed up queries rep.set_filters( t=[ "meth_bunker", @@ -385,6 +411,7 @@ def update_liquids_subvariables( "meth_exp", "meth_imp", "meth_ind_fs", + "furnace_methanol_refining" ] ) for c, full_name in zip(