From f08dfa005d8458035094c33046cd2183c3a4b2c4 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 13:10:51 +0100 Subject: [PATCH 01/15] Add functionality to customize cost reduction rates within modules --- message_ix_models/tools/costs/decay.py | 461 +++++++++++++------------ 1 file changed, 241 insertions(+), 220 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index c6077444fd..768b4f68b2 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -7,180 +7,259 @@ from .regional_differentiation import get_raw_technology_mapping, subset_module_map -def get_cost_reduction_data(module) -> pd.DataFrame: - """Get cost reduction data from file. - - Raw data on cost reduction in 2100 for technologies are read from - :file:`data/[module]/cost_reduction_[module].csv`, based on GEA data. +def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): + # Get reduction scenarios for energy module + scenarios_energy = pd.read_csv( + package_data_path("costs", "energy", "scenarios_reduction.csv") + ) - Parameters - ---------- - module : str - Model module + # for technologies in energy_map that are not in scenarios_energy, + # assume scenario reduction across all scenarios is "none" + # add same columns as scenarios_energy + # and set all values to "none" except for message_technology column + scenarios_energy_no_reduction = energy_map_df.query( + "message_technology not in @scenarios_energy.message_technology" + )[["message_technology"]].assign( + **{ + col: "none" + for col in scenarios_energy.columns + if col != "message_technology" + } + ) - Returns - ------- - pandas.DataFrame - DataFrame with columns: + # combine scenarios_energy and scenarios_energy_no_reduction into scenarios_energy + # order by message_technology + scenarios_energy = ( + pd.concat([scenarios_energy, scenarios_energy_no_reduction], ignore_index=True) + .sort_values("message_technology") + .reset_index(drop=True) + ) - - message_technology: name of technology in MESSAGEix - - reduction_rate: the cost reduction rate (either very_low, low, medium, high, - or very_high) - - cost_reduction: cost reduction in 2100 (%) - """ + if module != "energy": + if package_data_path("costs", module, "scenarios_reduction.csv"): + scenarios_module = pd.read_csv( + package_data_path("costs", module, "scenarios_reduction.csv") + ) - # Get full list of technologies from mapping - tech_map = energy_map = get_raw_technology_mapping("energy") + # if a technology exists in scenarios_module that exists in scen_red_energy, + # remove it from scenarios_energy + scenarios_energy = scenarios_energy[ + ~scenarios_energy["message_technology"].isin( + scenarios_module["message_technology"] + ) + ] - # if module is not energy, run subset_module_map - if module != "energy": - module_map = get_raw_technology_mapping(module) - module_sub = subset_module_map(module_map) + # append scen_red_module to scen_red_energy + scenarios_joined = scenarios_energy._append(scenarios_module).reset_index( + drop=True + ) - # Remove energy technologies that exist in module mapping - energy_map = energy_map.query( - "message_technology not in @module_sub.message_technology" + # In tech map, get technologies that are not in scenarios_joined + # but are mapped to energy technologies + # then, use the scenarios_reduction.csv from the energy module for those technologies + scenarios_module_map_to_energy = ( + tech_map_df.query( + "(message_technology not in @scenarios_joined.message_technology) and \ + (reg_diff_source == 'energy')" + ) + .merge( + scenarios_energy.rename( + columns={"message_technology": "base_message_technology"} + ), + how="left", + left_on="reg_diff_technology", + right_on="base_message_technology", + ) + .drop(columns=["base_message_technology", "reg_diff_technology"]) + .drop_duplicates() + .reset_index(drop=1) ) - tech_map = pd.concat([energy_map, module_sub], ignore_index=True) + scenarios_module_map_to_energy = scenarios_module_map_to_energy[ + scenarios_joined.columns.intersection( + scenarios_module_map_to_energy.columns + ) + ] - # Read in raw data - gea_file_path = package_data_path("costs", "energy", "cost_reduction.csv") - energy_rates = ( - pd.read_csv(gea_file_path, header=8) - .melt( - id_vars=["message_technology", "technology_type"], - var_name="reduction_rate", - value_name="cost_reduction", + scenarios_module_map_to_energy.query("message_technology == 'fc_h2_aluminum'") + tech_map_df.query("message_technology == 'fc_h2_aluminum'") + scenarios_energy.query("message_technology == 'h2_fc_I'") + + tech_map_df.query( + "(message_technology not in @scenarios_joined.message_technology) and \ + (reg_diff_source == 'energy')" + ).query("message_technology == 'fc_h2_aluminum'") + + # for all technologies that are not in scenarios_module and + # are not mapped to energy technologies, + # assume scenario reduction across all scenarios is "none" + # add same columns as scenarios_joined + # and set all values to "none" except for message_technology column + scenarios_module_no_reduction = tech_map_df.query( + "message_technology not in @scenarios_joined.message_technology and \ + reg_diff_source != 'energy'" + )[["message_technology"]].assign( + **{ + col: "none" + for col in scenarios_joined.columns + if col != "message_technology" + } ) - .assign( - technology_type=lambda x: x.technology_type.fillna("NA"), - cost_reduction=lambda x: x.cost_reduction.fillna(0), - ) - .drop_duplicates() - .reset_index(drop=1) - ).reindex(["message_technology", "reduction_rate", "cost_reduction"], axis=1) - - # For module technologies with map_tech == energy, map to base technologies - # and use cost reduction data - module_rates_energy = ( - tech_map.query("reg_diff_source == 'energy'") - .drop(columns=["reg_diff_source", "base_year_reference_region_cost"]) - .merge( - energy_rates.rename( - columns={"message_technology": "base_message_technology"} - ), - how="inner", - left_on="reg_diff_technology", - right_on="base_message_technology", + + # combine scenarios_joined, scenarios_module_map_to_energy, + # and scenarios_module_no_reduction + # order by message_technology + scenarios_all = ( + pd.concat( + [ + scenarios_joined, + scenarios_module_map_to_energy, + scenarios_module_no_reduction, + ], + ignore_index=True, + ) + .sort_values("message_technology") + .reset_index(drop=True) ) - .drop(columns=["base_message_technology", "reg_diff_technology"]) - .drop_duplicates() - .reset_index(drop=1) - ).reindex(["message_technology", "reduction_rate", "cost_reduction"], axis=1) - - # Combine technologies that have cost reduction rates - df_reduction_techs = pd.concat( - [energy_rates, module_rates_energy], ignore_index=True + + return scenarios_all if module != "energy" else scenarios_energy + + +# do similar in _get_module_scenarios_reduction but for cost_reduction.csv +# and call it _get_module_cost_reduction +def _get_module_cost_reduction(module, energy_map_df, tech_map_df): + # Get cost reduction for energy module + reduction_energy = pd.read_csv( + package_data_path("costs", "energy", "cost_reduction.csv"), comment="#" ) - df_reduction_techs = df_reduction_techs.drop_duplicates().reset_index(drop=1) - - # Create unique dataframe of cost reduction rates - # and make all cost_reduction values 0 - un_rates = pd.DataFrame( - { - "reduction_rate": ["none"], - "cost_reduction": [0], - "key": "z", - } + + # for technologies in energy_map that are not in reduction_energy, + # assume scenario reduction across all scenarios is "none" + # add same columns as reduction_energy + # and set all values to "none" except for message_technology column + reduction_energy_no_reduction = energy_map_df.query( + "message_technology not in @reduction_energy.message_technology" + )[["message_technology"]].assign( + **{col: 0 for col in reduction_energy.columns if col != "message_technology"} ) - # For remaining module technologies that are not mapped to energy technologies, - # assume no cost reduction - module_rates_noreduction = ( - tech_map.query( - "message_technology not in @df_reduction_techs.message_technology" - ) - .assign(key="z") - .merge(un_rates, on="key") - .drop(columns=["key"]) - ).reindex(["message_technology", "reduction_rate", "cost_reduction"], axis=1) - # Concatenate base and module rates - all_rates = pd.concat( - [energy_rates, module_rates_energy, module_rates_noreduction], - ignore_index=True, - ).reset_index(drop=1) + # combine reduction_energy and reduction_energy_no_reduction into reduction_energy + # order by message_technology + reduction_energy = ( + pd.concat([reduction_energy, reduction_energy_no_reduction], ignore_index=True) + .sort_values("message_technology") + .reset_index(drop=True) + .drop(columns=["technology_type"]) + ) - return all_rates + if module != "energy": + if package_data_path("costs", module, "cost_reduction.csv"): + reduction_module = pd.read_csv( + package_data_path("costs", module, "cost_reduction.csv"), comment="#" + ) + # if a technology exists in scen_red_module that exists in scen_red_energy, + # remove it from scen_red_energy + reduction_energy = reduction_energy[ + ~reduction_energy["message_technology"].isin( + reduction_module["message_technology"] + ) + ] -def get_technology_reduction_scenarios_data( - first_year: int, module: str -) -> pd.DataFrame: - """Read in technology first year and cost reduction scenarios. + # append scen_red_module to scen_red_energy + reduction_joined = ( + reduction_energy._append(reduction_module) + .reset_index(drop=True) + .drop(columns=["technology_type"]) + ) - Raw data on technology first year and reduction scenarios are read from - :file:`data/costs/[module]/first_year_[module]`. The first year the technology is - available in MESSAGEix is adjusted to be the base year if the original first year is - before the base year. + # In tech map, get technologies that are not in reduction_joined + # but are mapped to energy technologies + # then, use the reduction from the energy module for those technologies + reduction_module_map_to_energy = ( + tech_map_df.query( + "(message_technology not in @reduction_joined.message_technology) and \ + (reg_diff_source == 'energy')" + ) + .merge( + reduction_energy.rename( + columns={"message_technology": "base_message_technology"} + ), + how="inner", + left_on="reg_diff_technology", + right_on="base_message_technology", + ) + .drop(columns=["base_message_technology", "reg_diff_technology"]) + .drop_duplicates() + .reset_index(drop=1) + ) - Raw data on cost reduction scenarios are read from - :file:`data/costs/[module]/scenarios_reduction_[module].csv`. + reduction_module_map_to_energy = reduction_module_map_to_energy[ + reduction_joined.columns.intersection( + reduction_module_map_to_energy.columns + ) + ] - Assumptions are made for the non-energy module for technologies' cost reduction - scenarios that are not given. + # for all technologies that are not in reduction_module and + # are not mapped to energy technologies, + # assume scenario reduction across all scenarios is "none" + # add same columns as reduction_joined + # and set all values to "none" except for message_technology column + reduction_module_no_reduction = tech_map_df.query( + "message_technology not in @reduction_joined.message_technology and \ + reg_diff_source != 'energy'" + )[["message_technology"]].assign( + **{ + col: 0 + for col in reduction_joined.columns + if col != "message_technology" + } + ) - Parameters - ---------- - base_year : int, optional - The base year, by default set to global BASE_YEAR - module : str - Model module + # combine reduction_joined, reduction_module_map_to_energy, + # and reduction_module_no_reduction + # order by message_technology + reduction_all = ( + pd.concat( + [ + reduction_joined, + reduction_module_map_to_energy, + reduction_module_no_reduction, + ], + ignore_index=True, + ) + .sort_values("message_technology") + .reset_index(drop=True) + ) - Returns - ------- - pandas.DataFrame - DataFrame with columns: + return reduction_all if module != "energy" else reduction_energy - - message_technology: name of technology in MESSAGEix - - scenario: scenario (SSP1, SSP2, SSP3, SSP4, SSP5, or LED) - - first_technology_year: first year the technology is available in MESSAGEix. - - reduction_rate: the cost reduction rate (either very_low, low, medium, high, - or very_high) - """ - energy_first_year_file = package_data_path("costs", "energy", "tech_map.csv") - df_first_year = pd.read_csv(energy_first_year_file, skiprows=4)[ - ["message_technology", "first_year_original"] - ] +# create function to get technology reduction scenarios data +def get_technology_reduction_scenarios_data(first_year, module): + # Get full list of technologies from mapping + tech_map = energy_map = get_raw_technology_mapping("energy") + # if module is not energy, run subset_module_map if module != "energy": - module_first_year_file = package_data_path("costs", module, "tech_map.csv") - module_first_year = pd.read_csv(module_first_year_file)[ - ["message_technology", "first_year_original"] - ] - df_first_year = pd.concat( - [df_first_year, module_first_year], ignore_index=True - ).drop_duplicates() - - tech_map = tech_energy = get_raw_technology_mapping("energy") + module_map = get_raw_technology_mapping(module) + module_sub = subset_module_map(module_map) - if module != "energy": - tech_module = subset_module_map(get_raw_technology_mapping(module)) - tech_energy = tech_energy.query( - "message_technology not in @tech_module.message_technology" + # Remove energy technologies that exist in module mapping + energy_map = energy_map.query( + "message_technology not in @module_sub.message_technology" ) - tech_map = pd.concat([tech_energy, tech_module], ignore_index=True) - tech_map = tech_map.reindex( - ["message_technology", "reg_diff_source", "reg_diff_technology"], axis=1 - ).drop_duplicates() + tech_map = pd.concat([energy_map, module_sub], ignore_index=True) + + scenarios_reduction = _get_module_scenarios_reduction(module, energy_map, tech_map) + cost_reduction = _get_module_cost_reduction(module, energy_map, tech_map) - # Adjust first year: - # - if first year is missing, set to base year - # - if first year is after base year, then keep assigned first year - all_first_year = ( - pd.merge(tech_map, df_first_year, on="message_technology", how="left") + cost_reduction.query("message_technology == 'bio_hpl'") + + # get first year values + adj_first_year = ( + tech_map[["message_technology", "first_year_original"]] .assign( first_technology_year=lambda x: np.where( x.first_year_original.isnull(), @@ -196,78 +275,26 @@ def get_technology_reduction_scenarios_data( .drop(columns=["first_year_original"]) ) - # Create new column for scenario_technology - # - if reg_diff_source == weo, then scenario_technology = message_technology - # - if reg_diff_source == energy, then scenario_technology = reg_diff_technology - # - otherwise, scenario_technology = message_technology - adj_first_year = ( - all_first_year.assign( - scenario_technology=lambda x: np.where( - x.reg_diff_source == "weo", - x.message_technology, - np.where( - x.reg_diff_source == "energy", - x.reg_diff_technology, - x.message_technology, - ), - ) - ) - .drop(columns=["reg_diff_source", "reg_diff_technology"]) - .drop_duplicates() - .reset_index(drop=1) - ) - - # Merge with energy technologies that have given scenarios - energy_scen_file = package_data_path("costs", "energy", "scenarios_reduction.csv") - df_energy_scen = pd.read_csv(energy_scen_file).rename( - columns={"message_technology": "scenario_technology"} - ) - - existing_scens = ( - pd.merge( - adj_first_year, - df_energy_scen, - on=["scenario_technology"], - how="inner", - ) - .drop(columns=["scenario_technology"]) - .melt( - id_vars=[ - "message_technology", - "first_technology_year", - ], - var_name="scenario", - value_name="reduction_rate", - ) + # convert scenarios_reduction and cost_reduction to long format + scenarios_reduction_long = scenarios_reduction.melt( + id_vars=["message_technology"], var_name="scenario", value_name="reduction_rate" ) - - # Create dataframe of SSP1-SSP5 and LED scenarios with "none" cost reduction rate - un_scens = pd.DataFrame( - { - "scenario": ["SSP1", "SSP2", "SSP3", "SSP4", "SSP5", "LED"], - "reduction_rate": "none", - "key": "z", - } + cost_reduction_long = cost_reduction.melt( + id_vars=["message_technology"], + var_name="reduction_rate", + value_name="cost_reduction", ) - # Get remaining technologies that do not have given scenarios - remaining_scens = ( - adj_first_year.query( - "message_technology not in @existing_scens.message_technology.unique()" - ) - .assign(key="z") - .merge(un_scens, on="key") - .drop(columns=["key", "scenario_technology"]) - ) + # merge scenarios_reduction_long and cost_reduction_long + # with adj_first_year + df = scenarios_reduction_long.merge( + cost_reduction_long, on=["message_technology", "reduction_rate"], how="left" + ).merge(adj_first_year, on="message_technology", how="left") - # Concatenate all technologies - all_scens = ( - pd.concat([existing_scens, remaining_scens], ignore_index=True) - .sort_values(by=["message_technology", "scenario"]) - .reset_index(drop=1) - ) + # if reduction_rate is "none", then set cost_reduction to 0 + df["cost_reduction"] = np.where(df.reduction_rate == "none", 0, df.cost_reduction) - return all_scens + return df def project_ref_region_inv_costs_using_reduction_rates( @@ -309,15 +336,9 @@ def project_ref_region_inv_costs_using_reduction_rates( - inv_cost_ref_region_decay: investment cost in reference region in year. """ - # Get cost reduction data - df_cost_reduction = get_cost_reduction_data(config.module) - - # Get scenarios data - df_scenarios = get_technology_reduction_scenarios_data(config.y0, config.module) - - # Merge cost reduction data with cost reduction rates data - df_cost_reduction = df_cost_reduction.merge( - df_scenarios, on=["message_technology", "reduction_rate"], how="left" + # Get scenarios cost reduction data for technologies + df_cost_reduction = get_technology_reduction_scenarios_data( + config.y0, config.module ) # Filter for reference region, and merge with reduction scenarios and discount rates From 94684eb1583c6520f5c3a27c73b6943af605cfbc Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 13:27:57 +0100 Subject: [PATCH 02/15] Add test data for customizing cost reductions in materials module --- message_ix_models/data/costs/materials/cost_reduction.csv | 7 +++++++ .../data/costs/materials/scenarios_reduction.csv | 5 +++++ 2 files changed, 12 insertions(+) create mode 100755 message_ix_models/data/costs/materials/cost_reduction.csv create mode 100755 message_ix_models/data/costs/materials/scenarios_reduction.csv diff --git a/message_ix_models/data/costs/materials/cost_reduction.csv b/message_ix_models/data/costs/materials/cost_reduction.csv new file mode 100755 index 0000000000..9d07aa27f1 --- /dev/null +++ b/message_ix_models/data/costs/materials/cost_reduction.csv @@ -0,0 +1,7 @@ +# Cost reduction in 2100,,,,,, +# ,,,,,, +# Units: % ,,,,,, +message_technology,technology_type,very_low,low,medium,high,very_high +furnace_coal_cement,Materials,0.23,0.24,0.25,0.26,0.27 +gas_ct,Gas/Oil,0.99,0.99,0.99,0.99,0.99 +bio_hpl,Materials,0.77,0.77,0.77,0.77,0.77 \ No newline at end of file diff --git a/message_ix_models/data/costs/materials/scenarios_reduction.csv b/message_ix_models/data/costs/materials/scenarios_reduction.csv new file mode 100755 index 0000000000..9e7e005e00 --- /dev/null +++ b/message_ix_models/data/costs/materials/scenarios_reduction.csv @@ -0,0 +1,5 @@ +message_technology,SSP1,SSP2,SSP3,SSP4,SSP5,LED +furnace_coal_cement,low,low,low,low,low,low +manuf_steel,medium,medium,medium,medium,medium,medium +hp_elec_refining,high,high,high,high,high,high +bio_hpl,very_low,very_low,very_low,very_low,very_low,very_low \ No newline at end of file From 6cc856c22f7399e393c03445c00a7238c8dd3ce7 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 13:31:14 +0100 Subject: [PATCH 03/15] Remove test for get_cost_reduction_data() because function has been removed --- message_ix_models/tests/tools/costs/test_decay.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/message_ix_models/tests/tools/costs/test_decay.py b/message_ix_models/tests/tools/costs/test_decay.py index 2ae706e3a3..74b10edc8b 100644 --- a/message_ix_models/tests/tools/costs/test_decay.py +++ b/message_ix_models/tests/tools/costs/test_decay.py @@ -4,7 +4,6 @@ from message_ix_models.tools.costs import Config from message_ix_models.tools.costs.decay import ( - get_cost_reduction_data, get_technology_reduction_scenarios_data, project_ref_region_inv_costs_using_reduction_rates, ) @@ -21,18 +20,6 @@ ("cooling", {"coal_ppl__cl_fresh", "gas_cc__air", "nuc_lc__ot_fresh"}), ), ) -def test_get_cost_reduction_data(module: str, t_exp) -> None: - # The function runs without error - result = get_cost_reduction_data(module) - - # Expected MESSAGEix-GLOBIOM technologies are present in the data - assert t_exp <= set(result.message_technology.unique()) - - # Values of the "cost reduction" columns are between 0 and 1 - stats = result.cost_reduction.describe() - assert 0 <= stats["min"] and stats["max"] <= 1 - - @pytest.mark.parametrize("module", ("energy", "materials", "cooling")) def test_get_technology_reduction_scenarios_data(module: str) -> None: config = Config() From 0e3ba6389ea76df1f91546dc94ff0f2bb069ebde Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 13:33:50 +0100 Subject: [PATCH 04/15] Remove pytest parameterization for test_get_cost_reduction_data() --- message_ix_models/tests/tools/costs/test_decay.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/message_ix_models/tests/tools/costs/test_decay.py b/message_ix_models/tests/tools/costs/test_decay.py index 74b10edc8b..11b898c12a 100644 --- a/message_ix_models/tests/tools/costs/test_decay.py +++ b/message_ix_models/tests/tools/costs/test_decay.py @@ -12,14 +12,6 @@ ) -@pytest.mark.parametrize( - "module, t_exp", - ( - ("energy", {"coal_ppl", "gas_ppl", "gas_cc", "solar_res1"}), - ("materials", {"biomass_NH3", "MTO_petro", "furnace_foil_steel"}), - ("cooling", {"coal_ppl__cl_fresh", "gas_cc__air", "nuc_lc__ot_fresh"}), - ), -) @pytest.mark.parametrize("module", ("energy", "materials", "cooling")) def test_get_technology_reduction_scenarios_data(module: str) -> None: config = Config() From 349c8eed714fa8bfff6fe8dff5dc16c75931c4ec Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 14:22:54 +0100 Subject: [PATCH 05/15] Edit to pass linting --- message_ix_models/tools/costs/decay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 768b4f68b2..cc3f91bd7f 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -56,7 +56,7 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): # In tech map, get technologies that are not in scenarios_joined # but are mapped to energy technologies - # then, use the scenarios_reduction.csv from the energy module for those technologies + # then use the scenarios reduction from the energy module for those technologies scenarios_module_map_to_energy = ( tech_map_df.query( "(message_technology not in @scenarios_joined.message_technology) and \ From cce50716ab4cb3917f08746c9a97200273994572 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 15:16:55 +0100 Subject: [PATCH 06/15] Add `else` statement for non-energy modules without custom files --- message_ix_models/tools/costs/decay.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index cc3f91bd7f..65db7f09d7 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -53,6 +53,8 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): scenarios_joined = scenarios_energy._append(scenarios_module).reset_index( drop=True ) + else: + scenarios_joined = scenarios_energy # In tech map, get technologies that are not in scenarios_joined # but are mapped to energy technologies @@ -172,6 +174,8 @@ def _get_module_cost_reduction(module, energy_map_df, tech_map_df): .reset_index(drop=True) .drop(columns=["technology_type"]) ) + else: + reduction_joined = reduction_energy # In tech map, get technologies that are not in reduction_joined # but are mapped to energy technologies From 45381dd2831c1beef497a0e05794a2428ab78e46 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 15:39:26 +0100 Subject: [PATCH 07/15] Modify if-else statements for reading in custom files --- message_ix_models/tools/costs/decay.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 65db7f09d7..08eaf21039 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -36,10 +36,12 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): ) if module != "energy": - if package_data_path("costs", module, "scenarios_reduction.csv"): - scenarios_module = pd.read_csv( - package_data_path("costs", module, "scenarios_reduction.csv") - ) + ffile = package_data_path("costs", module, "scenarios_reduction.csv") + + # if file exists, read it + # else, scenarios_joined is the same as scenarios_energy + if ffile: + scenarios_module = pd.read_csv(ffile) # if a technology exists in scenarios_module that exists in scen_red_energy, # remove it from scenarios_energy @@ -54,7 +56,7 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): drop=True ) else: - scenarios_joined = scenarios_energy + scenarios_joined = scenarios_energy.copy() # In tech map, get technologies that are not in scenarios_joined # but are mapped to energy technologies @@ -155,10 +157,9 @@ def _get_module_cost_reduction(module, energy_map_df, tech_map_df): ) if module != "energy": - if package_data_path("costs", module, "cost_reduction.csv"): - reduction_module = pd.read_csv( - package_data_path("costs", module, "cost_reduction.csv"), comment="#" - ) + ffile = package_data_path("costs", module, "cost_reduction.csv") + if ffile: + reduction_module = pd.read_csv(ffile, comment="#") # if a technology exists in scen_red_module that exists in scen_red_energy, # remove it from scen_red_energy @@ -175,7 +176,7 @@ def _get_module_cost_reduction(module, energy_map_df, tech_map_df): .drop(columns=["technology_type"]) ) else: - reduction_joined = reduction_energy + reduction_joined = reduction_energy.copy() # In tech map, get technologies that are not in reduction_joined # but are mapped to energy technologies From 89f7b3dd93c962f143618c05bf293aeb573cba05 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 15:45:00 +0100 Subject: [PATCH 08/15] Fix check for if files exist --- message_ix_models/tools/costs/decay.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 08eaf21039..66e29bfa8f 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -1,3 +1,5 @@ +import os + import numpy as np import pandas as pd @@ -40,7 +42,7 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): # if file exists, read it # else, scenarios_joined is the same as scenarios_energy - if ffile: + if os.path.exists(ffile): scenarios_module = pd.read_csv(ffile) # if a technology exists in scenarios_module that exists in scen_red_energy, @@ -158,7 +160,8 @@ def _get_module_cost_reduction(module, energy_map_df, tech_map_df): if module != "energy": ffile = package_data_path("costs", module, "cost_reduction.csv") - if ffile: + + if os.path.exists(ffile): reduction_module = pd.read_csv(ffile, comment="#") # if a technology exists in scen_red_module that exists in scen_red_energy, From 9d33ffc5bcf8a1543ac584b984c271c5b8304750 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 16:39:20 +0100 Subject: [PATCH 09/15] Add docstrings for new functions --- message_ix_models/tools/costs/decay.py | 47 ++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 66e29bfa8f..65e5acd072 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -10,6 +10,29 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): + """Get scenarios reduction categories for a module. + + Parameters + ---------- + module : str + The module for which to get scenarios reduction categories. + energy_map_df : pandas.DataFrame + The technology mapping for the energy module. + tech_map_df : pandas.DataFrame + The technology mapping for the specific module. + + Returns + ------- + pandas.DataFrame + DataFrame with columns: + - message_technology: name of technology in MESSAGEix + - SSP1: scenario reduction category for SSP1 + - SSP2: scenario reduction category for SSP2 + - SSP3: scenario reduction category for SSP3 + - SSP4: scenario reduction category for SSP4 + - SSP5: scenario reduction category for SSP5 + - LED: scenario reduction category for LED + """ # Get reduction scenarios for energy module scenarios_energy = pd.read_csv( package_data_path("costs", "energy", "scenarios_reduction.csv") @@ -131,9 +154,29 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): return scenarios_all if module != "energy" else scenarios_energy -# do similar in _get_module_scenarios_reduction but for cost_reduction.csv -# and call it _get_module_cost_reduction def _get_module_cost_reduction(module, energy_map_df, tech_map_df): + """Get cost reduction values for technologies in a module. + + Parameters + ---------- + module : str + The module for which to get cost reduction values. + energy_map_df : pandas.DataFrame + The technology mapping for the energy module. + tech_map_df : pandas.DataFrame + The technology mapping for the specific module. + + Returns + ------- + pandas.DataFrame + DataFrame with columns: + - message_technology: name of technology in MESSAGEix + - very_low: cost reduction for "none" scenario + - low: cost reduction for "low" scenario + - medium: cost reduction for "moderate" scenario + - high: cost reduction for "high" scenario + - very_high: cost reduction for "very high" scenario + """ # Get cost reduction for energy module reduction_energy = pd.read_csv( package_data_path("costs", "energy", "cost_reduction.csv"), comment="#" From 68a1d3b381fac31f60f74dfea8628a3280a825c2 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Wed, 27 Nov 2024 16:41:40 +0100 Subject: [PATCH 10/15] Add PR #255 to doc/whatsnew --- doc/whatsnew.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 1b8271b081..52d2fe2100 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -15,7 +15,7 @@ Next release - Expand :doc:`repro` with sections on :ref:`repro-doc` and :ref:`versioning`, including :ref:`a list of external model names and ‘versions’ ` like “MESSAGEix-GLOBIOM 2.0” (:issue:`224`, :pull:`226`). - Update :doc:`/transport/index` (:pull:`213`). - Add "LED", "SSP4", and "SSP5" as values for the :program:`--ssp=…` option in :func:`.common_params` (:pull:`233`). -- Fix and update :doc:`/api/tools-costs` (:pull:`219`, :pull:`206`, :pull:`221`, :pull:`227`, :pull:`222`) +- Fix and update :doc:`/api/tools-costs` (:pull:`219`, :pull:`206`, :pull:`221`, :pull:`227`, :pull:`222`, :pull:`255`) - Fix naming of GDP and population columns in SSP data aggregation (:pull:`219`). - Edit inputs for storage, CSP, hydrogen, and industry technologies (:pull:`206`). @@ -24,6 +24,7 @@ Next release - Reconfigure use and implementation of technology variants/modules to be more agnostic (:pull:`221`). - Change cost decay to reach reduction percentage specified on the year 2100 (:pull:`227`). - Add `cooling` technology variant/module (:pull:`222`). + - Add functionality to specify cost reduction values and cost reduction scenarios in a module (:pull:`255`). - Improve and extend :doc:`/material/index` (:pull:`218`, :pull:`253`). - Release of MESSAGEix-Materials 1.1.0 (:doc:`/material/v1.1.0`). From 5fe759e6b01c97f727375d1228bd5aa7312922cf Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Mon, 2 Dec 2024 17:00:22 +0100 Subject: [PATCH 11/15] Add type hints --- message_ix_models/tools/costs/decay.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 65e5acd072..617393eef0 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -9,7 +9,9 @@ from .regional_differentiation import get_raw_technology_mapping, subset_module_map -def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): +def _get_module_scenarios_reduction( + module: str, energy_map_df: pd.DataFrame, tech_map_df: pd.DataFrame +) -> pd.DataFrame: """Get scenarios reduction categories for a module. Parameters @@ -154,7 +156,9 @@ def _get_module_scenarios_reduction(module, energy_map_df, tech_map_df): return scenarios_all if module != "energy" else scenarios_energy -def _get_module_cost_reduction(module, energy_map_df, tech_map_df): +def _get_module_cost_reduction( + module: str, energy_map_df: pd.DataFrame, tech_map_df: pd.DataFrame +) -> pd.DataFrame: """Get cost reduction values for technologies in a module. Parameters @@ -287,7 +291,9 @@ def _get_module_cost_reduction(module, energy_map_df, tech_map_df): # create function to get technology reduction scenarios data -def get_technology_reduction_scenarios_data(first_year, module): +def get_technology_reduction_scenarios_data( + first_year: int, module: str +) -> pd.DataFrame: # Get full list of technologies from mapping tech_map = energy_map = get_raw_technology_mapping("energy") From 7945323efeb2728044d780103f3de8ddf126473d Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Mon, 2 Dec 2024 17:10:14 +0100 Subject: [PATCH 12/15] Add tests for `_get_module_cost_reduction()` and `_get_module_scenarios_reduction()` --- .../tests/tools/costs/test_decay.py | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/message_ix_models/tests/tools/costs/test_decay.py b/message_ix_models/tests/tools/costs/test_decay.py index 11b898c12a..7e9ac1a0a1 100644 --- a/message_ix_models/tests/tools/costs/test_decay.py +++ b/message_ix_models/tests/tools/costs/test_decay.py @@ -1,19 +1,89 @@ from typing import Literal +import pandas as pd import pytest from message_ix_models.tools.costs import Config from message_ix_models.tools.costs.decay import ( + _get_module_cost_reduction, + _get_module_scenarios_reduction, get_technology_reduction_scenarios_data, project_ref_region_inv_costs_using_reduction_rates, ) from message_ix_models.tools.costs.regional_differentiation import ( apply_regional_differentiation, + get_raw_technology_mapping, + subset_module_map, ) +@pytest.mark.parametrize( + "module, t_exp", + ( + ("energy", {"coal_ppl", "gas_ppl", "gas_cc", "solar_res1"}), + ("materials", {"biomass_NH3", "MTO_petro", "furnace_foil_steel"}), + ("cooling", {"coal_ppl__cl_fresh", "gas_cc__air", "nuc_lc__ot_fresh"}), + ), +) +def test_get_module_scenarios_reduction( + module: Literal["energy", "materials", "cooling"], t_exp +) -> None: + tech_map = energy_map = get_raw_technology_mapping("energy") + + # if module is not energy, run subset_module_map + if module != "energy": + module_map = get_raw_technology_mapping(module) + module_sub = subset_module_map(module_map) + + # Remove energy technologies that exist in module mapping + energy_map = energy_map.query( + "message_technology not in @module_sub.message_technology" + ) + + tech_map = pd.concat([energy_map, module_sub], ignore_index=True) + + result = _get_module_scenarios_reduction(module, energy_map, tech_map) + + # Expected MESSAGEix-GLOBIOM technologies are present in the data + assert t_exp <= set(result.message_technology.unique()) + + +@pytest.mark.parametrize( + "module, t_exp", + ( + ("energy", {"coal_ppl", "gas_ppl", "gas_cc", "solar_res1"}), + ("materials", {"biomass_NH3", "MTO_petro", "furnace_foil_steel"}), + ("cooling", {"coal_ppl__cl_fresh", "gas_cc__air", "nuc_lc__ot_fresh"}), + ), +) +def test_get_module_cost_reduction( + module: Literal["energy", "materials", "cooling"], t_exp +) -> None: + tech_map = energy_map = get_raw_technology_mapping("energy") + + # if module is not energy, run subset_module_map + if module != "energy": + module_map = get_raw_technology_mapping(module) + module_sub = subset_module_map(module_map) + + # Remove energy technologies that exist in module mapping + energy_map = energy_map.query( + "message_technology not in @module_sub.message_technology" + ) + + tech_map = pd.concat([energy_map, module_sub], ignore_index=True) + + # The function runs without error + result = _get_module_cost_reduction(module, energy_map, tech_map) + + # Expected MESSAGEix-GLOBIOM technologies are present in the data + assert t_exp <= set(result.message_technology.unique()) + + @pytest.mark.parametrize("module", ("energy", "materials", "cooling")) -def test_get_technology_reduction_scenarios_data(module: str) -> None: +def test_get_technology_reduction_scenarios_data( + module: Literal["energy", "materials", "cooling"], +) -> None: config = Config() # The function runs without error result = get_technology_reduction_scenarios_data(config.y0, module=module) From 80a530d6e7260f66cbb338552a9e3282eb766292 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Mon, 2 Dec 2024 17:11:00 +0100 Subject: [PATCH 13/15] Remove dummy data for materials module --- message_ix_models/data/costs/materials/cost_reduction.csv | 5 +---- .../data/costs/materials/scenarios_reduction.csv | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/message_ix_models/data/costs/materials/cost_reduction.csv b/message_ix_models/data/costs/materials/cost_reduction.csv index 9d07aa27f1..8acb5a4959 100755 --- a/message_ix_models/data/costs/materials/cost_reduction.csv +++ b/message_ix_models/data/costs/materials/cost_reduction.csv @@ -1,7 +1,4 @@ # Cost reduction in 2100,,,,,, # ,,,,,, # Units: % ,,,,,, -message_technology,technology_type,very_low,low,medium,high,very_high -furnace_coal_cement,Materials,0.23,0.24,0.25,0.26,0.27 -gas_ct,Gas/Oil,0.99,0.99,0.99,0.99,0.99 -bio_hpl,Materials,0.77,0.77,0.77,0.77,0.77 \ No newline at end of file +message_technology,technology_type,very_low,low,medium,high,very_high \ No newline at end of file diff --git a/message_ix_models/data/costs/materials/scenarios_reduction.csv b/message_ix_models/data/costs/materials/scenarios_reduction.csv index 9e7e005e00..f084dbbf07 100755 --- a/message_ix_models/data/costs/materials/scenarios_reduction.csv +++ b/message_ix_models/data/costs/materials/scenarios_reduction.csv @@ -1,5 +1 @@ -message_technology,SSP1,SSP2,SSP3,SSP4,SSP5,LED -furnace_coal_cement,low,low,low,low,low,low -manuf_steel,medium,medium,medium,medium,medium,medium -hp_elec_refining,high,high,high,high,high,high -bio_hpl,very_low,very_low,very_low,very_low,very_low,very_low \ No newline at end of file +message_technology,SSP1,SSP2,SSP3,SSP4,SSP5,LED \ No newline at end of file From 8ed052312b4f50e7a011abbbedf99a3d6e71fa13 Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Mon, 2 Dec 2024 17:12:46 +0100 Subject: [PATCH 14/15] Change type hint for `module` from string to Literal --- message_ix_models/tools/costs/decay.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/message_ix_models/tools/costs/decay.py b/message_ix_models/tools/costs/decay.py index 617393eef0..c55384ace6 100644 --- a/message_ix_models/tools/costs/decay.py +++ b/message_ix_models/tools/costs/decay.py @@ -1,4 +1,5 @@ import os +from typing import Literal import numpy as np import pandas as pd @@ -10,7 +11,9 @@ def _get_module_scenarios_reduction( - module: str, energy_map_df: pd.DataFrame, tech_map_df: pd.DataFrame + module: Literal["energy", "materials", "cooling"], + energy_map_df: pd.DataFrame, + tech_map_df: pd.DataFrame, ) -> pd.DataFrame: """Get scenarios reduction categories for a module. @@ -157,7 +160,9 @@ def _get_module_scenarios_reduction( def _get_module_cost_reduction( - module: str, energy_map_df: pd.DataFrame, tech_map_df: pd.DataFrame + module: Literal["energy", "materials", "cooling"], + energy_map_df: pd.DataFrame, + tech_map_df: pd.DataFrame, ) -> pd.DataFrame: """Get cost reduction values for technologies in a module. @@ -292,7 +297,7 @@ def _get_module_cost_reduction( # create function to get technology reduction scenarios data def get_technology_reduction_scenarios_data( - first_year: int, module: str + first_year: int, module: Literal["energy", "materials", "cooling"] ) -> pd.DataFrame: # Get full list of technologies from mapping tech_map = energy_map = get_raw_technology_mapping("energy") From 5993c5584d3cf0b674f40e200b8640d9090201da Mon Sep 17 00:00:00 2001 From: Measrainsey Meng Date: Tue, 3 Dec 2024 11:20:20 +0100 Subject: [PATCH 15/15] Add type hints for technology sets --- message_ix_models/tests/tools/costs/test_decay.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/message_ix_models/tests/tools/costs/test_decay.py b/message_ix_models/tests/tools/costs/test_decay.py index 7e9ac1a0a1..68ba9fce98 100644 --- a/message_ix_models/tests/tools/costs/test_decay.py +++ b/message_ix_models/tests/tools/costs/test_decay.py @@ -26,7 +26,7 @@ ), ) def test_get_module_scenarios_reduction( - module: Literal["energy", "materials", "cooling"], t_exp + module: Literal["energy", "materials", "cooling"], t_exp: set[str] ) -> None: tech_map = energy_map = get_raw_technology_mapping("energy") @@ -57,7 +57,7 @@ def test_get_module_scenarios_reduction( ), ) def test_get_module_cost_reduction( - module: Literal["energy", "materials", "cooling"], t_exp + module: Literal["energy", "materials", "cooling"], t_exp: set[str] ) -> None: tech_map = energy_map = get_raw_technology_mapping("energy") @@ -111,7 +111,9 @@ def test_get_technology_reduction_scenarios_data( ), ) def test_project_ref_region_inv_costs_using_reduction_rates( - module: Literal["energy", "materials", "cooling"], t_exp, t_excluded + module: Literal["energy", "materials", "cooling"], + t_exp: set[str], + t_excluded: set[str], ) -> None: # Set up config = Config(module=module)