From bc2a6b21d999121585fcf200051409b95d527976 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 14 Mar 2024 16:29:32 +0100 Subject: [PATCH 01/56] example variables and models --- esmvaltool/diag_scripts/perfmetrics/main.py | 107 +++++++++++++++++ .../recipes/recipe_perfmetrics_python.yml | 111 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 esmvaltool/diag_scripts/perfmetrics/main.py create mode 100644 esmvaltool/recipes/recipe_perfmetrics_python.yml diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py new file mode 100644 index 0000000000..e9b736a550 --- /dev/null +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -0,0 +1,107 @@ +import matplotlib as mpl +from esmvalcore import preprocessor as pp +import logging +import matplotlib.pyplot as plt +import esmvaltool.diag_scripts.shared as e +from esmvaltool.diag_scripts.shared import ( + # ProvenanceLogger, + get_plot_filename, + group_metadata, + run_diagnostic, + select_metadata, +) + +import iris +import numpy as np +import matplotlib.pyplot as plt + + +def heatmap(data, row_labels, col_labels, ax=None, + cbar_kw=None, cbarlabel="", **kwargs): + """ + Create a heatmap from a numpy array and two lists of labels. + + Parameters + ---------- + data + A 2D numpy array of shape (M, N). + row_labels + A list or array of length M with the labels for the rows. + col_labels + A list or array of length N with the labels for the columns. + ax + A `matplotlib.axes.Axes` instance to which the heatmap is plotted. If + not provided, use current axes or create a new one. Optional. + cbar_kw + A dictionary with arguments to `matplotlib.Figure.colorbar`. Optional. + cbarlabel + The label for the colorbar. Optional. + **kwargs + All other arguments are forwarded to `imshow`. + """ + + if ax is None: + ax = plt.gca() + + if cbar_kw is None: + cbar_kw = {} + + # Plot the heatmap + im = ax.imshow(data, **kwargs) + + # Create colorbar + cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw) + cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") + + # Show all ticks and label them with the respective list entries. + ax.set_xticks(np.arange(data.shape[1]), labels=col_labels) + ax.set_yticks(np.arange(data.shape[0]), labels=row_labels) + + # Rotate the tick labels and set their alignment. + plt.setp(ax.get_xticklabels(), rotation=90, ha="right", + va="top", rotation_mode="anchor") + + # Turn spines off and create white grid. + # ax.spines[:].set_visible(False) + + ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True) + ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True) + ax.grid(which="minor", color="black", linestyle='-', linewidth=1) + ax.tick_params(which="both", bottom=False, left=False) + + return im, cbar + + +def remove_reference(metas): + return [meta for meta in metas if not meta.get("reference_for_metric", False)] + + +def plot(cfg, metas): + x = cfg.get("x", "dataset") + y = cfg.get("y", "variable_group") + y_labels = group_metadata(metas, y).keys() + x_labels = group_metadata(metas, x).keys() + data = np.zeros((len(y_labels), len(x_labels))) + for i, y_label in enumerate(y_labels): + #y_metas = select_metadata(metas, variable_group=y_label) + for j, x_label in enumerate(x_labels): + selection = {x: x_label, y: y_label} + meta = select_metadata(metas, **selection)[0] + cube = iris.load_cube(meta["filename"]) + data[i][j] = cube.data + ax = heatmap(data, y_labels, x_labels, **{"cmap": "RdYlBu_r"}, cbar_kw={"extend": "both"}) + plt.savefig("heatmap.png") + + +def main(cfg): + metas = cfg["input_data"].values() + metas = remove_reference(metas) + plot(cfg, metas) + # grouped = group_metadata(metas, "project") + # for group, group_metas in grouped.items(): + # plot_group() + + +if __name__ == "__main__": + with run_diagnostic() as config: + main(config) \ No newline at end of file diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml new file mode 100644 index 0000000000..33ca4333ce --- /dev/null +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -0,0 +1,111 @@ +# ESMValTool +# +--- +documentation: + title: Performance metrics plots. + description: > + Compare performance of ICON-Seamless vs. CMIP6 models. + authors: + - schlund_manuel + - ruhe_lukas + # - cammarano_diego + references: + - eyring21ipcc + + +preprocessors: + + extract_time: &extract_time + custom_order: true + extract_time: + start_year: 1990 + start_month: 1 + start_day: 1 + end_year: 1995 + end_month: 12 + end_day: 31 + regrid: + target_grid: 2x2 + scheme: nearest + distance_metric: + metric: pearsonr + coords: null + + + + +cmip6_default: &cmip6 {dataset: ACCESS-CM2, grid: gn, + ensemble: r1i1p1f1, project: CMIP6, timerange: '1980/1999'} +diagnostics: + perfmetric: + description: perfmetric in python + variables: + pr: &var_default + preprocessor: extract_time + mip: Amon + exp: historical + # ensemble: r1i1p1 + # alternative_dataset: GHCN + tas: + <<: *var_default + clt: + <<: *var_default + psl: + <<: *var_default + + additional_datasets: + # - {dataset: ERA5, project: native6, type: reanaly, + # version: v1, tier: 3, reference_for_metric: true} + - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} + - {<<: *cmip6, dataset: CMCC-ESM2} + - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} + - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} + - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} + - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} + - {<<: *cmip6, dataset: GISS-E2-1-G-CC} + # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} + # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} + - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} + - {<<: *cmip6, dataset: MCM-UA-1-0} + scripts: + python: + script: perfmetrics/main.py + group_by: project + + # tas: &pcmip6, dataset: MCM-UA-1-0} + # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3} + # - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2} + + # clt: + # <<:torical + # additional_datasets: + # - {<<: *cmip6, dataset: BCC-ESM1} + # - {dataset: ESACCI-CLOUD, project: OBS, type: sat, + # version: AVHRR-fv3.0, tier: 2, timerange: '1982/1999'} + # - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, + # timerange: '1982/1999'} + + # pr: + # - {dataset: GPCP-SG, project: obs4MIPs, level: L3, version: v2.3, tier: 1} + # - {dataset: GHCN, project: OBS, type: ground, version: 1, tier: 2} + + # psl: + # <<: aset: MCM-UA-1-0} + # - {dataset: JRA-55, project: ana4mips, type: reanalysis, tier: 1} + # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3} + + + # rlut: + # 6, dataset: MCM-UA-1-0} + # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, version: Ed2-8, + # tier: 1, start_year: 2001, end_year: 2015} + + # rsut: + # <<: *perfp6, dataset: KIOST-ESM, grid: gr1} + # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, version: Ed2-8, + # tier: 1, start_year: 2001, end_year: 2015} + From b74414fe579445c772ad5c5929a5e42f8753b004 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 14 Mar 2024 19:01:03 +0100 Subject: [PATCH 02/56] added some datasets for fixed calendar --- .../recipes/recipe_perfmetrics_python.yml | 124 +++++++++++++++--- 1 file changed, 108 insertions(+), 16 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index 33ca4333ce..84d93b643b 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -15,62 +15,154 @@ documentation: preprocessors: - extract_time: &extract_time + + default: custom_order: true extract_time: start_year: 1990 start_month: 1 start_day: 1 - end_year: 1995 + end_year: 1992 end_month: 12 end_day: 31 regrid: - target_grid: 2x2 + target_grid: 3x3 scheme: nearest + regrid_time: + calendar: standard + frequency: mon distance_metric: metric: pearsonr - coords: null - + # coords: null + + +cmip6_default: &cmip6 + grid: gn + ensemble: r1i1p1f1 + project: CMIP6 + timerange: '1990/1992' + +datasets: + # ICON + # - {project: ICON, + # timerange: '1991/1992', + # dataset: ICON-Seamless, + # exp: hist001, + # horizontal_grid: /work/pd1295/ICON/grids/public/edzw/icon_grid_0030_R02B05_G.nc, + # supplementary_variables: [ + # {short_name: areacella, skip: true}, + # {short_name: areacello, skip: true}, + # {short_name: sftlf, skip: true}, + # {short_name: sftof, skip: true}, + # ], + # ugrid: false, + # } + # - {project: ICON, + # timerange: '1991/1992', + # dataset: ICON-Seamless, + # exp: hist002, + # horizontal_grid: /work/pd1295/ICON/grids/public/edzw/icon_grid_0030_R02B05_G.nc, + # supplementary_variables: [ + # {short_name: areacella, skip: true}, + # {short_name: areacello, skip: true}, + # {short_name: sftlf, skip: true}, + # {short_name: sftof, skip: true}, + # ], + # ugrid: false, + # } + # CMIP6 + # - {<<: *cmip6, dataset: ACCESS-CM2, reference_for_metric: true} + - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO} + - {<<: *cmip6, dataset: AWI-CM-1-1-MR} + - {<<: *cmip6, dataset: AWI-ESM-1-1-LR} + + - {<<: *cmip6, dataset: CESM2-FV2, institute: NCAR} + - {<<: *cmip6, dataset: CESM2-WACCM, institute: NCAR} + - {<<: *cmip6, dataset: CESM2-WACCM-FV2, institute: NCAR} + - {<<: *cmip6, dataset: CIESM, grid: gr} + - {<<: *cmip6, dataset: CMCC-CM2-HR4} + # # - {<<: *cmip6, dataset: CMCC-CM2-SR5} + # - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} + # # - {<<: *cmip6, dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr} + # # - {<<: *cmip6, dataset: CNRM-ESM2-1, ensemble: r1i1p1f2, grid: gr} + # # - {<<: *cmip6, dataset: EC-Earth3-Veg-LR, grid: gr} + # - {<<: *cmip6, dataset: FGOALS-f3-L, grid: gr} + # - {<<: *cmip6, dataset: FGOALS-g3} + # - {<<: *cmip6, dataset: GFDL-CM4, grid: gr1} + # - {<<: *cmip6, dataset: GISS-E2-1-G} + # - {<<: *cmip6, dataset: GISS-E2-1-H} + # - {<<: *cmip6, dataset: HadGEM3-GC31-LL, ensemble: r1i1p1f3} + # - {<<: *cmip6, dataset: HadGEM3-GC31-MM, ensemble: r1i1p1f3} + + + # - {<<: *cmip6, dataset: IITM-ESM} + # - {<<: *cmip6, dataset: INM-CM4-8, grid: gr1} + # - {<<: *cmip6, dataset: INM-CM5-0, grid: gr1} + # - {<<: *cmip6, dataset: IPSL-CM6A-LR, grid: gr} + # - {<<: *cmip6, dataset: MIROC6, reference_for_metric: true} + # - {<<: *cmip6, dataset: MIROC-ES2L, ensemble: r1i1p1f2} + # - {<<: *cmip6, dataset: MPI-ESM-1-2-HAM} + # - {<<: *cmip6, dataset: MPI-ESM1-2-LR} + # - {<<: *cmip6, dataset: MPI-ESM1-2-HR} + # - {<<: *cmip6, dataset: MRI-ESM2-0} + # - {<<: *cmip6, dataset: NESM3} + # - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} + # - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} + # - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} + # - {<<: *cmip6, dataset: SAM0-UNICON} + # - {<<: *cmip6, dataset: TaiESM1} + # - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} - -cmip6_default: &cmip6 {dataset: ACCESS-CM2, grid: gn, - ensemble: r1i1p1f1, project: CMIP6, timerange: '1980/1999'} diagnostics: perfmetric: description: perfmetric in python variables: pr: &var_default - preprocessor: extract_time + preprocessor: default mip: Amon exp: historical # ensemble: r1i1p1 # alternative_dataset: GHCN tas: <<: *var_default - clt: + # rlut: + # <<: *var_default + ps: + <<: *var_default + # sfcWind: + # <<: *var_default + clt: <<: *var_default + # rlut: + # <<: *var_default psl: <<: *var_default + additional_datasets: # - {dataset: ERA5, project: native6, type: reanaly, # version: v1, tier: 3, reference_for_metric: true} + - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} - {<<: *cmip6, dataset: CMCC-ESM2} - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} - - {<<: *cmip6, dataset: GISS-E2-1-G-CC} + + # grid wrong? missing data + # - {<<: *cmip6, dataset: MCM-UA-1-0} + # - {<<: *cmip6, dataset: GISS-E2-1-G-CC} # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} + - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} - - {<<: *cmip6, dataset: MCM-UA-1-0} + - {<<: *cmip6, dataset: EC-Earth3, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} + scripts: python: script: perfmetrics/main.py From f78d2e51e768185aefdd786d1b36b68bbaba1a93 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Mar 2024 08:20:18 +0100 Subject: [PATCH 03/56] first approach group gaps WIP --- esmvaltool/diag_scripts/perfmetrics/main.py | 123 ++++++++++++++++---- 1 file changed, 100 insertions(+), 23 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index e9b736a550..e97d88f7d8 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -1,5 +1,23 @@ +# This diagnostic provides plot functionalities +# for performance metrics. +# The multi model overview heatmap might be usefull for different +# tasks and therefore this diagnostic tries to be as flexible as possible. +# X and Y axis, grouping parameter and slits for each rectangle can be +# configured in the recipe. All *_by parameters can be set to any metadata +# key. To split by 'reference' this key needs to be set as extra_facet in recipe. +# NOTE: should different references be done by aliases or specific extra_facet in recipe? +# +# x_by: [alias] +# y_by: [variable_group] +# group_by: [project] gaps always applied in x direction +# split_by: [None] +# plot_kwargs: {} +# cbar_kwargs: {} + + import matplotlib as mpl from esmvalcore import preprocessor as pp +import itertools import logging import matplotlib.pyplot as plt import esmvaltool.diag_scripts.shared as e @@ -16,6 +34,7 @@ import matplotlib.pyplot as plt + def heatmap(data, row_labels, col_labels, ax=None, cbar_kw=None, cbarlabel="", **kwargs): """ @@ -46,61 +65,119 @@ def heatmap(data, row_labels, col_labels, ax=None, if cbar_kw is None: cbar_kw = {} - # Plot the heatmap im = ax.imshow(data, **kwargs) - # Create colorbar cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw) cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") - # Show all ticks and label them with the respective list entries. ax.set_xticks(np.arange(data.shape[1]), labels=col_labels) ax.set_yticks(np.arange(data.shape[0]), labels=row_labels) - # Rotate the tick labels and set their alignment. plt.setp(ax.get_xticklabels(), rotation=90, ha="right", - va="top", rotation_mode="anchor") + va="center", rotation_mode="anchor") # Turn spines off and create white grid. # ax.spines[:].set_visible(False) - ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True) ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True) ax.grid(which="minor", color="black", linestyle='-', linewidth=1) ax.tick_params(which="both", bottom=False, left=False) - - return im, cbar + return ax, cbar def remove_reference(metas): - return [meta for meta in metas if not meta.get("reference_for_metric", False)] + """"return metadata that are not reference for metric""" + return [meta for meta in metas + if not meta.get("reference_for_metric", False)] -def plot(cfg, metas): - x = cfg.get("x", "dataset") - y = cfg.get("y", "variable_group") - y_labels = group_metadata(metas, y).keys() - x_labels = group_metadata(metas, x).keys() +def plot(cfg, metas, groups=None): + # print(metas) + y_labels = list(group_metadata(metas, cfg["y_by"]).keys()) + x_labels = list(group_metadata(metas, cfg["x_by"]).keys()) + # print(x_labels) data = np.zeros((len(y_labels), len(x_labels))) - for i, y_label in enumerate(y_labels): - #y_metas = select_metadata(metas, variable_group=y_label) - for j, x_label in enumerate(x_labels): - selection = {x: x_label, y: y_label} + if groups is not None: + # add empty column as spacer + data = np.zeros((len(y_labels), len(x_labels)+len(groups)-1)) + print(data) + # add empty labels at certain positions for gaps + # drop last position (not needed) + print(groups) + gap_positions = list(groups.values())[:-1] + gap_positions.reverse() + for gap in gap_positions: + x_labels.insert(gap, "") + data = np.zeros((len(y_labels), len(x_labels))) + print(x_labels) + for x, x_label in enumerate(x_labels): + if x_label == "": + data[:, x] = 0 + continue + for y, y_label in enumerate(y_labels): + selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} meta = select_metadata(metas, **selection)[0] cube = iris.load_cube(meta["filename"]) - data[i][j] = cube.data - ax = heatmap(data, y_labels, x_labels, **{"cmap": "RdYlBu_r"}, cbar_kw={"extend": "both"}) - plt.savefig("heatmap.png") + data[y, x] = cube.data + # if x == list(groups.values())[gaps]: print("GAP ADDED at ", x) + # gaps += 1 + # data[x+gaps][y] = 0 + + cbar_kw = {"extend": "both"} + cbar_kw.update(cfg.get("cbar_kw", {})) + ax, cbar = heatmap(data, y_labels, x_labels, **cfg["plot_kwargs"], cbar_kw=cbar_kw) + + # hide gap by drawing white rectangle + for x, x_label in enumerate(x_labels): + if x_label == "": + ax.add_patch(mpl.patches.Rectangle((x-0.45, 0-0.55), 0.9, 9, fill=True, color="white", zorder=10)) + # ax.add_patch(mpl.patches.Rectangle((0, 0), 1, 1, fill=True, color="white", zorder=10)) + # set any axis properties + ax.set(**cfg["axes_properties"]) + basename = "performance_metrics.png" + fname = get_plot_filename(basename, cfg) + plt.savefig(fname) + print("Figure saved:") + print(fname) + + +def apply_grouping(cfg, metas): + """returns sorted metadata, and group labels with positions""" + group_by = cfg.get("group_by", "project") + grouped = group_metadata(metas, group_by) + counts = [] + labels = [] + for group, metas in grouped.items(): + labels.append(group) + x_entries = len(group_metadata(metas, cfg["x_by"])) + counts.append(x_entries) + positions = np.cumsum(counts) + groups = dict(zip(labels, positions)) + metas = list(itertools.chain(*grouped.values())) + return metas, groups + + +def set_defaults(cfg): + """set default values for most important config parameters""" + cfg.setdefault("x_by", "dataset") + cfg.setdefault("y_by", "variable_group") + cfg.setdefault("group_by", "project") + cfg.setdefault("split_by", None) + cfg.setdefault("axes_properties", {}) + cfg.setdefault("plot_kwargs", {}) + cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") def main(cfg): + set_defaults(cfg) metas = cfg["input_data"].values() metas = remove_reference(metas) - plot(cfg, metas) + metas, groups = apply_grouping(cfg, metas) + plot(cfg, metas, groups=groups) # grouped = group_metadata(metas, "project") # for group, group_metas in grouped.items(): # plot_group() - + if __name__ == "__main__": with run_diagnostic() as config: From 93c55443df2c1a6dfa3b1ad6757aba9e3ae301f4 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Mar 2024 14:32:34 +0100 Subject: [PATCH 04/56] imagegrid for grouping, labels, colorbar, docstring --- esmvaltool/diag_scripts/perfmetrics/main.py | 207 +++++++++++--------- 1 file changed, 112 insertions(+), 95 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index e97d88f7d8..97039440d0 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -1,19 +1,47 @@ -# This diagnostic provides plot functionalities -# for performance metrics. -# The multi model overview heatmap might be usefull for different -# tasks and therefore this diagnostic tries to be as flexible as possible. -# X and Y axis, grouping parameter and slits for each rectangle can be -# configured in the recipe. All *_by parameters can be set to any metadata -# key. To split by 'reference' this key needs to be set as extra_facet in recipe. -# NOTE: should different references be done by aliases or specific extra_facet in recipe? -# -# x_by: [alias] -# y_by: [variable_group] -# group_by: [project] gaps always applied in x direction -# split_by: [None] -# plot_kwargs: {} -# cbar_kwargs: {} - +""" +This diagnostic provides plot functionalities +for performance metrics. +The multi model overview heatmap might be usefull for different +tasks and therefore this diagnostic tries to be as flexible as possible. +X and Y axis, grouping parameter and slits for each rectangle can be +configured in the recipe. All *_by parameters can be set to any metadata +key. To split by 'reference' this key needs to be set as extra_facet in recipe. + +Configuration parameters through recipe: +---------------------------------------- +x_by: str, optional + Metadata key for x coordinate. + By default 'alias'. +y_by: str, optional + Metadata key for y coordinate + By default 'variable_group'. +group_by: str, optional + Metadata key for grouping. + Grouping is always applied in x direction. Can be set to None to skip grouping into subplots. + By default 'project'. +split_by: str, optional + Not implemented yet. + By default None. +plot_kwargs: dict, optional + Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. + Colormaps will be converted to 11 discrete steps automatically. Default colormap is RdYlBu_r but can be changed with cmap. Other common keywords: vmin, vmax + By default {}. +cbar_kwargs: dict, optional + Dictionary that gets passed to `matplotlib.pyplot.colorbar()`. + E.g. label, ticks... + By default {}. +plot_properties: dict, optional + Dictionary that gets passed to `matplotlib.axes.Axes.set()`. + Subplots can be widely customized. For a full list of + properties see: https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html#matplotlib.axes.Axes.set + E.g. xlabel, ylabel, yticklabels, xmargin... + By default {}. +figsize: list(float), optional + [width, height] of the figure in inches. The final figure will + be saved with bbox_inches="tight", which can change the + resulting aspect ratio. + By default: [5, 3] +""" import matplotlib as mpl from esmvalcore import preprocessor as pp @@ -21,6 +49,7 @@ import logging import matplotlib.pyplot as plt import esmvaltool.diag_scripts.shared as e +from mpl_toolkits.axes_grid1 import ImageGrid from esmvaltool.diag_scripts.shared import ( # ProvenanceLogger, get_plot_filename, @@ -35,54 +64,37 @@ -def heatmap(data, row_labels, col_labels, ax=None, - cbar_kw=None, cbarlabel="", **kwargs): - """ - Create a heatmap from a numpy array and two lists of labels. - - Parameters - ---------- - data - A 2D numpy array of shape (M, N). - row_labels - A list or array of length M with the labels for the rows. - col_labels - A list or array of length N with the labels for the columns. - ax - A `matplotlib.axes.Axes` instance to which the heatmap is plotted. If - not provided, use current axes or create a new one. Optional. - cbar_kw - A dictionary with arguments to `matplotlib.Figure.colorbar`. Optional. - cbarlabel - The label for the colorbar. Optional. - **kwargs - All other arguments are forwarded to `imshow`. - """ - - if ax is None: - ax = plt.gca() - - if cbar_kw is None: - cbar_kw = {} - - im = ax.imshow(data, **kwargs) - # Create colorbar - cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw) - cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") +def unify_limits(cfg, grid): + vmin, vmax = np.inf, -np.inf + images = [ax.get_images()[0] for ax in grid] + for im in images: + vmin = min(vmin, im.get_clim()[0]) + vmax = max(vmax, im.get_clim()[1]) + for im in images: + print("setting clim to", vmin, vmax) + im.set_clim(vmin, vmax) + + +def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): + """Create an image for given data.""" + im = ax.imshow(data, **plot_kwargs) # Show all ticks and label them with the respective list entries. ax.set_xticks(np.arange(data.shape[1]), labels=col_labels) ax.set_yticks(np.arange(data.shape[0]), labels=row_labels) # Rotate the tick labels and set their alignment. - plt.setp(ax.get_xticklabels(), rotation=90, ha="right", - va="center", rotation_mode="anchor") - + plt.setp( + ax.get_xticklabels(), + rotation=90, + ha="right", + va="center", + rotation_mode="anchor") # Turn spines off and create white grid. # ax.spines[:].set_visible(False) ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True) ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True) ax.grid(which="minor", color="black", linestyle='-', linewidth=1) ax.tick_params(which="both", bottom=False, left=False) - return ax, cbar + return im def remove_reference(metas): @@ -91,52 +103,58 @@ def remove_reference(metas): if not meta.get("reference_for_metric", False)] -def plot(cfg, metas, groups=None): - # print(metas) +def plot_group(cfg, ax, metas, title=None): + """create matrix for one subplot at axes ax""" y_labels = list(group_metadata(metas, cfg["y_by"]).keys()) x_labels = list(group_metadata(metas, cfg["x_by"]).keys()) - # print(x_labels) - data = np.zeros((len(y_labels), len(x_labels))) - if groups is not None: - # add empty column as spacer - data = np.zeros((len(y_labels), len(x_labels)+len(groups)-1)) - print(data) - # add empty labels at certain positions for gaps - # drop last position (not needed) - print(groups) - gap_positions = list(groups.values())[:-1] - gap_positions.reverse() - for gap in gap_positions: - x_labels.insert(gap, "") + # TODO: splitted rectangles overlay multiple triangles. + if cfg["split_by"] is not None: + s_labels = list(group_metadata(metas, cfg["split_by"]).keys()) data = np.zeros((len(y_labels), len(x_labels))) - print(x_labels) + # load data from nc files into matrix for x, x_label in enumerate(x_labels): - if x_label == "": - data[:, x] = 0 - continue for y, y_label in enumerate(y_labels): selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} meta = select_metadata(metas, **selection)[0] + # TODO: select_single_metadata() to shared? cube = iris.load_cube(meta["filename"]) data[y, x] = cube.data - # if x == list(groups.values())[gaps]: print("GAP ADDED at ", x) - # gaps += 1 - # data[x+gaps][y] = 0 - - cbar_kw = {"extend": "both"} - cbar_kw.update(cfg.get("cbar_kw", {})) - ax, cbar = heatmap(data, y_labels, x_labels, **cfg["plot_kwargs"], cbar_kw=cbar_kw) - - # hide gap by drawing white rectangle - for x, x_label in enumerate(x_labels): - if x_label == "": - ax.add_patch(mpl.patches.Rectangle((x-0.45, 0-0.55), 0.9, 9, fill=True, color="white", zorder=10)) - # ax.add_patch(mpl.patches.Rectangle((0, 0), 1, 1, fill=True, color="white", zorder=10)) - # set any axis properties + # plot matrix + im = plot_matrix(data, y_labels, x_labels, ax, cfg["plot_kwargs"]) + if title is not None: + ax.set_title(title) ax.set(**cfg["axes_properties"]) + return im, data + + +def plot(cfg, grouped_metas): + """creates figure with subplots for each group""" + fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) + grid = ImageGrid( + fig, 111, # similar to subplot(111) + cbar_mode="single", + cbar_location="right", + cbar_pad=0.1, + cbar_size=0.1, + nrows_ncols=(1, len(grouped_metas)), + axes_pad=0.1) + # remap colorbar to 11 discrete steps + cmap = mpl.cm.get_cmap(cfg.get("cmap", "RdYlBu_r"), 11) + cfg["plot_kwargs"]["cmap"] = cmap + for i, (group, metas) in enumerate(grouped_metas.items()): + title = group if len(grouped_metas) > 1 else None + im, data = plot_group(cfg, grid[i], metas, title=title) + # datas.append(data) + # data = np.array(datas) + # ax.imshow(im, origin="lower", vmin=vmin, + # vmax=vmax, interpolation="nearest") + # set same clims (vmin, vmax) for all subplots + grid.cbar_axes[0].colorbar(im, **cfg["cbar_kw"]) + # cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") basename = "performance_metrics.png" fname = get_plot_filename(basename, cfg) - plt.savefig(fname) + # plt.tight_layout() + plt.savefig(fname, bbox_inches="tight") print("Figure saved:") print(fname) @@ -159,10 +177,11 @@ def apply_grouping(cfg, metas): def set_defaults(cfg): """set default values for most important config parameters""" - cfg.setdefault("x_by", "dataset") + cfg.setdefault("x_by", "alias") cfg.setdefault("y_by", "variable_group") cfg.setdefault("group_by", "project") cfg.setdefault("split_by", None) + cfg.setdefault("cbar_kw", {}) cfg.setdefault("axes_properties", {}) cfg.setdefault("plot_kwargs", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") @@ -172,13 +191,11 @@ def main(cfg): set_defaults(cfg) metas = cfg["input_data"].values() metas = remove_reference(metas) - metas, groups = apply_grouping(cfg, metas) - plot(cfg, metas, groups=groups) - # grouped = group_metadata(metas, "project") - # for group, group_metas in grouped.items(): - # plot_group() + grouped_metas = group_metadata(metas, cfg["group_by"]) + cfg["figsize"] = (7.5, 5.5) + plot(cfg, grouped_metas) if __name__ == "__main__": with run_diagnostic() as config: - main(config) \ No newline at end of file + main(config) From db30131fc1a913d2f92a0749f70cbc06797dba06 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Mar 2024 16:11:37 +0100 Subject: [PATCH 05/56] WIP triangle overlays --- esmvaltool/diag_scripts/perfmetrics/main.py | 89 ++++++++++++--------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 97039440d0..3f8777fbaf 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -1,30 +1,40 @@ -""" -This diagnostic provides plot functionalities -for performance metrics. +"""Overview plot for performance metrics. + +Description +----------- +This diagnostic provides plot functionalities for performance metrics. The multi model overview heatmap might be usefull for different tasks and therefore this diagnostic tries to be as flexible as possible. X and Y axis, grouping parameter and slits for each rectangle can be configured in the recipe. All *_by parameters can be set to any metadata key. To split by 'reference' this key needs to be set as extra_facet in recipe. +Author +------ +Lukas Ruhe (Universität Bremen, Germany) +Diego Cammarano + Configuration parameters through recipe: ---------------------------------------- x_by: str, optional - Metadata key for x coordinate. + Metadata key for x coordinate. By default 'alias'. y_by: str, optional - Metadata key for y coordinate + Metadata key for y coordinate. By default 'variable_group'. group_by: str, optional Metadata key for grouping. - Grouping is always applied in x direction. Can be set to None to skip grouping into subplots. + Grouping is always applied in x direction. Can be set to None to skip + grouping into subplots. By default 'project'. split_by: str, optional Not implemented yet. By default None. -plot_kwargs: dict, optional +plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. - Colormaps will be converted to 11 discrete steps automatically. Default colormap is RdYlBu_r but can be changed with cmap. Other common keywords: vmin, vmax + Colormaps will be converted to 11 discrete steps automatically. Default + colormap is RdYlBu_r but can be changed with cmap. + Other common keywords: vmin, vmax By default {}. cbar_kwargs: dict, optional Dictionary that gets passed to `matplotlib.pyplot.colorbar()`. @@ -33,45 +43,34 @@ plot_properties: dict, optional Dictionary that gets passed to `matplotlib.axes.Axes.set()`. Subplots can be widely customized. For a full list of - properties see: https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html#matplotlib.axes.Axes.set + properties see: + https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html E.g. xlabel, ylabel, yticklabels, xmargin... By default {}. figsize: list(float), optional - [width, height] of the figure in inches. The final figure will - be saved with bbox_inches="tight", which can change the - resulting aspect ratio. - By default: [5, 3] + [width, height] of the figure in inches. The final figure will be saved with + bbox_inches="tight", which can change the resulting aspect ratio. + By default [5, 3]. """ -import matplotlib as mpl -from esmvalcore import preprocessor as pp import itertools -import logging -import matplotlib.pyplot as plt -import esmvaltool.diag_scripts.shared as e -from mpl_toolkits.axes_grid1 import ImageGrid -from esmvaltool.diag_scripts.shared import ( - # ProvenanceLogger, - get_plot_filename, - group_metadata, - run_diagnostic, - select_metadata, -) - import iris -import numpy as np +import matplotlib as mpl import matplotlib.pyplot as plt - +import numpy as np +from esmvaltool.diag_scripts.shared import ( + get_plot_filename, group_metadata, run_diagnostic, select_metadata) +from mpl_toolkits.axes_grid1 import ImageGrid def unify_limits(cfg, grid): + """set same limits for all subplots""" vmin, vmax = np.inf, -np.inf images = [ax.get_images()[0] for ax in grid] for im in images: vmin = min(vmin, im.get_clim()[0]) vmax = max(vmax, im.get_clim()[1]) for im in images: - print("setting clim to", vmin, vmax) im.set_clim(vmin, vmax) @@ -103,26 +102,44 @@ def remove_reference(metas): if not meta.get("reference_for_metric", False)] +def plot_additional_splits(cfg, ax, splitted): + """add additional references as triangular overlays""" + if len(splitted) < 2: + return + print("Overlays for ", len(splitted)-1) + for split, metas in splitted.items()[1:]: + print(split) + pass + + def plot_group(cfg, ax, metas, title=None): """create matrix for one subplot at axes ax""" + splitted = group_metadata(metas, cfg["split_by"]) y_labels = list(group_metadata(metas, cfg["y_by"]).keys()) x_labels = list(group_metadata(metas, cfg["x_by"]).keys()) - # TODO: splitted rectangles overlay multiple triangles. - if cfg["split_by"] is not None: - s_labels = list(group_metadata(metas, cfg["split_by"]).keys()) data = np.zeros((len(y_labels), len(x_labels))) # load data from nc files into matrix for x, x_label in enumerate(x_labels): for y, y_label in enumerate(y_labels): selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} - meta = select_metadata(metas, **selection)[0] - # TODO: select_single_metadata() to shared? + if cfg["split_by"] is not None: + selection.update({cfg["split_by"]: splitted.keys()[0]}) + try: + # TODO: select_single_metadata() to shared? + meta = select_metadata(metas, **selection)[0] + except IndexError: + print(f"No data found for {selection}") + data[y, x] = np.nan + continue cube = iris.load_cube(meta["filename"]) data[y, x] = cube.data # plot matrix im = plot_matrix(data, y_labels, x_labels, ax, cfg["plot_kwargs"]) if title is not None: ax.set_title(title) + # overlay splits (additional references) + if cfg["split_by"] is not None: + plot_additional_splits(cfg, ax, splitted) ax.set(**cfg["axes_properties"]) return im, data @@ -135,7 +152,7 @@ def plot(cfg, grouped_metas): cbar_mode="single", cbar_location="right", cbar_pad=0.1, - cbar_size=0.1, + cbar_size=0.2, nrows_ncols=(1, len(grouped_metas)), axes_pad=0.1) # remap colorbar to 11 discrete steps From 215ee95072f2aca11b8769eb58918b072f309144 Mon Sep 17 00:00:00 2001 From: Diego Cammarano Date: Thu, 21 Mar 2024 16:27:31 +0100 Subject: [PATCH 06/56] added CMIP6 experiments --- .../recipes/recipe_perfmetrics_python.yml | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index 84d93b643b..53bad57d42 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -70,48 +70,71 @@ datasets: # ], # ugrid: false, # } + # - {project: ICON, + # dataset: ICON, + # exp: ag_atm_amip_r2b5_cov4_sfs4, + # horizontal_grid: /work/bd1179/b309170/icon-ml_models/icon-a-ml/experiments/ag_atm_amip_r2b5_cov4_sfs4/icon_grid_0006_R02B05_G.nc, + # supplementary_variables: [ + # {short_name: areacella, skip: true}, + # {short_name: areacello, skip: true}, + # {short_name: sftlf, skip: true}, + # {short_name: sftof, skip: true}, + # ], + # frequency: day, + # timerange: "19790101/19880101", + # } + # - {project: ICON, + # dataset: ICON, + # exp: ag_atm_amip_r2b5, + # horizontal_grid: /work/bd1179/b309170/icon-ml_models/icon-a-ml/experiments/ag_atm_amip_r2b5/icon_grid_0006_R02B05_G.nc, + # supplementary_variables: [ + # {short_name: areacella, skip: true}, + # {short_name: areacello, skip: true}, + # {short_name: sftlf, skip: true}, + # {short_name: sftof, skip: true}, + # ], + # frequency: day, + # timerange: "19790101/19880101", + # } # CMIP6 - # - {<<: *cmip6, dataset: ACCESS-CM2, reference_for_metric: true} + - {<<: *cmip6, dataset: ACCESS-CM2, reference_for_metric: false} - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO} - {<<: *cmip6, dataset: AWI-CM-1-1-MR} - {<<: *cmip6, dataset: AWI-ESM-1-1-LR} - - {<<: *cmip6, dataset: CESM2-FV2, institute: NCAR} - {<<: *cmip6, dataset: CESM2-WACCM, institute: NCAR} - {<<: *cmip6, dataset: CESM2-WACCM-FV2, institute: NCAR} - {<<: *cmip6, dataset: CIESM, grid: gr} - {<<: *cmip6, dataset: CMCC-CM2-HR4} - # # - {<<: *cmip6, dataset: CMCC-CM2-SR5} - # - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} - # # - {<<: *cmip6, dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr} - # # - {<<: *cmip6, dataset: CNRM-ESM2-1, ensemble: r1i1p1f2, grid: gr} - # # - {<<: *cmip6, dataset: EC-Earth3-Veg-LR, grid: gr} - # - {<<: *cmip6, dataset: FGOALS-f3-L, grid: gr} - # - {<<: *cmip6, dataset: FGOALS-g3} - # - {<<: *cmip6, dataset: GFDL-CM4, grid: gr1} - # - {<<: *cmip6, dataset: GISS-E2-1-G} - # - {<<: *cmip6, dataset: GISS-E2-1-H} - # - {<<: *cmip6, dataset: HadGEM3-GC31-LL, ensemble: r1i1p1f3} - # - {<<: *cmip6, dataset: HadGEM3-GC31-MM, ensemble: r1i1p1f3} - - - # - {<<: *cmip6, dataset: IITM-ESM} - # - {<<: *cmip6, dataset: INM-CM4-8, grid: gr1} - # - {<<: *cmip6, dataset: INM-CM5-0, grid: gr1} - # - {<<: *cmip6, dataset: IPSL-CM6A-LR, grid: gr} - # - {<<: *cmip6, dataset: MIROC6, reference_for_metric: true} - # - {<<: *cmip6, dataset: MIROC-ES2L, ensemble: r1i1p1f2} - # - {<<: *cmip6, dataset: MPI-ESM-1-2-HAM} - # - {<<: *cmip6, dataset: MPI-ESM1-2-LR} - # - {<<: *cmip6, dataset: MPI-ESM1-2-HR} - # - {<<: *cmip6, dataset: MRI-ESM2-0} - # - {<<: *cmip6, dataset: NESM3} - # - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} - # - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} - # - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} - # - {<<: *cmip6, dataset: SAM0-UNICON} - # - {<<: *cmip6, dataset: TaiESM1} - # - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} + - {<<: *cmip6, dataset: CMCC-CM2-SR5} + - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: CNRM-ESM2-1, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-Veg-LR, grid: gr} + - {<<: *cmip6, dataset: FGOALS-f3-L, grid: gr} + - {<<: *cmip6, dataset: FGOALS-g3} + - {<<: *cmip6, dataset: GFDL-CM4, grid: gr1} + - {<<: *cmip6, dataset: GISS-E2-1-G} + - {<<: *cmip6, dataset: GISS-E2-1-H} + - {<<: *cmip6, dataset: HadGEM3-GC31-LL, ensemble: r1i1p1f3} + - {<<: *cmip6, dataset: HadGEM3-GC31-MM, ensemble: r1i1p1f3} + - {<<: *cmip6, dataset: IITM-ESM} + - {<<: *cmip6, dataset: INM-CM4-8, grid: gr1} + - {<<: *cmip6, dataset: INM-CM5-0, grid: gr1} + - {<<: *cmip6, dataset: IPSL-CM6A-LR, grid: gr} + - {<<: *cmip6, dataset: MIROC6, reference_for_metric: false} + - {<<: *cmip6, dataset: MIROC-ES2L, ensemble: r1i1p1f2} + - {<<: *cmip6, dataset: MPI-ESM-1-2-HAM} + - {<<: *cmip6, dataset: MPI-ESM1-2-LR} + - {<<: *cmip6, dataset: MPI-ESM1-2-HR} + - {<<: *cmip6, dataset: MRI-ESM2-0} + - {<<: *cmip6, dataset: NESM3} + - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} + - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} + - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} + - {<<: *cmip6, dataset: SAM0-UNICON} + - {<<: *cmip6, dataset: TaiESM1} + - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} diagnostics: From b4f8238a4addb7a08109a0cd6148f2007b7a3860 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Mar 2024 16:38:46 +0100 Subject: [PATCH 07/56] WIP split by reference --- esmvaltool/diag_scripts/perfmetrics/main.py | 169 +++++++++++++----- .../recipes/recipe_perfmetrics_python.yml | 29 ++- 2 files changed, 145 insertions(+), 53 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 3f8777fbaf..76ac1cb6e5 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -59,8 +59,13 @@ import matplotlib.pyplot as plt import numpy as np from esmvaltool.diag_scripts.shared import ( - get_plot_filename, group_metadata, run_diagnostic, select_metadata) + get_plot_filename, + group_metadata, + run_diagnostic, + select_metadata, +) from mpl_toolkits.axes_grid1 import ImageGrid +import matplotlib.patches as patches def unify_limits(cfg, grid): @@ -86,60 +91,114 @@ def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): rotation=90, ha="right", va="center", - rotation_mode="anchor") + rotation_mode="anchor", + ) # Turn spines off and create white grid. # ax.spines[:].set_visible(False) - ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True) - ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True) - ax.grid(which="minor", color="black", linestyle='-', linewidth=1) + ax.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) + ax.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) + ax.grid(which="minor", color="black", linestyle="-", linewidth=1) ax.tick_params(which="both", bottom=False, left=False) return im def remove_reference(metas): - """"return metadata that are not reference for metric""" - return [meta for meta in metas - if not meta.get("reference_for_metric", False)] + """ "return metadata that are not reference for metric""" + return [ + meta for meta in metas if not meta.get("reference_for_metric", False) + ] -def plot_additional_splits(cfg, ax, splitted): - """add additional references as triangular overlays""" - if len(splitted) < 2: - return - print("Overlays for ", len(splitted)-1) - for split, metas in splitted.items()[1:]: - print(split) - pass - - -def plot_group(cfg, ax, metas, title=None): - """create matrix for one subplot at axes ax""" +def load_data(cfg, metas): + """load nc files into dictionary of numpy arrays + dict keys are the values for each split (i.e. reference). + """ splitted = group_metadata(metas, cfg["split_by"]) y_labels = list(group_metadata(metas, cfg["y_by"]).keys()) x_labels = list(group_metadata(metas, cfg["x_by"]).keys()) - data = np.zeros((len(y_labels), len(x_labels))) - # load data from nc files into matrix - for x, x_label in enumerate(x_labels): + data = {key: np.zeros((len(y_labels), len(x_labels))) for key in splitted} + for x, x_label in enumerate(x_labels): for y, y_label in enumerate(y_labels): - selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} - if cfg["split_by"] is not None: - selection.update({cfg["split_by"]: splitted.keys()[0]}) - try: - # TODO: select_single_metadata() to shared? - meta = select_metadata(metas, **selection)[0] - except IndexError: - print(f"No data found for {selection}") - data[y, x] = np.nan - continue - cube = iris.load_cube(meta["filename"]) - data[y, x] = cube.data - # plot matrix - im = plot_matrix(data, y_labels, x_labels, ax, cfg["plot_kwargs"]) + for split, split_metas in splitted.items(): + selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} + try: + meta = select_metadata(split_metas, **selection)[0] + except IndexError: + print(f"No data found for {selection}") + data[split][y, x] = np.nan + continue + cube = iris.load_cube(meta["filename"]) + data[split][y, x] = cube.data + return data, x_labels, y_labels + + +def overlay_reference(cfg, ax, data, triangle): + """create triangular overlays for given data and axes.""" + # use same colors as in main plot + cmap = ax.get_images()[0].get_cmap() + norm = ax.get_images()[0].norm + # print("plotting overlay for", split) + # print(data) + # print(ax.get_images()[0]) + for i, j in itertools.product(*map(range, data.shape)): + if np.isnan(data[i, j]): + continue + print(data[i, j]) + color = cmap(norm(data[i, j])) + edges = [(e[0] + j, e[1] + i) for e in triangle] + patch = patches.Polygon( + edges, + closed=True, + facecolor=color, + edgecolor="black", + linewidth=0.5, + fill=True, + ) + ax.add_patch(patch) + + +def plot_additional_splits(cfg, grid, datas): + """add references as triangular overlays""" + # if len(datas) < 2: + # return + # if len(datas) > 4: + # print("Too many splits for overlay, only 4 will be plotted.") + half = [ + [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)], # lower right + ] + quarters = [ + [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left + [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right + [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom + [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top (?) + ] + if len(datas) == 2: + quarters = half + + for ggg, data in enumerate(datas): + data.pop(cfg["reference"]) # TODO: fix me + for sss, (split, array) in enumerate(data.items()): + # if split == cfg["reference"]: + # continue + print("plotting overlay for", split, array.shape) + overlay_reference(cfg, grid[ggg], array, quarters[sss]) + + +def plot_group(cfg, ax, metas, title=None): + """create matrix for one subplot in ax using plt.imshow() + returns image object and dict of numpy array for each reference (split). + """ + data, x_labels, y_labels = load_data(cfg, metas) + # plot matrix with simple rectangles + im = plot_matrix( + data[cfg["reference"]], y_labels, x_labels, ax, cfg["plot_kwargs"] + ) if title is not None: ax.set_title(title) # overlay splits (additional references) - if cfg["split_by"] is not None: - plot_additional_splits(cfg, ax, splitted) + # cmap might change after plot_group. set vmin and vmax before plotting? + # if len(data) > 1: + # plot_additional_splits(cfg, ax, data) ax.set(**cfg["axes_properties"]) return im, data @@ -148,26 +207,30 @@ def plot(cfg, grouped_metas): """creates figure with subplots for each group""" fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) grid = ImageGrid( - fig, 111, # similar to subplot(111) + fig, + 111, # similar to subplot(111) cbar_mode="single", cbar_location="right", cbar_pad=0.1, cbar_size=0.2, nrows_ncols=(1, len(grouped_metas)), - axes_pad=0.1) - # remap colorbar to 11 discrete steps - cmap = mpl.cm.get_cmap(cfg.get("cmap", "RdYlBu_r"), 11) + axes_pad=0.1, + ) + # remap colorbar to 10 discrete steps + cmap = mpl.cm.get_cmap(cfg.get("cmap", "RdYlBu_r"), 10) cfg["plot_kwargs"]["cmap"] = cmap + datas = [] for i, (group, metas) in enumerate(grouped_metas.items()): title = group if len(grouped_metas) > 1 else None im, data = plot_group(cfg, grid[i], metas, title=title) - # datas.append(data) - # data = np.array(datas) - # ax.imshow(im, origin="lower", vmin=vmin, - # vmax=vmax, interpolation="nearest") - # set same clims (vmin, vmax) for all subplots + datas.append(data) # collect data for overlay + # use same colorrange and colorbar for all subplots: + unify_limits(cfg, grid) grid.cbar_axes[0].colorbar(im, **cfg["cbar_kw"]) # cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") + + # plot overlay with same colors + plot_additional_splits(cfg, grid, datas) basename = "performance_metrics.png" fname = get_plot_filename(basename, cfg) # plt.tight_layout() @@ -202,14 +265,24 @@ def set_defaults(cfg): cfg.setdefault("axes_properties", {}) cfg.setdefault("plot_kwargs", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") + cfg["plot_kwargs"].setdefault("vmin", 0) + cfg["plot_kwargs"].setdefault("vmax", 1) def main(cfg): + """run the diagnostic""" set_defaults(cfg) metas = cfg["input_data"].values() metas = remove_reference(metas) grouped_metas = group_metadata(metas, cfg["group_by"]) cfg["figsize"] = (7.5, 5.5) + cfg["y_by"] = "short_name" + cfg[ + "split_by" + ] = "ref" # TODO: hardcode split as extra facet? and use None as default? + cfg[ + "reference" + ] = "ref1" # TODO: this becomes obsolet if we use extra facet plot(cfg, grouped_metas) diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index 84d93b643b..1a6b9d0a07 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -4,9 +4,8 @@ documentation: title: Performance metrics plots. description: > - Compare performance of ICON-Seamless vs. CMIP6 models. + Compare performance of model simulations to a reference dataset. authors: - - schlund_manuel - ruhe_lukas # - cammarano_diego references: @@ -27,7 +26,9 @@ preprocessors: end_day: 31 regrid: target_grid: 3x3 - scheme: nearest + # scheme: nearest + scheme: + reference: esmf_regrid.schemes:ESMFAreaWeighted regrid_time: calendar: standard frequency: mon @@ -72,7 +73,6 @@ datasets: # } # CMIP6 # - {<<: *cmip6, dataset: ACCESS-CM2, reference_for_metric: true} - - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO} - {<<: *cmip6, dataset: AWI-CM-1-1-MR} - {<<: *cmip6, dataset: AWI-ESM-1-1-LR} @@ -122,21 +122,39 @@ diagnostics: preprocessor: default mip: Amon exp: historical + ref: ref1 # ensemble: r1i1p1 # alternative_dataset: GHCN + additional_datasets: &ref1 + - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} + + pr_ref2: &var_default_ref2 + <<: *var_default + short_name: pr + ref: ref2 + additional_datasets: &ref2 + - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO, reference_for_metric: true} tas: + ref: ref1 <<: *var_default + + tas_ref2: + short_name: tas + <<: *var_default_ref2 # rlut: # <<: *var_default ps: + ref: ref1 <<: *var_default # sfcWind: # <<: *var_default clt: + ref: ref1 <<: *var_default # rlut: # <<: *var_default psl: + ref: ref1 <<: *var_default @@ -144,7 +162,8 @@ diagnostics: # - {dataset: ERA5, project: native6, type: reanaly, # version: v1, tier: 3, reference_for_metric: true} - - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} + - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, + ensemble: r1i1p1} - {<<: *cmip6, dataset: CMCC-ESM2} - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} From 5c30d4a078fa85af6a21315fbaeebff1be86038b Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Mar 2024 17:35:20 +0100 Subject: [PATCH 08/56] added icon example --- .../recipes/recipe_perfmetrics_python.yml | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index 1a6b9d0a07..86aceaff59 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -13,8 +13,6 @@ documentation: preprocessors: - - default: custom_order: true extract_time: @@ -127,7 +125,23 @@ diagnostics: # alternative_dataset: GHCN additional_datasets: &ref1 - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} - + - {project: ICON, dataset: ICON, exp: icon-2.6.1_atm_amip_R2B4_r1v1i1p1l1f1} + # - {project: ICON, dataset: ICON, exp: ag_atm_amip_r2b4_cfm, supplementary_variables: [{short_name: areacella, skip: true}]} + + # - { + # project: ICON, + # timerange: '1990/1991', + # dataset: ICON-Seamless, + # exp: hist001, + # horizontal_grid: /work/pd1295/ICON/grids/public/edzw/icon_grid_0030_R02B05_G.nc, + # supplementary_variables: [ + # {short_name: areacella, skip: true}, + # {short_name: areacello, skip: true}, + # {short_name: sftlf, skip: true}, + # {short_name: sftof, skip: true}, + # ], + # ugrid: false, + # } pr_ref2: &var_default_ref2 <<: *var_default short_name: pr @@ -153,34 +167,34 @@ diagnostics: <<: *var_default # rlut: # <<: *var_default - psl: - ref: ref1 - <<: *var_default + # psl: + # ref: ref1 + # <<: *var_default - additional_datasets: + # additional_datasets: # - {dataset: ERA5, project: native6, type: reanaly, # version: v1, tier: 3, reference_for_metric: true} - - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, - ensemble: r1i1p1} - - {<<: *cmip6, dataset: CMCC-ESM2} - - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} - - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} - - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} - - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} + # - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, + # ensemble: r1i1p1} + # - {<<: *cmip6, dataset: CMCC-ESM2} + # - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} + # - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} + # - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} + # - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} - # grid wrong? missing data - # - {<<: *cmip6, dataset: MCM-UA-1-0} - # - {<<: *cmip6, dataset: GISS-E2-1-G-CC} - # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} - # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} - - - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} - - {<<: *cmip6, dataset: EC-Earth3, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} + # # grid wrong? missing data + # # - {<<: *cmip6, dataset: MCM-UA-1-0} + # # - {<<: *cmip6, dataset: GISS-E2-1-G-CC} + # # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} + # # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} + + # - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} + # - {<<: *cmip6, dataset: EC-Earth3, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} + # - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} scripts: python: From 42fdd819a4d1da154b83057a323224664744f08c Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 27 Mar 2024 13:08:26 +0100 Subject: [PATCH 09/56] split square feature, xarray instead of nested dicts --- esmvaltool/diag_scripts/perfmetrics/main.py | 269 +++++++++++++------- 1 file changed, 171 insertions(+), 98 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 76ac1cb6e5..0d12e482c1 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -28,8 +28,20 @@ grouping into subplots. By default 'project'. split_by: str, optional - Not implemented yet. - By default None. + The rectangles can be splitted into 2-4 triangles. This is used to show + metrics for different references. For this case there is no need to change + this parameter. Multiple variables can be set in the recipe with `split` + assigned as extra_facet to label the different references. Data without + a split assigned will be plotted as main rectangles, this can be changed + by setting default_split parameter. + By default 'split'. +default_split: str, optional + Data labeled with this string, will be used as main rectangles. All other + splits will be plotted as overlays. This can be used to choose the base + reference, while all references are labeled for the legend. +plot_legend: bool, optional + If True, a legend will be plotted showing which additional reference, + belongs to the triangle positions. plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. Colormaps will be converted to 11 discrete steps automatically. Default @@ -53,10 +65,11 @@ By default [5, 3]. """ +import logging import itertools -import iris import matplotlib as mpl import matplotlib.pyplot as plt +import matplotlib.patches as patches import numpy as np from esmvaltool.diag_scripts.shared import ( get_plot_filename, @@ -65,7 +78,9 @@ select_metadata, ) from mpl_toolkits.axes_grid1 import ImageGrid -import matplotlib.patches as patches +import xarray as xr + +log = logging.getLogger(__name__) def unify_limits(cfg, grid): @@ -103,47 +118,87 @@ def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): def remove_reference(metas): - """ "return metadata that are not reference for metric""" - return [ - meta for meta in metas if not meta.get("reference_for_metric", False) - ] + """remove reference for metric from list of metadata + setting split=none allows to omit it in recipe, but handle it as special + case of splitted data. + list() creates a copy with same references to allow removing in place + """ + for meta in list(metas): + if meta.get("reference_for_metric", False): + metas.remove(meta) + + +def add_split_none(metas): + """list of metadata with split=None if no split is given""" + for meta in metas: + if "split" not in meta: + meta["split"] = None + + +def open_file(metadata, **selection): + """try to find a single file for selection and return data. + If multiple files are found, raise an error. + If no file is found, return np.nan. + """ + metas = select_metadata(metadata, **selection) + if len(metas) > 1: + raise ValueError(f"Multiple files found for {selection}") + if len(metas) < 1: + log.debug("No Metadata found for %s", selection) + return np.nan + log.warning("Metadata found for %s", selection) + ds = xr.open_dataset(metas[0]["filename"]) + varname = list(ds.data_vars.keys())[0] + return ds[varname].values.item() + # iris.load_cube(metas[0]["filename"]).data def load_data(cfg, metas): - """load nc files into dictionary of numpy arrays - dict keys are the values for each split (i.e. reference). + """load all nc files from metadata into xarray dataset. + The dataset contains all relevant information for the plot. + Coord names are metadata keys, ordered as x, y, group, split. + The default reference is None, or if all references are named + the first from the list. """ - splitted = group_metadata(metas, cfg["split_by"]) - y_labels = list(group_metadata(metas, cfg["y_by"]).keys()) - x_labels = list(group_metadata(metas, cfg["x_by"]).keys()) - data = {key: np.zeros((len(y_labels), len(x_labels))) for key in splitted} - for x, x_label in enumerate(x_labels): - for y, y_label in enumerate(y_labels): - for split, split_metas in splitted.items(): - selection = {cfg["x_by"]: x_label, cfg["y_by"]: y_label} - try: - meta = select_metadata(split_metas, **selection)[0] - except IndexError: - print(f"No data found for {selection}") - data[split][y, x] = np.nan - continue - cube = iris.load_cube(meta["filename"]) - data[split][y, x] = cube.data - return data, x_labels, y_labels - - -def overlay_reference(cfg, ax, data, triangle): + coords = { # order matters: x, y, group, split + cfg["x_by"]: list(group_metadata(metas, cfg["x_by"]).keys()), + cfg["y_by"]: list(group_metadata(metas, cfg["y_by"]).keys()), + cfg["group_by"]: list(group_metadata(metas, cfg["group_by"]).keys()), + cfg["split_by"]: list(group_metadata(metas, cfg["split_by"]).keys()), + } + shape = [len(coord) for coord in coords.values()] + var_data = xr.DataArray(np.full(shape, np.nan), dims=list(coords.keys())) + data = xr.Dataset( + {"var": var_data}, + coords=coords) + # loop over each cell (coord combination) and load data if existing + for coord_tuple in itertools.product(*coords.values()): + selection = dict(zip(coords.keys(), coord_tuple)) + data['var'].loc[selection] = open_file(metas, **selection) + # data[coord_tuple] = (list(coords.keys(), value)) + if None in data.coords[cfg["split_by"]].values: + cfg.update({"default_split": None}) + else: + cfg.update({"default_split": data.coords[cfg["split_by"]].values[0]}) + log.debug("using %s as default split", cfg["default_split"]) + log.debug("Loaded Data:") + log.debug(data) + return data + + +# def references_legend(cfg, fig, metas): +# """create legend for references""" +# splits = + + +def overlay_reference(ax, data, triangle): """create triangular overlays for given data and axes.""" # use same colors as in main plot cmap = ax.get_images()[0].get_cmap() norm = ax.get_images()[0].norm - # print("plotting overlay for", split) - # print(data) - # print(ax.get_images()[0]) for i, j in itertools.product(*map(range, data.shape)): if np.isnan(data[i, j]): continue - print(data[i, j]) color = cmap(norm(data[i, j])) edges = [(e[0] + j, e[1] + i) for e in triangle] patch = patches.Polygon( @@ -157,55 +212,70 @@ def overlay_reference(cfg, ax, data, triangle): ax.add_patch(patch) -def plot_additional_splits(cfg, grid, datas): - """add references as triangular overlays""" - # if len(datas) < 2: - # return - # if len(datas) > 4: - # print("Too many splits for overlay, only 4 will be plotted.") - half = [ - [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)], # lower right - ] - quarters = [ - [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left - [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right - [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom - [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top (?) - ] - if len(datas) == 2: - quarters = half - - for ggg, data in enumerate(datas): - data.pop(cfg["reference"]) # TODO: fix me - for sss, (split, array) in enumerate(data.items()): - # if split == cfg["reference"]: - # continue - print("plotting overlay for", split, array.shape) - overlay_reference(cfg, grid[ggg], array, quarters[sss]) - - -def plot_group(cfg, ax, metas, title=None): +def plot_group(cfg, ax, data, title=None): """create matrix for one subplot in ax using plt.imshow() - returns image object and dict of numpy array for each reference (split). + by default split None is used, if all splits are named the first is used. + other splits will be added by overlaying triangles. """ - data, x_labels, y_labels = load_data(cfg, metas) - # plot matrix with simple rectangles - im = plot_matrix( - data[cfg["reference"]], y_labels, x_labels, ax, cfg["plot_kwargs"] + split = data.sel({cfg["split_by"]: cfg["default_split"]}) + print(f"Plotting group {title}") + print(split) + plot_matrix( + split.values.T, # 2d numpy array + split.coords[cfg["y_by"]].values, # y_labels + split.coords[cfg["x_by"]].values, # x_labels + ax, + cfg["plot_kwargs"], # TODO: pass cfg instead? ) if title is not None: ax.set_title(title) - # overlay splits (additional references) - # cmap might change after plot_group. set vmin and vmax before plotting? - # if len(data) > 1: - # plot_additional_splits(cfg, ax, data) ax.set(**cfg["axes_properties"]) - return im, data -def plot(cfg, grouped_metas): - """creates figure with subplots for each group""" +def plot_overlays(cfg, grid, data): + """call overlay_reference for each split in data and each group in grid. + """ + split_count = data.shape[3] + group_count = data.shape[2] + for i in range(group_count): + if split_count < 2: + log.warning("No additional splits for overlay.") + break + quarters = [ + [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left + [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right + [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom + [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top + ] + half = [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)] + if split_count > 4: + log.warning("Too many splits for overlay, only 3 will be plotted.") + group_data = data.isel({cfg["group_by"]: i}) + group_data = group_data.dropna(cfg["x_by"], how="all") + print(group_data) + + for sss in range(split_count): + split = group_data.isel({cfg["split_by"]: sss}) + split_label = split.coords[cfg["split_by"]].values.item() + # print(split) + # if cfg["split_by"] in split.dims: + # split = split.squeeze(cfg["split_by"]) + if split_label == cfg["default_split"]: + log.debug("Skipping default split for overlay.") + continue + if split_count == 2: + edges = half + else: + edges = quarters[sss] + overlay_reference(grid[i], split.values.T, edges) + + +def plot(cfg, data): + """creates figure with subplots for each group, sets same color range and + overlays additional references based on the content of data (xr.DataArray) + """ fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) + group_count = len(data.coords[cfg["group_by"]]) grid = ImageGrid( fig, 111, # similar to subplot(111) @@ -213,25 +283,30 @@ def plot(cfg, grouped_metas): cbar_location="right", cbar_pad=0.1, cbar_size=0.2, - nrows_ncols=(1, len(grouped_metas)), + nrows_ncols=(1, group_count), axes_pad=0.1, ) # remap colorbar to 10 discrete steps cmap = mpl.cm.get_cmap(cfg.get("cmap", "RdYlBu_r"), 10) cfg["plot_kwargs"]["cmap"] = cmap - datas = [] - for i, (group, metas) in enumerate(grouped_metas.items()): - title = group if len(grouped_metas) > 1 else None - im, data = plot_group(cfg, grid[i], metas, title=title) - datas.append(data) # collect data for overlay + for i in range(group_count): + group = data.isel({cfg["group_by"]: i}) + group = group.dropna(cfg["x_by"], how="all") + title = None + if group_count > 1: + title = group.coords[cfg["group_by"]].values.item() + plot_group(cfg, grid[i], group, title=title) # use same colorrange and colorbar for all subplots: unify_limits(cfg, grid) - grid.cbar_axes[0].colorbar(im, **cfg["cbar_kw"]) - # cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") - - # plot overlay with same colors - plot_additional_splits(cfg, grid, datas) - basename = "performance_metrics.png" + # set cb of first image as single cb for the figure + grid.cbar_axes[0].colorbar(grid[0].get_images()[0], **cfg["cbar_kwargs"]) + if data.shape[3] > 1: + plot_overlays(cfg, grid, data) + # replace_ticklabels with dict given in config + # if "xticklabels" in cfg["axes_properties"]: + # for ax in grid: + # ax.set_xticklabels(cfg["axes_properties"]["xticklabels"]) + basename = "performance_metrics" fname = get_plot_filename(basename, cfg) # plt.tight_layout() plt.savefig(fname, bbox_inches="tight") @@ -241,6 +316,7 @@ def plot(cfg, grouped_metas): def apply_grouping(cfg, metas): """returns sorted metadata, and group labels with positions""" + # NOTE obsolet for xarray group_by = cfg.get("group_by", "project") grouped = group_metadata(metas, group_by) counts = [] @@ -260,8 +336,8 @@ def set_defaults(cfg): cfg.setdefault("x_by", "alias") cfg.setdefault("y_by", "variable_group") cfg.setdefault("group_by", "project") - cfg.setdefault("split_by", None) - cfg.setdefault("cbar_kw", {}) + cfg.setdefault("split_by", "split") # extra facet + cfg.setdefault("cbar_kwargs", {}) cfg.setdefault("axes_properties", {}) cfg.setdefault("plot_kwargs", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") @@ -272,18 +348,15 @@ def set_defaults(cfg): def main(cfg): """run the diagnostic""" set_defaults(cfg) - metas = cfg["input_data"].values() - metas = remove_reference(metas) - grouped_metas = group_metadata(metas, cfg["group_by"]) cfg["figsize"] = (7.5, 5.5) cfg["y_by"] = "short_name" - cfg[ - "split_by" - ] = "ref" # TODO: hardcode split as extra facet? and use None as default? - cfg[ - "reference" - ] = "ref1" # TODO: this becomes obsolet if we use extra facet - plot(cfg, grouped_metas) + metas = list(cfg["input_data"].values()) + remove_reference(metas) + add_split_none(metas) + dataset = load_data(cfg, metas) + plot(cfg, dataset["var"]) + # grouped_metas = group_metadata(metas, cfg["group_by"]) + # plot(cfg, grouped_metas) if __name__ == "__main__": From 4ab76cbb99eab1cf413b764bc3869c9ad4cb1c0d Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 27 Mar 2024 13:08:41 +0100 Subject: [PATCH 10/56] 3 recipe examples --- .../recipes/recipe_perfmetrics_python.yml | 237 +++++++++++------- 1 file changed, 145 insertions(+), 92 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index 6661ec183f..d34f5144f2 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -8,30 +8,30 @@ documentation: authors: - ruhe_lukas # - cammarano_diego + maintainer: + - ruhe_lukas references: - eyring21ipcc preprocessors: - default: + default: &default_preproc custom_order: true - extract_time: - start_year: 1990 - start_month: 1 - start_day: 1 - end_year: 1992 - end_month: 12 - end_day: 31 regrid: target_grid: 3x3 - # scheme: nearest - scheme: - reference: esmf_regrid.schemes:ESMFAreaWeighted + scheme: nearest + # for icon: + # scheme: + # reference: esmf_regrid.schemes:ESMFAreaWeighted regrid_time: calendar: standard frequency: mon distance_metric: metric: pearsonr + rmse: + <<: *default_preproc + distance_metric: + metric: rmse cmip6_default: &cmip6 @@ -40,26 +40,44 @@ cmip6_default: &cmip6 project: CMIP6 timerange: '1990/1992' -datasets: - # ICON - # CMIP6 + +cmip6_examples: &cmip6_examples + - {<<: *cmip6, dataset: MRI-ESM2-0} + - {<<: *cmip6, dataset: NESM3} + - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} + - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} + - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} + - {<<: *cmip6, dataset: SAM0-UNICON} + - {<<: *cmip6, dataset: TaiESM1} + - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} + +cmip6_remaining: &cmip6_remaining - {<<: *cmip6, dataset: ACCESS-CM2} - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO} - {<<: *cmip6, dataset: AWI-CM-1-1-MR} - {<<: *cmip6, dataset: AWI-ESM-1-1-LR} - {<<: *cmip6, dataset: CESM2-FV2, institute: NCAR} - - {<<: *cmip6, dataset: CESM2-WACCM, institute: NCAR} - {<<: *cmip6, dataset: CESM2-WACCM-FV2, institute: NCAR} + - {<<: *cmip6, dataset: CESM2-WACCM, institute: NCAR} - {<<: *cmip6, dataset: CIESM, grid: gr} - {<<: *cmip6, dataset: CMCC-CM2-HR4} - {<<: *cmip6, dataset: CMCC-CM2-SR5} - - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: CMCC-ESM2} - {<<: *cmip6, dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} - {<<: *cmip6, dataset: CNRM-ESM2-1, ensemble: r1i1p1f2, grid: gr} + - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} + - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} + - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} - {<<: *cmip6, dataset: EC-Earth3-Veg-LR, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} + - {<<: *cmip6, dataset: EC-Earth3, grid: gr} - {<<: *cmip6, dataset: FGOALS-f3-L, grid: gr} - {<<: *cmip6, dataset: FGOALS-g3} - {<<: *cmip6, dataset: GFDL-CM4, grid: gr1} + - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} - {<<: *cmip6, dataset: GISS-E2-1-G} - {<<: *cmip6, dataset: GISS-E2-1-H} - {<<: *cmip6, dataset: HadGEM3-GC31-LL, ensemble: r1i1p1f3} @@ -68,121 +86,156 @@ datasets: - {<<: *cmip6, dataset: INM-CM4-8, grid: gr1} - {<<: *cmip6, dataset: INM-CM5-0, grid: gr1} - {<<: *cmip6, dataset: IPSL-CM6A-LR, grid: gr} - - {<<: *cmip6, dataset: MIROC6, reference_for_metric: false} + - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} - {<<: *cmip6, dataset: MIROC-ES2L, ensemble: r1i1p1f2} + - {<<: *cmip6, dataset: MIROC6} - {<<: *cmip6, dataset: MPI-ESM-1-2-HAM} - - {<<: *cmip6, dataset: MPI-ESM1-2-LR} - {<<: *cmip6, dataset: MPI-ESM1-2-HR} - - {<<: *cmip6, dataset: MRI-ESM2-0} - - {<<: *cmip6, dataset: NESM3} - - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} - - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} - - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} - - {<<: *cmip6, dataset: SAM0-UNICON} - - {<<: *cmip6, dataset: TaiESM1} - - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} - # - {<<: *cmip6, dataset: CMCC-ESM2} - # - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} - # - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} - # - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} - # - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} - + - {<<: *cmip6, dataset: MPI-ESM1-2-LR} + +datasets: + *cmip6_examples + # ICON + # - {project: ICON, dataset: ICON, exp: icon-2.6.1_atm_amip_R2B4_r1v1i1p1l1f1, timerange: '19900101/19930101'} + # # grid wrong? missing data # # - {<<: *cmip6, dataset: MCM-UA-1-0} # # - {<<: *cmip6, dataset: GISS-E2-1-G-CC} # # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} # # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} - # - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} - # - {<<: *cmip6, dataset: EC-Earth3, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} - # - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} diagnostics: - perfmetric: - description: perfmetric in python - variables: + simple: + variables: &simple_variables pr: &var_default preprocessor: default mip: Amon exp: historical - ref: ref1 - additional_datasets: &ref1 - - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true} - - {project: ICON, dataset: ICON, exp: icon-2.6.1_atm_amip_R2B4_r1v1i1p1l1f1} - pr_ref2: &var_default_ref2 + additional_datasets: &ref + # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3, reference_for_metric: true} + - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true, timerange: 1990/1992} + tas: + <<: *var_default + ps: + <<: *var_default + clt: + <<: *var_default + scripts: + perfmetrics: + script: perfmetrics/main.py + y_by: variable_group + + complex: + description: > + A more complex example with extra variables to support different reference datasets, + groups for datasets and some plot customization. + variables: + pr: + <<: *var_default + y_label: Precipitation + pr_vs_access: &var_default_ref2 <<: *var_default short_name: pr - ref: ref2 - additional_datasets: &ref2 + split: "ACCESS" + y_label: Precipitation + additional_datasets: &ref_access - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO, reference_for_metric: true} + pr_vs_era: + <<: *var_default + short_name: pr + split: "ERA5" + y_label: Precipitation + additional_datasets: &ref_era + - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3, timerange: 1990/1992, reference_for_metric: true} tas: - ref: ref1 <<: *var_default + y_label: Temperature # additional_datasets: - # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3} # - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2} - - tas_ref2: + tas_vs_access: short_name: tas <<: *var_default_ref2 - # rlut: - # <<: *var_default + y_label: Temperature + rlut: + <<: *var_default + y_label: "LW radiation out" + ps: - ref: ref1 <<: *var_default + y_label: "Surface pressure" # sfcWind: # <<: *var_default clt: - ref: ref1 <<: *var_default - # rlut: - # <<: *var_default + y_label: "Cloud cover" + clt_vs_esacci: + <<: *var_default + short_name: clt + split: "ESACCI" + y_label: "Cloud cover" + additional_datasets: + - {reference_for_metric: true, dataset: ESACCI-CLOUD, project: OBS, type: sat, version: AVHRR-fv3.0, tier: 2, timerange: '1990/1992'} psl: - ref: ref1 <<: *var_default + y_label: "Sea level pressure" additional_datasets: # - {dataset: ERA5, project: native6, type: reanaly, # version: v1, tier: 3, reference_for_metric: true} - - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} scripts: - python: + perfmetrics: script: perfmetrics/main.py + x_by: dataset + y_by: y_label + # split_by is the 'split' extra facet by default (set in variables) group_by: project - - # tas: &pcmip6, dataset: MCM-UA-1-0} - - # clt: - # <<:torical - # additional_datasets: - # - {<<: *cmip6, dataset: BCC-ESM1} - # - {dataset: ESACCI-CLOUD, project: OBS, type: sat, - # version: AVHRR-fv3.0, tier: 2, timerange: '1982/1999'} - # - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, - # timerange: '1982/1999'} - - # pr: - # - {dataset: GPCP-SG, project: obs4MIPs, level: L3, version: v2.3, tier: 1} - # - {dataset: GHCN, project: OBS, type: ground, version: 1, tier: 2} - - # psl: - # <<: aset: MCM-UA-1-0} - # - {dataset: JRA-55, project: ana4mips, type: reanalysis, tier: 1} - # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3} - - - # rlut: - # 6, dataset: MCM-UA-1-0} - # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, version: Ed2-8, - # tier: 1, start_year: 2001, end_year: 2015} - - # rsut: - # <<: *perfp6, dataset: KIOST-ESM, grid: gr1} - # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, version: Ed2-8, - # tier: 1, start_year: 2001, end_year: 2015} - + additional_datasets: + - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} + plot_kwargs: + vmin: 0.0 + vmax: 0.6 + cbar_kwargs: + label: "Pearson correlation coefficient" + ticks: [0, 0.3, 0.6] + extend: both + cmap: "Reds" + metrics: + variables: + pr: + <<: *var_default + y_label: "Precipitation" + pr_rmse: + short_name: ps + split: "RMSE" + y_label: "Precipitation" + <<: *var_default + preprocessor: rmse + tas: + <<: *var_default + y_label: "Temperature" + tas_rmse: + short_name: tas + split: "RMSE" + y_label: "Temperature" # extra_facet to share y tick + <<: *var_default + preprocessor: rmse + clt: + <<: *var_default + y_label: "Cloud cover" + clt_rmse: + short_name: clt + y_label: "Cloud cover" + split: "RMSE" + <<: *var_default + preprocessor: rmse + scripts: + perfmetrics: + script: perfmetrics/main.py + y_by: y_label + plot_kwargs: + vmin: 0 + vmax: 1.0 \ No newline at end of file From 32ad0346c48cf0b90dc427a36e632ba658751f23 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 27 Mar 2024 13:13:39 +0100 Subject: [PATCH 11/56] fix config overwrite --- esmvaltool/diag_scripts/perfmetrics/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 0d12e482c1..0c71ab0583 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -340,6 +340,7 @@ def set_defaults(cfg): cfg.setdefault("cbar_kwargs", {}) cfg.setdefault("axes_properties", {}) cfg.setdefault("plot_kwargs", {}) + cfg.setdefault("figsize", (7.5, 5.5)) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") cfg["plot_kwargs"].setdefault("vmin", 0) cfg["plot_kwargs"].setdefault("vmax", 1) @@ -348,15 +349,11 @@ def set_defaults(cfg): def main(cfg): """run the diagnostic""" set_defaults(cfg) - cfg["figsize"] = (7.5, 5.5) - cfg["y_by"] = "short_name" metas = list(cfg["input_data"].values()) remove_reference(metas) add_split_none(metas) dataset = load_data(cfg, metas) plot(cfg, dataset["var"]) - # grouped_metas = group_metadata(metas, cfg["group_by"]) - # plot(cfg, grouped_metas) if __name__ == "__main__": From d66e5e21c6c174a8b212e78dd856b09cfe39f1f2 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 28 Mar 2024 17:10:11 +0100 Subject: [PATCH 12/56] legend for multiple references --- esmvaltool/diag_scripts/perfmetrics/main.py | 157 +++++++++++++------- 1 file changed, 106 insertions(+), 51 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 0c71ab0583..896b727653 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -39,9 +39,24 @@ Data labeled with this string, will be used as main rectangles. All other splits will be plotted as overlays. This can be used to choose the base reference, while all references are labeled for the legend. -plot_legend: bool, optional - If True, a legend will be plotted showing which additional reference, - belongs to the triangle positions. +legend: dict, optional + Customize, if, how and where the legend is plotted. The 'best' position + and size of the legend depends on multiple parameters of the figure + (i.e. lengths of labels, aspect ratio of the plots...). And might require + manual adjustment of `x`, `y` and `size` to fit the figure layout. + Keys (each optional) that will be handled are: + position: str or None, optional + Position of the legend. Can be 'right' or 'left'. Or set to None to + disable plotting the legend. By default 'right'. + x_offset: float, optional + Manually adjust horizontal position to save space or fix overlap. + Number given in Inches. By default 0. + y_offset: float, optional + Manually adjust vertical position to save space or fix overlap. + Number given in Inches. By default 0. + size: float, optional + Size of the legend in Inches. By default 0.3. + TODO: set defaults and add offsets to legend function. plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. Colormaps will be converted to 11 discrete steps automatically. Default @@ -186,9 +201,61 @@ def load_data(cfg, metas): return data -# def references_legend(cfg, fig, metas): -# """create legend for references""" -# splits = +def split_legend(cfg, grid, data): + """create legend for references, based on split coordinate in the dataset. + Mpl handles axes positions in relative figure coordinates. To anchor the + legend to the origin of the first graph (bottom left) with fixed size, + without messing up the layout for changing figure sizes a few extra steps + are required. + TODO: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies + this a bit by using `append_axes`. + """ + + fig = grid[0].get_figure() + fig.canvas.draw() # set axes position in figure (dont call tight_layout()) + size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) + fig_w, fig_h = fig.get_size_inches() # physical size of figure + ax_w, ax_h = (size / fig_w, size / fig_h) # legend size in figure coords + gap_x, gap_y = (0.3 / fig_w, 0.3 / fig_h) # margins to plot in fig coords + # anchor legend on origin of first plot or colorbar + anchor = grid[0].get_position().bounds # relative figure coordinates + if cfg["legend"].get("position", "right") == "right": + cbar_x = grid.cbar_axes[0].get_position().bounds[0] + gap_x *= 0.8 # compensate colorbar padding + anchor = (cbar_x+gap_x, anchor[1]-gap_y-ax_h) + else: + anchor = (anchor[0]-gap_x-ax_w, anchor[1]-gap_y-ax_h) + # create legend as empty imshow like axes in figure coordinates + legend = fig.add_axes([anchor[0], anchor[1], ax_w, ax_h]) + legend.imshow(np.zeros((1, 1))) # same axes properties as main plot + legend.set_xticks([]) + legend.set_yticks([]) + axy = legend.twinx() # add twins to allow axes labels on all sides + axy.set_yticks([]) + axx = legend.twiny() + axx.set_xticks([]) + + labels = data.coords[cfg["split_by"]].values + label_at = [ # order matches get_triangle_nodes (halfs and quarters) + legend.set_ylabel, # left + axy.set_ylabel, # right + legend.set_xlabel, # bottom + axx.set_xlabel, # top + ] + for i, _ in enumerate(labels): + nodes = get_triangle_nodes(i, len(labels)) + colors = ["#bbb", "#ccc", "#ddd", "#eee"] + patch = patches.Polygon( + nodes, + closed=True, + facecolor=colors[i], + edgecolor="black", + linewidth=0.5, + fill=True, + ) + legend.add_patch(patch) + label_at[i](labels[i]) + print(labels) def overlay_reference(ax, data, triangle): @@ -225,13 +292,35 @@ def plot_group(cfg, ax, data, title=None): split.coords[cfg["y_by"]].values, # y_labels split.coords[cfg["x_by"]].values, # x_labels ax, - cfg["plot_kwargs"], # TODO: pass cfg instead? + cfg["plot_kwargs"], ) if title is not None: ax.set_title(title) ax.set(**cfg["axes_properties"]) +def get_triangle_nodes(position, total_count=2): + """returns list of three tuples with relative x, y coordinates for nodes + of triangle (-0.5 to +0.5) at given quarters (total_count>2) or + halfs (total_count==2). + NOTE: Order matters. Ensure axis labels for the legend match when changing. + """ + if total_count < 3: + halfs = [ + [(0.5, -0.5), (-0.5, -0.5), (-0.5, 0.5)], # top left + [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)], # bottom right + ] + return halfs[position] + else: + quarters = [ + [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left + [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right + [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom + [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top + ] + return quarters[position] + + def plot_overlays(cfg, grid, data): """call overlay_reference for each split in data and each group in grid. """ @@ -239,35 +328,20 @@ def plot_overlays(cfg, grid, data): group_count = data.shape[2] for i in range(group_count): if split_count < 2: - log.warning("No additional splits for overlay.") + log.debug("No additional splits for overlay.") break - quarters = [ - [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left - [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right - [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom - [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top - ] - half = [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)] if split_count > 4: log.warning("Too many splits for overlay, only 3 will be plotted.") group_data = data.isel({cfg["group_by"]: i}) group_data = group_data.dropna(cfg["x_by"], how="all") - print(group_data) - for sss in range(split_count): split = group_data.isel({cfg["split_by"]: sss}) split_label = split.coords[cfg["split_by"]].values.item() - # print(split) - # if cfg["split_by"] in split.dims: - # split = split.squeeze(cfg["split_by"]) if split_label == cfg["default_split"]: log.debug("Skipping default split for overlay.") continue - if split_count == 2: - edges = half - else: - edges = quarters[sss] - overlay_reference(grid[i], split.values.T, edges) + nodes = get_triangle_nodes(sss, split_count) + overlay_reference(grid[i], split.values.T, nodes) def plot(cfg, data): @@ -302,33 +376,13 @@ def plot(cfg, data): grid.cbar_axes[0].colorbar(grid[0].get_images()[0], **cfg["cbar_kwargs"]) if data.shape[3] > 1: plot_overlays(cfg, grid, data) - # replace_ticklabels with dict given in config - # if "xticklabels" in cfg["axes_properties"]: - # for ax in grid: - # ax.set_xticklabels(cfg["axes_properties"]["xticklabels"]) + if cfg.get("plot_legend", True): + split_legend(cfg, grid, data) basename = "performance_metrics" fname = get_plot_filename(basename, cfg) - # plt.tight_layout() - plt.savefig(fname, bbox_inches="tight") - print("Figure saved:") - print(fname) - - -def apply_grouping(cfg, metas): - """returns sorted metadata, and group labels with positions""" - # NOTE obsolet for xarray - group_by = cfg.get("group_by", "project") - grouped = group_metadata(metas, group_by) - counts = [] - labels = [] - for group, metas in grouped.items(): - labels.append(group) - x_entries = len(group_metadata(metas, cfg["x_by"])) - counts.append(x_entries) - positions = np.cumsum(counts) - groups = dict(zip(labels, positions)) - metas = list(itertools.chain(*grouped.values())) - return metas, groups + plt.savefig(fname, bbox_inches="tight") # , bbox_inches=cfg.get("bbox_inches", "tight")) + log.info("Figure saved:") + log.info(fname) def set_defaults(cfg): @@ -340,7 +394,8 @@ def set_defaults(cfg): cfg.setdefault("cbar_kwargs", {}) cfg.setdefault("axes_properties", {}) cfg.setdefault("plot_kwargs", {}) - cfg.setdefault("figsize", (7.5, 5.5)) + cfg.setdefault("figsize", (7.5, 3.5)) + cfg.setdefault("legend", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") cfg["plot_kwargs"].setdefault("vmin", 0) cfg["plot_kwargs"].setdefault("vmax", 1) From 30541be010703d8f9418687c436e7a9132c98d54 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 2 Apr 2024 11:16:31 +0200 Subject: [PATCH 13/56] pre-commit --- esmvaltool/diag_scripts/perfmetrics/main.py | 99 +++++++++++---------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 896b727653..8c1c1e1744 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -3,7 +3,7 @@ Description ----------- This diagnostic provides plot functionalities for performance metrics. -The multi model overview heatmap might be usefull for different +The multi model overview heatmap might be useful for different tasks and therefore this diagnostic tries to be as flexible as possible. X and Y axis, grouping parameter and slits for each rectangle can be configured in the recipe. All *_by parameters can be set to any metadata @@ -28,7 +28,7 @@ grouping into subplots. By default 'project'. split_by: str, optional - The rectangles can be splitted into 2-4 triangles. This is used to show + The rectangles can be split into 2-4 triangles. This is used to show metrics for different references. For this case there is no need to change this parameter. Multiple variables can be set in the recipe with `split` assigned as extra_facet to label the different references. Data without @@ -40,8 +40,8 @@ splits will be plotted as overlays. This can be used to choose the base reference, while all references are labeled for the legend. legend: dict, optional - Customize, if, how and where the legend is plotted. The 'best' position - and size of the legend depends on multiple parameters of the figure + Customize, if, how and where the legend is plotted. The 'best' position + and size of the legend depends on multiple parameters of the figure (i.e. lengths of labels, aspect ratio of the plots...). And might require manual adjustment of `x`, `y` and `size` to fit the figure layout. Keys (each optional) that will be handled are: @@ -80,26 +80,28 @@ By default [5, 3]. """ -import logging import itertools +import logging + import matplotlib as mpl import matplotlib.pyplot as plt -import matplotlib.patches as patches import numpy as np +import xarray as xr +from matplotlib import patches +from mpl_toolkits.axes_grid1 import ImageGrid + from esmvaltool.diag_scripts.shared import ( get_plot_filename, group_metadata, run_diagnostic, select_metadata, ) -from mpl_toolkits.axes_grid1 import ImageGrid -import xarray as xr log = logging.getLogger(__name__) def unify_limits(cfg, grid): - """set same limits for all subplots""" + """Set same limits for all subplots.""" vmin, vmax = np.inf, -np.inf images = [ax.get_images()[0] for ax in grid] for im in images: @@ -133,10 +135,11 @@ def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): def remove_reference(metas): - """remove reference for metric from list of metadata - setting split=none allows to omit it in recipe, but handle it as special - case of splitted data. - list() creates a copy with same references to allow removing in place + """Remove reference for metric from list of metadata setting split=none + allows to omit it in recipe, but handle it as special case of split data. + + list() creates a copy with same references to allow removing in + place """ for meta in list(metas): if meta.get("reference_for_metric", False): @@ -144,16 +147,17 @@ def remove_reference(metas): def add_split_none(metas): - """list of metadata with split=None if no split is given""" + """List of metadata with split=None if no split is given.""" for meta in metas: if "split" not in meta: meta["split"] = None def open_file(metadata, **selection): - """try to find a single file for selection and return data. - If multiple files are found, raise an error. - If no file is found, return np.nan. + """Try to find a single file for selection and return data. + + If multiple files are found, raise an error. If no file is found, + return np.nan. """ metas = select_metadata(metadata, **selection) if len(metas) > 1: @@ -169,11 +173,12 @@ def open_file(metadata, **selection): def load_data(cfg, metas): - """load all nc files from metadata into xarray dataset. - The dataset contains all relevant information for the plot. - Coord names are metadata keys, ordered as x, y, group, split. - The default reference is None, or if all references are named - the first from the list. + """Load all nc files from metadata into xarray dataset. + + The dataset contains all relevant information for the plot. Coord + names are metadata keys, ordered as x, y, group, split. The default + reference is None, or if all references are named the first from the + list. """ coords = { # order matters: x, y, group, split cfg["x_by"]: list(group_metadata(metas, cfg["x_by"]).keys()), @@ -183,9 +188,7 @@ def load_data(cfg, metas): } shape = [len(coord) for coord in coords.values()] var_data = xr.DataArray(np.full(shape, np.nan), dims=list(coords.keys())) - data = xr.Dataset( - {"var": var_data}, - coords=coords) + data = xr.Dataset({"var": var_data}, coords=coords) # loop over each cell (coord combination) and load data if existing for coord_tuple in itertools.product(*coords.values()): selection = dict(zip(coords.keys(), coord_tuple)) @@ -202,7 +205,8 @@ def load_data(cfg, metas): def split_legend(cfg, grid, data): - """create legend for references, based on split coordinate in the dataset. + """Create legend for references, based on split coordinate in the dataset. + Mpl handles axes positions in relative figure coordinates. To anchor the legend to the origin of the first graph (bottom left) with fixed size, without messing up the layout for changing figure sizes a few extra steps @@ -210,7 +214,7 @@ def split_legend(cfg, grid, data): TODO: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies this a bit by using `append_axes`. """ - + fig = grid[0].get_figure() fig.canvas.draw() # set axes position in figure (dont call tight_layout()) size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) @@ -222,9 +226,9 @@ def split_legend(cfg, grid, data): if cfg["legend"].get("position", "right") == "right": cbar_x = grid.cbar_axes[0].get_position().bounds[0] gap_x *= 0.8 # compensate colorbar padding - anchor = (cbar_x+gap_x, anchor[1]-gap_y-ax_h) + anchor = (cbar_x + gap_x, anchor[1] - gap_y - ax_h) else: - anchor = (anchor[0]-gap_x-ax_w, anchor[1]-gap_y-ax_h) + anchor = (anchor[0] - gap_x - ax_w, anchor[1] - gap_y - ax_h) # create legend as empty imshow like axes in figure coordinates legend = fig.add_axes([anchor[0], anchor[1], ax_w, ax_h]) legend.imshow(np.zeros((1, 1))) # same axes properties as main plot @@ -236,7 +240,7 @@ def split_legend(cfg, grid, data): axx.set_xticks([]) labels = data.coords[cfg["split_by"]].values - label_at = [ # order matches get_triangle_nodes (halfs and quarters) + label_at = [ # order matches get_triangle_nodes (halves and quarters) legend.set_ylabel, # left axy.set_ylabel, # right legend.set_xlabel, # bottom @@ -259,7 +263,7 @@ def split_legend(cfg, grid, data): def overlay_reference(ax, data, triangle): - """create triangular overlays for given data and axes.""" + """Create triangular overlays for given data and axes.""" # use same colors as in main plot cmap = ax.get_images()[0].get_cmap() norm = ax.get_images()[0].norm @@ -280,9 +284,10 @@ def overlay_reference(ax, data, triangle): def plot_group(cfg, ax, data, title=None): - """create matrix for one subplot in ax using plt.imshow() - by default split None is used, if all splits are named the first is used. - other splits will be added by overlaying triangles. + """Create matrix for one subplot in ax using plt.imshow() + + by default split None is used, if all splits are named the first is + used. Other splits will be added by overlaying triangles. """ split = data.sel({cfg["split_by"]: cfg["default_split"]}) print(f"Plotting group {title}") @@ -300,17 +305,18 @@ def plot_group(cfg, ax, data, title=None): def get_triangle_nodes(position, total_count=2): - """returns list of three tuples with relative x, y coordinates for nodes - of triangle (-0.5 to +0.5) at given quarters (total_count>2) or - halfs (total_count==2). + """Returns list of three tuples with relative x, y coordinates for nodes of + triangle (-0.5 to +0.5) at given quarters (total_count>2) or halves + (total_count==2). + NOTE: Order matters. Ensure axis labels for the legend match when changing. """ if total_count < 3: - halfs = [ + halves = [ [(0.5, -0.5), (-0.5, -0.5), (-0.5, 0.5)], # top left [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)], # bottom right ] - return halfs[position] + return halves[position] else: quarters = [ [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left @@ -322,8 +328,7 @@ def get_triangle_nodes(position, total_count=2): def plot_overlays(cfg, grid, data): - """call overlay_reference for each split in data and each group in grid. - """ + """Call overlay_reference for each split in data and each group in grid.""" split_count = data.shape[3] group_count = data.shape[2] for i in range(group_count): @@ -345,9 +350,9 @@ def plot_overlays(cfg, grid, data): def plot(cfg, data): - """creates figure with subplots for each group, sets same color range and - overlays additional references based on the content of data (xr.DataArray) - """ + """Creates figure with subplots for each group, sets same color range and + overlays additional references based on the content of data + (xr.DataArray)""" fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) grid = ImageGrid( @@ -380,13 +385,13 @@ def plot(cfg, data): split_legend(cfg, grid, data) basename = "performance_metrics" fname = get_plot_filename(basename, cfg) - plt.savefig(fname, bbox_inches="tight") # , bbox_inches=cfg.get("bbox_inches", "tight")) + plt.savefig(fname, bbox_inches="tight") log.info("Figure saved:") log.info(fname) def set_defaults(cfg): - """set default values for most important config parameters""" + """Set default values for most important config parameters.""" cfg.setdefault("x_by", "alias") cfg.setdefault("y_by", "variable_group") cfg.setdefault("group_by", "project") @@ -402,7 +407,7 @@ def set_defaults(cfg): def main(cfg): - """run the diagnostic""" + """Run the diagnostic.""" set_defaults(cfg) metas = list(cfg["input_data"].values()) remove_reference(metas) From d6a541fcd88a19a6748c8fa90013c4ba49171cd1 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 2 Apr 2024 17:33:56 +0200 Subject: [PATCH 14/56] add normalization for metrics --- esmvaltool/diag_scripts/perfmetrics/main.py | 54 +++++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index 8c1c1e1744..b47bdb71da 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -16,6 +16,10 @@ Configuration parameters through recipe: ---------------------------------------- +normalize: str or None, optional + ('mean', 'median', 'centered_mean', 'centered_median', None). + Subtract median/mean if centered. Divide by median/mean if not None. + By default None. x_by: str, optional Metadata key for x coordinate. By default 'alias'. @@ -39,6 +43,7 @@ Data labeled with this string, will be used as main rectangles. All other splits will be plotted as overlays. This can be used to choose the base reference, while all references are labeled for the legend. + By default None. legend: dict, optional Customize, if, how and where the legend is plotted. The 'best' position and size of the legend depends on multiple parameters of the figure @@ -56,7 +61,6 @@ Number given in Inches. By default 0. size: float, optional Size of the legend in Inches. By default 0.3. - TODO: set defaults and add offsets to legend function. plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. Colormaps will be converted to 11 discrete steps automatically. Default @@ -74,6 +78,10 @@ https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html E.g. xlabel, ylabel, yticklabels, xmargin... By default {}. +nan_color: str or None, optional + Matplotlib named color or hexcode for NaN values. If set to None, + no triagnles are plotted for NaN values. + By default 'white'. figsize: list(float), optional [width, height] of the figure in inches. The final figure will be saved with bbox_inches="tight", which can change the resulting aspect ratio. @@ -211,10 +219,9 @@ def split_legend(cfg, grid, data): legend to the origin of the first graph (bottom left) with fixed size, without messing up the layout for changing figure sizes a few extra steps are required. - TODO: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies + NOTE: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies this a bit by using `append_axes`. """ - fig = grid[0].get_figure() fig.canvas.draw() # set axes position in figure (dont call tight_layout()) size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) @@ -226,9 +233,11 @@ def split_legend(cfg, grid, data): if cfg["legend"].get("position", "right") == "right": cbar_x = grid.cbar_axes[0].get_position().bounds[0] gap_x *= 0.8 # compensate colorbar padding - anchor = (cbar_x + gap_x, anchor[1] - gap_y - ax_h) + anchor = (cbar_x + gap_x + cfg["legend"]["x_offset"], + anchor[1] - gap_y - ax_h + cfg["legend"]["y_offset"]) else: - anchor = (anchor[0] - gap_x - ax_w, anchor[1] - gap_y - ax_h) + anchor = (anchor[0] - gap_x - ax_w + cfg["legend"]["x_offset"], + anchor[1] - gap_y - ax_h + cfg["legend"]["y_offset"]) # create legend as empty imshow like axes in figure coordinates legend = fig.add_axes([anchor[0], anchor[1], ax_w, ax_h]) legend.imshow(np.zeros((1, 1))) # same axes properties as main plot @@ -262,13 +271,15 @@ def split_legend(cfg, grid, data): print(labels) -def overlay_reference(ax, data, triangle): +def overlay_reference(cfg, ax, data, triangle): """Create triangular overlays for given data and axes.""" # use same colors as in main plot cmap = ax.get_images()[0].get_cmap() norm = ax.get_images()[0].norm + if cfg["nan_color"] is not None: + cmap.set_bad(cfg["nan_color"]) for i, j in itertools.product(*map(range, data.shape)): - if np.isnan(data[i, j]): + if np.isnan(data[i, j]) and cfg["nan_color"] is None: continue color = cmap(norm(data[i, j])) edges = [(e[0] + j, e[1] + i) for e in triangle] @@ -346,7 +357,7 @@ def plot_overlays(cfg, grid, data): log.debug("Skipping default split for overlay.") continue nodes = get_triangle_nodes(sss, split_count) - overlay_reference(grid[i], split.values.T, nodes) + overlay_reference(cfg, grid[i], split.values.T, nodes) def plot(cfg, data): @@ -390,20 +401,39 @@ def plot(cfg, data): log.info(fname) +def normalize(array, method, dims): + shift = 0 + norm = 1 + if "mean" in method: + norm = array.mean(dim=dims) + elif "median" in method: + norm = array.median(dim=dims) + if "centered" in method: + shift = norm + normalized = (array - shift) / norm + return normalized + + def set_defaults(cfg): """Set default values for most important config parameters.""" + cfg.setdefault("normalize", "centered_median") cfg.setdefault("x_by", "alias") cfg.setdefault("y_by", "variable_group") cfg.setdefault("group_by", "project") cfg.setdefault("split_by", "split") # extra facet + cfg.setdefault("default_split", None) cfg.setdefault("cbar_kwargs", {}) cfg.setdefault("axes_properties", {}) + cfg.setdefault("nan_color", 'white') cfg.setdefault("plot_kwargs", {}) - cfg.setdefault("figsize", (7.5, 3.5)) - cfg.setdefault("legend", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") cfg["plot_kwargs"].setdefault("vmin", 0) cfg["plot_kwargs"].setdefault("vmax", 1) + cfg.setdefault("figsize", (7.5, 3.5)) + cfg.setdefault("legend", {}) + cfg["legend"].setdefault("x_offset", 0) + cfg["legend"].setdefault("y_offset", 0) + cfg["legend"].setdefault("size", 0.3) def main(cfg): @@ -413,6 +443,10 @@ def main(cfg): remove_reference(metas) add_split_none(metas) dataset = load_data(cfg, metas) + if cfg["normalize"] is not None: + dataset["var"] = normalize(dataset["var"], cfg["normalize"], + [cfg["x_by"], cfg["group_by"]]) + plot(cfg, dataset["var"]) From 7fb8a10fef3b5d20e604a8f8c021883a0e7ee889 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 3 Apr 2024 11:23:12 +0200 Subject: [PATCH 15/56] codacy fixes --- esmvaltool/diag_scripts/perfmetrics/main.py | 121 ++++++++++---------- 1 file changed, 59 insertions(+), 62 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/main.py index b47bdb71da..98ace314fc 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/main.py @@ -108,26 +108,26 @@ log = logging.getLogger(__name__) -def unify_limits(cfg, grid): +def unify_limits(grid): """Set same limits for all subplots.""" vmin, vmax = np.inf, -np.inf images = [ax.get_images()[0] for ax in grid] - for im in images: - vmin = min(vmin, im.get_clim()[0]) - vmax = max(vmax, im.get_clim()[1]) - for im in images: - im.set_clim(vmin, vmax) + for img in images: + vmin = min(vmin, img.get_clim()[0]) + vmax = max(vmax, img.get_clim()[1]) + for img in images: + img.set_clim(vmin, vmax) -def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): +def plot_matrix(data, row_labels, col_labels, axe, plot_kwargs): """Create an image for given data.""" - im = ax.imshow(data, **plot_kwargs) + img = axe.imshow(data, **plot_kwargs) # Show all ticks and label them with the respective list entries. - ax.set_xticks(np.arange(data.shape[1]), labels=col_labels) - ax.set_yticks(np.arange(data.shape[0]), labels=row_labels) + axe.set_xticks(np.arange(data.shape[1]), labels=col_labels) + axe.set_yticks(np.arange(data.shape[0]), labels=row_labels) # Rotate the tick labels and set their alignment. plt.setp( - ax.get_xticklabels(), + axe.get_xticklabels(), rotation=90, ha="right", va="center", @@ -135,19 +135,17 @@ def plot_matrix(data, row_labels, col_labels, ax, plot_kwargs): ) # Turn spines off and create white grid. # ax.spines[:].set_visible(False) - ax.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) - ax.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) - ax.grid(which="minor", color="black", linestyle="-", linewidth=1) - ax.tick_params(which="both", bottom=False, left=False) - return im + axe.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) + axe.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) + axe.grid(which="minor", color="black", linestyle="-", linewidth=1) + axe.tick_params(which="both", bottom=False, left=False) + return img def remove_reference(metas): - """Remove reference for metric from list of metadata setting split=none - allows to omit it in recipe, but handle it as special case of split data. + """Remove reference for metric from list of metadata. - list() creates a copy with same references to allow removing in - place + NOTE: list() creates a copy with same references to allow removing in place """ for meta in list(metas): if meta.get("reference_for_metric", False): @@ -174,9 +172,9 @@ def open_file(metadata, **selection): log.debug("No Metadata found for %s", selection) return np.nan log.warning("Metadata found for %s", selection) - ds = xr.open_dataset(metas[0]["filename"]) - varname = list(ds.data_vars.keys())[0] - return ds[varname].values.item() + das = xr.open_dataset(metas[0]["filename"]) + varname = list(das.data_vars.keys())[0] + return das[varname].values.item() # iris.load_cube(metas[0]["filename"]).data @@ -225,37 +223,35 @@ def split_legend(cfg, grid, data): fig = grid[0].get_figure() fig.canvas.draw() # set axes position in figure (dont call tight_layout()) size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) - fig_w, fig_h = fig.get_size_inches() # physical size of figure - ax_w, ax_h = (size / fig_w, size / fig_h) # legend size in figure coords - gap_x, gap_y = (0.3 / fig_w, 0.3 / fig_h) # margins to plot in fig coords + fig_size = fig.get_size_inches() # physical size of figure + ax_size = (size / fig_size[0], size / fig_size[1]) # legend (fig coords) + gaps = (0.3 / fig_size[0], 0.3 / fig_size[1]) # margins (fig coords) # anchor legend on origin of first plot or colorbar anchor = grid[0].get_position().bounds # relative figure coordinates if cfg["legend"].get("position", "right") == "right": cbar_x = grid.cbar_axes[0].get_position().bounds[0] - gap_x *= 0.8 # compensate colorbar padding - anchor = (cbar_x + gap_x + cfg["legend"]["x_offset"], - anchor[1] - gap_y - ax_h + cfg["legend"]["y_offset"]) + gaps[0] *= 0.8 # compensate colorbar padding + anchor = (cbar_x + gaps[0] + cfg["legend"]["x_offset"], + anchor[1] - gaps[1] - ax_size[1] + cfg["legend"]["y_offset"]) else: - anchor = (anchor[0] - gap_x - ax_w + cfg["legend"]["x_offset"], - anchor[1] - gap_y - ax_h + cfg["legend"]["y_offset"]) + anchor = (anchor[0] - gaps[0] - ax_size[0] + cfg["legend"]["x_offset"], + anchor[1] - gaps[1] - ax_size[1] + cfg["legend"]["y_offset"]) # create legend as empty imshow like axes in figure coordinates - legend = fig.add_axes([anchor[0], anchor[1], ax_w, ax_h]) + legend = fig.add_axes([anchor[0], anchor[1], ax_size[0], ax_size[1]]) legend.imshow(np.zeros((1, 1))) # same axes properties as main plot legend.set_xticks([]) legend.set_yticks([]) - axy = legend.twinx() # add twins to allow axes labels on all sides - axy.set_yticks([]) - axx = legend.twiny() - axx.set_xticks([]) - + twins = [legend.twiny(), legend.twinx()] + twins[1].set_yticks([]) + twins[0].set_xticks([]) labels = data.coords[cfg["split_by"]].values label_at = [ # order matches get_triangle_nodes (halves and quarters) legend.set_ylabel, # left - axy.set_ylabel, # right + twins[1].set_ylabel, # right legend.set_xlabel, # bottom - axx.set_xlabel, # top + twins[0].set_xlabel, # top ] - for i, _ in enumerate(labels): + for i, label in enumerate(labels): nodes = get_triangle_nodes(i, len(labels)) colors = ["#bbb", "#ccc", "#ddd", "#eee"] patch = patches.Polygon( @@ -267,15 +263,14 @@ def split_legend(cfg, grid, data): fill=True, ) legend.add_patch(patch) - label_at[i](labels[i]) - print(labels) + label_at[i](label) -def overlay_reference(cfg, ax, data, triangle): +def overlay_reference(cfg, axe, data, triangle): """Create triangular overlays for given data and axes.""" # use same colors as in main plot - cmap = ax.get_images()[0].get_cmap() - norm = ax.get_images()[0].norm + cmap = axe.get_images()[0].get_cmap() + norm = axe.get_images()[0].norm if cfg["nan_color"] is not None: cmap.set_bad(cfg["nan_color"]) for i, j in itertools.product(*map(range, data.shape)): @@ -291,10 +286,10 @@ def overlay_reference(cfg, ax, data, triangle): linewidth=0.5, fill=True, ) - ax.add_patch(patch) + axe.add_patch(patch) -def plot_group(cfg, ax, data, title=None): +def plot_group(cfg, axe, data, title=None): """Create matrix for one subplot in ax using plt.imshow() by default split None is used, if all splits are named the first is @@ -307,19 +302,21 @@ def plot_group(cfg, ax, data, title=None): split.values.T, # 2d numpy array split.coords[cfg["y_by"]].values, # y_labels split.coords[cfg["x_by"]].values, # x_labels - ax, + axe, cfg["plot_kwargs"], ) if title is not None: - ax.set_title(title) - ax.set(**cfg["axes_properties"]) + axe.set_title(title) + axe.set(**cfg["axes_properties"]) def get_triangle_nodes(position, total_count=2): - """Returns list of three tuples with relative x, y coordinates for nodes of - triangle (-0.5 to +0.5) at given quarters (total_count>2) or halves - (total_count==2). + """Returns list of nodes with relative x, y coordinates. + The nodes of the triangle are given as list of three tuples. Each tuples + contains relative coordinates (-0.5 to +0.5). For total of <= 2 a top left + (position=0) and bottom right (position=1) rectangle is returned. + For higher counts (3 or 4) one quartile is returned for each position. NOTE: Order matters. Ensure axis labels for the legend match when changing. """ if total_count < 3: @@ -328,14 +325,13 @@ def get_triangle_nodes(position, total_count=2): [(0.5, -0.5), (0.5, 0.5), (-0.5, 0.5)], # bottom right ] return halves[position] - else: - quarters = [ - [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left - [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right - [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom - [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top - ] - return quarters[position] + quarters = [ + [(-0.5, -0.5), (0, 0), (-0.5, 0.5)], # left + [(0.5, -0.5), (0, 0), (0.5, 0.5)], # right + [(-0.5, 0.5), (0, 0), (0.5, 0.5)], # bottom + [(-0.5, -0.5), (0, 0), (0.5, -0.5)], # top + ] + return quarters[position] def plot_overlays(cfg, grid, data): @@ -387,7 +383,7 @@ def plot(cfg, data): title = group.coords[cfg["group_by"]].values.item() plot_group(cfg, grid[i], group, title=title) # use same colorrange and colorbar for all subplots: - unify_limits(cfg, grid) + unify_limits(grid) # set cb of first image as single cb for the figure grid.cbar_axes[0].colorbar(grid[0].get_images()[0], **cfg["cbar_kwargs"]) if data.shape[3] > 1: @@ -402,6 +398,7 @@ def plot(cfg, data): def normalize(array, method, dims): + """Divide and shift values along dims depending on method.""" shift = 0 norm = 1 if "mean" in method: From eba93db12dfc118b601dbeb9ecf6ed300068be21 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 3 Apr 2024 17:15:13 +0200 Subject: [PATCH 16/56] WIP Documentation --- .../esmvaltool.diag_scripts.perfmetrics.rst | 20 +++ .../portrait_plot.rst | 6 + doc/sphinx/source/api/esmvaltool.rst | 1 + doc/sphinx/source/recipes/index.rst | 1 + .../source/recipes/recipe_perfmetrics.rst | 92 ++++++++--- .../recipes/recipe_perfmetrics_python.rst | 146 ++++++++++++++++++ .../perfmetrics/{main.py => portrait_plot.py} | 2 +- .../recipes/recipe_perfmetrics_python.yml | 32 ++-- 8 files changed, 263 insertions(+), 37 deletions(-) create mode 100644 doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst create mode 100644 doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst create mode 100644 doc/sphinx/source/recipes/recipe_perfmetrics_python.rst rename esmvaltool/diag_scripts/perfmetrics/{main.py => portrait_plot.py} (99%) diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst new file mode 100644 index 0000000000..a4936689bc --- /dev/null +++ b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst @@ -0,0 +1,20 @@ +.. _api.esmvaltool.diag_scripts.perfmetrics: + +Performance Metrics +=================== + +This module contains various reusable diagnostics and plot scripts. + + +Examples +-------- + +* :ref:`recipe_perfmetrics_python ` + + +Diagnostic scripts +------------------ +.. toctree:: + :maxdepth: 1 + + esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst new file mode 100644 index 0000000000..3679d7015b --- /dev/null +++ b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst @@ -0,0 +1,6 @@ +.. _api.esmvaltool.diag_scripts.perfmetrics.portrait_plot: + +Plot performance metrics of multiple datasets vs up to four references +====================================================================== + +.. automodule:: esmvaltool.diag_scripts.perfmetrics.portrait_plot diff --git a/doc/sphinx/source/api/esmvaltool.rst b/doc/sphinx/source/api/esmvaltool.rst index b080b81ac8..f3553d7056 100644 --- a/doc/sphinx/source/api/esmvaltool.rst +++ b/doc/sphinx/source/api/esmvaltool.rst @@ -29,3 +29,4 @@ Diagnostic Scripts esmvaltool.diag_scripts.ocean esmvaltool.diag_scripts.psyplot_diag esmvaltool.diag_scripts.seaborn_diag + esmvaltool.diag_scripts.perfmetrics diff --git a/doc/sphinx/source/recipes/index.rst b/doc/sphinx/source/recipes/index.rst index edcc48977a..53ca7e08fb 100644 --- a/doc/sphinx/source/recipes/index.rst +++ b/doc/sphinx/source/recipes/index.rst @@ -69,6 +69,7 @@ Climate metrics :maxdepth: 1 recipe_perfmetrics + recipe_perfmetrics_python recipe_smpi Future projections diff --git a/doc/sphinx/source/recipes/recipe_perfmetrics.rst b/doc/sphinx/source/recipes/recipe_perfmetrics.rst index 067b65af85..25a32eb6d8 100644 --- a/doc/sphinx/source/recipes/recipe_perfmetrics.rst +++ b/doc/sphinx/source/recipes/recipe_perfmetrics.rst @@ -3,12 +3,26 @@ Performance metrics for essential climate parameters ==================================================== +.. note:: + + We are working on a reimplemenation of the + :ref:`performance metrics in python `. + Overview -------- -The goal is to create a standard recipe for the calculation of performance metrics to quantify the ability of the models to reproduce the climatological mean annual cycle for selected "Essential Climate Variables" (ECVs) plus some additional corresponding diagnostics and plots to better understand and interpret the results. +The goal is to create a standard recipe for the calculation of performance +metrics to quantify the ability of the models to reproduce the climatological +mean annual cycle for selected "Essential Climate Variables" (ECVs) plus some +additional corresponding diagnostics and plots to better understand and +interpret the results. + +The recipe can be used to calculate performance metrics at different vertical +levels (e.g., 5, 30, 200, 850 hPa as in +`Gleckler et al. (2008) `_) and in +different regions. As an additional reference, we consider +`Righi et al. (2015) `_. -The recipe can be used to calculate performance metrics at different vertical levels (e.g., 5, 30, 200, 850 hPa as in `Gleckler et al. (2008) `_ and in different regions. As an additional reference, we consider `Righi et al. (2015) `_. Available recipes and diagnostics ----------------------------------- @@ -21,12 +35,19 @@ Recipes are stored in recipes/ Diagnostics are stored in diag_scripts/perfmetrics/ -* main.ncl: calculates and (optionally) plots annual/seasonal cycles, zonal means, lat-lon fields and time-lat-lon fields. The calculated fields can also be plotted as difference w.r.t. a given reference dataset. main.ncl also calculates RMSD, bias and taylor metrics. Input data have to be regridded to a common grid in the preprocessor. Each plot type is created by a separated routine, as detailed below. +* main.ncl: calculates and (optionally) plots annual/seasonal cycles, zonal + means, lat-lon fields and time-lat-lon fields. The calculated fields can also + be plotted as difference w.r.t. a given reference dataset. main.ncl also + calculates RMSD, bias and taylor metrics. Input data have to be regridded to + a common grid in the preprocessor. Each plot type is created by a separated + routine, as detailed below. * cycle.ncl: creates an annual/seasonal cycle plot. * zonal.ncl: creates a zonal (lat-pressure) plot. * latlon.ncl: creates a lat-lon plot. -* cycle_latlon.ncl: precalculates the metrics for a time-lat-lon field, with different options for normalization. -* collect.ncl: collects and plots the metrics previously calculated by cycle_latlon.ncl. +* cycle_latlon.ncl: precalculates the metrics for a time-lat-lon field, with + different options for normalization. +* collect.ncl: collects and plots the metrics previously calculated by + cycle_latlon.ncl. User settings in recipe ----------------------- @@ -37,9 +58,12 @@ User settings in recipe *Required settings (scripts)* - * plot_type: cycle (time), zonal (plev, lat), latlon (lat, lon), cycle_latlon (time, lat, lon), cycle_zonal (time, plev, lat) + * plot_type: cycle (time), zonal (plev, lat), latlon (lat, lon), cycle_latlon + (time, lat, lon), cycle_zonal (time, plev, lat) * time_avg: type of time average (monthlyclim, seasonalclim, annualclim) - * region: selected region (global, trop, nhext, shext, nhtrop, shtrop, nh, sh, nhmidlat, shmidlat, nhpolar, shpolar, eq) + * region: selected region (global, trop, nhext, shext, nhtrop, shtrop, nh, + sh, nhmidlat, shmidlat, nhpolar, shpolar, eq) + *Optional settings (scripts)* @@ -51,9 +75,12 @@ User settings in recipe * projection: map projection for plot_type latlon (default: CylindricalEquidistant) * plot_diff: draws difference plots (default: False) * calc_grading: calculates grading metrics (default: False) - * stippling: uses stippling to mark statistically significant differences (default: False = mask out non-significant differences in gray) - * show_global_avg: diplays the global avaerage of the input field as string at the top-right of lat-lon plots (default: False) - * annots: choose the annotation style, e.g. ```alias``` which would display the alias of the dataset as title (applies to plot_type zonal and cycle_zonal) + * stippling: uses stippling to mark statistically significant differences + (default: False = mask out non-significant differences in gray) + * show_global_avg: displays the global avaerage of the input field as string + at the top-right of lat-lon plots (default: False) + * annots: choose the annotation style, e.g. ```alias``` which would display + the alias of the dataset as title (applies to plot_type zonal and cycle_zonal) * metric: chosen grading metric(s) (if calc_grading is True) * normalization: metric normalization (for RMSD and BIAS metrics only) * abs_levs: list of contour levels for absolute plot @@ -114,8 +141,8 @@ User settings in recipe *Optional settings (scripts)* - * label_lo: adds lower triange for values outside range - * label_hi: adds upper triange for values outside range + * label_lo: adds lower triangle for values outside range + * label_hi: adds upper triangle for values outside range * cm_interval: min and max color of the color table * cm_reverse: reverses the color table * sort: sorts datasets in alphabetic order (excluding MMM) @@ -157,14 +184,21 @@ Variables Observations and reformat scripts --------------------------------- -The following list shows the currently used observational data sets for this recipe with their variable names and the reference to their respective reformat scripts in parentheses. Please note that obs4MIPs data can be used directly without any reformating. For non-obs4MIPs data use `esmvaltool data info DATASET` or see headers of cmorization scripts (in `/esmvaltool/cmorizers/data/formatters/datasets/ -`_) for downloading and processing instructions. +The following list shows the currently used observational data sets for this +recipe with their variable names and the reference to their respective reformat +scripts in parentheses. Please note that obs4MIPs data can be used directly +without any reformatitng. For non-obs4MIPs data use `esmvaltool data info DATASET` +or see headers of cmorization scripts (in `/esmvaltool/cmorizers/data/formatters/datasets/ +`_) +for downloading and processing instructions. + #. recipe_perfmetrics_CMIP5.yml * AIRS (hus - obs4MIPs) * CERES-EBAF (rlut, rlutcs, rsut, rsutcs - obs4MIPs) * ERA-Interim (tas, ta, ua, va, zg, hus - esmvaltool/cmorizers/data/formatters/datasets/era-interim.py) - * ESACCI-AEROSOL (od550aer, od870aer, od550abs, od550lt1aer - esmvaltool/cmorizers/data/formatters/datasets/esacci-aerosol.ncl) + * ESACCI-AEROSOL (od550aer, od870aer, od550abs, od550lt1aer - + esmvaltool/cmorizers/data/formatters/datasets/esacci-aerosol.ncl) * ESACCI-CLOUD (clt - esmvaltool/cmorizers/data/formatters/datasets/esacci-cloud.ncl) * ESACCI-OZONE (toz - esmvaltool/cmorizers/data/formatters/datasets/esacci-ozone.ncl) * ESACCI-SOILMOISTURE (sm - esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.ncl) @@ -190,9 +224,13 @@ The following list shows the currently used observational data sets for this rec References ---------- -* Gleckler, P. J., K. E. Taylor, and C. Doutriaux, Performance metrics for climate models, J. Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). +* Gleckler, P. J., K. E. Taylor, and C. Doutriaux, Performance metrics for climate models, J. + Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). + +* Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., + and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, + Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). -* Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). Example plots ------------- @@ -200,17 +238,24 @@ Example plots .. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_1.png :width: 90% - Annual cycle of globally averaged temperature at 850 hPa (time period 1980-2005) for different CMIP5 models (historical simulation) (thin colored lines) in comparison to ERA-Interim (thick yellow line) and NCEP-NCAR-R1 (thick black dashed line) reanalysis data. + Annual cycle of globally averaged temperature at 850 hPa (time period 1980-2005) + for different CMIP5 models (historical simulation) (thin colored lines) in comparison to + ERA-Interim (thick yellow line) and NCEP-NCAR-R1 (thick black dashed line) reanalysis data. .. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_2.png :width: 90% - Taylor diagram of globally averaged temperature at 850 hPa (ta) and longwave cloud radiative effect (lwcre) for different CMIP5 models (historical simulation, 1980-2005). Reference data (REF) are ERA-Interim for temperature (1980-2005) and CERES-EBAF (2001-2012) for longwave cloud radiative effect. + Taylor diagram of globally averaged temperature at 850 hPa (ta) and longwave cloud + radiative effect (lwcre) for different CMIP5 models (historical simulation, 1980-2005). + Reference data (REF) are ERA-Interim for temperature (1980-2005) and CERES-EBAF (2001-2012) + for longwave cloud radiative effect. .. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_3.png :width: 90% - Difference in annual mean of zonally averaged temperature (time period 1980-2005) between the CMIP5 model MPI-ESM-MR (historical simulation) and ERA-Interim. Stippled areas indicdate differences that are statistically significant at a 95% confidence level. + Difference in annual mean of zonally averaged temperature (time period 1980-2005) between the + CMIP5 model MPI-ESM-MR (historical simulation) and ERA-Interim. Stippled areas indicdate + differences that are statistically significant at a 95% confidence level. .. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_4.png :width: 90% @@ -221,4 +266,9 @@ Example plots :width: 90% :align: center - Relative space-time root-mean-square deviation (RMSD) calculated from the climatological seasonal cycle of CMIP5 simulations. A relative performance is displayed, with blue shading indicating better and red shading indicating worse performance than the median of all model results. A diagonal split of a grid square shows the relative error with respect to the reference data set (lower right triangle) and the alternative data set (upper left triangle). White boxes are used when data are not available for a given model and variable. + Relative space-time root-mean-square deviation (RMSD) calculated from the climatological + seasonal cycle of CMIP5 simulations. A relative performance is displayed, with blue shading + indicating better and red shading indicating worse performance than the median of all model results. + A diagonal split of a grid square shows the relative error with respect to the reference data set + (lower right triangle) and the alternative data set (upper left triangle). + White boxes are used when data are not available for a given model and variable. diff --git a/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst b/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst new file mode 100644 index 0000000000..8f3f7f173d --- /dev/null +++ b/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst @@ -0,0 +1,146 @@ +.. _recipe_perfmetrics_python: + +Performance metrics for essential climate parameters in python +============================================================== + +.. note:: + + This recipe uses python diagnostics to reproduce parts of the evaluation + done in the + :ref:`original recipe based on NCL diagnostics `: + It aims for a complete replacement of all involved NCL diagnostics. So + far, only portrait plots (including performance metrics) are supported. + +Overview +-------- + +The goal is to create a standard recipe for the calculation of performance metrics to quantify the ability of the models to reproduce the climatological mean annual cycle for selected "Essential Climate Variables" (ECVs) plus some additional corresponding diagnostics and plots to better understand and interpret the results. + +The recipe can be used to calculate performance metrics at different vertical levels (e.g., 5, 30, 200, 850 hPa as in `Gleckler et al. (2008) `_) and in different regions. As an additional reference, we consider `Righi et al. (2015) `_. +Brief description of the diagnostic. + + +Available recipes and diagnostics +--------------------------------- + +Recipes are stored in esmvaltool/recipes/ + + * recipe_perfmetrics_python.yml + * recipe_perfmetrics_CMIP5_python.yml + +Diagnostics are stored in esmvaltool/diag_scripts/perfmetrics/ + + * portrait_plot.py: Plot metrics for any variable for multiple datasets and + up to four references. + + +User settings in recipe +----------------------- + +#. Script perfmetrics/portrait_plot.py + + This plot expects a scalar value in each input file and at most one input + file for each subset of metadata that belongs to a cell or part of cell in + the figure. + By default cells are plotted for combinations of `short_name`, + `dataset`, `project` and `split`. + Where `split` is an optional extra_facet for variables. + However, all this can be customized using the `x_by`, + `y_by`, `group_by` and `split_by` script settings. + For a complete and detailed list of settings see the + :ref:`API documentation `. + While this allows very flexible use for any kind of data, there are some + limitations as well: The grouping (separated + plots in the figure) and normalization is always applied along the x-axis. + With default settings this means normalizing all metrics for each variable + and grouping all datasets by project. + + To plot distance metrics like RMSE, pearson R, bias etc. the + :func:`distance_metrics ` preprocessor or + custom diagnostics can be used. + + + +Variables +--------- + +.. note:: + + The recipe generally works for any variable that is preprocessed correctly. + To use different preprocessors or reference datasets it could be useful + to create different variable groups and link them with the same extra_facet + like `variable_name` See recipe for examples. Below listed are the variables + needed to produce the example figures. + + +#. recipe_perfmetrics_CMIP5.yml + + * clt (atmos, monthly mean, longitude latitude time) + * hus (atmos, monthly mean, longitude latitude lev time) + * od550aer, od870aer, od550abs, od550lt1aer (aero, monthly mean, longitude latitude time) + * pr (atmos, monthly mean, longitude latitude time) + * rlut, rlutcs, rsut, rsutcs (atmos, monthly mean, longitude latitude time) + * sm (land, monthly mean, longitude latitude time) + * ta (atmos, monthly mean, longitude latitude lev time) + * tas (atmos, monthly mean, longitude latitude time) + * toz (atmos, monthly mean, longitude latitude time) + * ts (atmos, monthly mean, longitude latitude time) + * ua (atmos, monthly mean, longitude latitude lev time) + * va (atmos, monthly mean, longitude latitude lev time) + * zg (atmos, monthly mean, longitude latitude lev time) + + +Observations and reformat scripts +--------------------------------- + +*Note: (1) obs4MIPs data can be used directly without any preprocessing; +(2) see headers of reformat scripts for non-obs4MIPs data for download +instructions.* + +The following list shows the currently used observational data sets for this recipe with their variable names and the reference to their respective reformat scripts in parentheses. Please note that obs4MIPs data can be used directly without any reformatting. For non-obs4MIPs data use `esmvaltool data info DATASET` or see headers of cmorization scripts (in `/esmvaltool/cmorizers/data/formatters/datasets/ +`_) for downloading and processing instructions. + +#. recipe_perfmetrics_CMIP5.yml + + * AIRS (hus - obs4MIPs) + * CERES-EBAF (rlut, rlutcs, rsut, rsutcs - obs4MIPs) + * ERA-Interim (tas, ta, ua, va, zg, hus - esmvaltool/cmorizers/data/formatters/datasets/era-interim.py) + * ESACCI-AEROSOL (od550aer, od870aer, od550abs, od550lt1aer - esmvaltool/cmorizers/data/formatters/datasets/esacci-aerosol.ncl) + * ESACCI-CLOUD (clt - esmvaltool/cmorizers/data/formatters/datasets/esacci-cloud.ncl) + * ESACCI-OZONE (toz - esmvaltool/cmorizers/data/formatters/datasets/esacci-ozone.ncl) + * ESACCI-SOILMOISTURE (sm - esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.ncl) + * ESACCI-SST (ts - esmvaltool/ucmorizers/data/formatters/datasets/esacci-sst.py) + * GPCP-SG (pr - obs4MIPs) + * HadISST (ts - esmvaltool/cmorizers/data/formatters/datasets/hadisst.ncl) + * MODIS (od550aer - esmvaltool/cmorizers/data/formatters/datasets/modis.ncl) + * NCEP-NCAR-R1 (tas, ta, ua, va, zg - esmvaltool/cmorizers/data/formatters/datasets/ncep_ncar_r1.py) + * NIWA-BS (toz - esmvaltool/cmorizers/data/formatters/datasets/niwa_bs.ncl) + * PATMOS-x (clt - esmvaltool/cmorizers/data/formatters/datasets/patmos_x.ncl) + + +References +---------- + + +* Gleckler, P. J., K. E. Taylor, and C. Doutriaux, Performance metrics for climate models, J. + Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). + +* Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., + and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, + Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). + + +Example plots +------------- + +.. _fig_perfmetrics_python_portrait_plot: + +.. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_5_python.png + :width: 90% + :align: center + + + Relative space-time root-mean-square deviation (RMSD) calculated from the climatological + seasonal cycle of CMIP5 simulations. A relative performance is displayed, with blue shading + indicating better and red shading indicating worse performance than the median of all model results. + A diagonal split of a grid square shows the relative error with respect to the reference data set diff --git a/esmvaltool/diag_scripts/perfmetrics/main.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py similarity index 99% rename from esmvaltool/diag_scripts/perfmetrics/main.py rename to esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 98ace314fc..99060c4cf3 100644 --- a/esmvaltool/diag_scripts/perfmetrics/main.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -225,7 +225,7 @@ def split_legend(cfg, grid, data): size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) fig_size = fig.get_size_inches() # physical size of figure ax_size = (size / fig_size[0], size / fig_size[1]) # legend (fig coords) - gaps = (0.3 / fig_size[0], 0.3 / fig_size[1]) # margins (fig coords) + gaps = [0.3 / fig_size[0], 0.3 / fig_size[1]] # margins (fig coords) # anchor legend on origin of first plot or colorbar anchor = grid[0].get_position().bounds # relative figure coordinates if cfg["legend"].get("position", "right") == "right": diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml index d34f5144f2..1d044a2829 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_python.yml @@ -1,5 +1,5 @@ # ESMValTool -# +# --- documentation: title: Performance metrics plots. @@ -123,15 +123,15 @@ diagnostics: <<: *var_default scripts: perfmetrics: - script: perfmetrics/main.py + script: perfmetrics/portrait_plot.py y_by: variable_group complex: description: > - A more complex example with extra variables to support different reference datasets, + A more complex example with extra variables to support different reference datasets, groups for datasets and some plot customization. variables: - pr: + pr: <<: *var_default y_label: Precipitation pr_vs_access: &var_default_ref2 @@ -147,7 +147,8 @@ diagnostics: split: "ERA5" y_label: Precipitation additional_datasets: &ref_era - - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3, timerange: 1990/1992, reference_for_metric: true} + - {dataset: ERA5, project: native6, type: reanaly, version: v1, + tier: 3, timerange: 1990/1992, reference_for_metric: true} tas: <<: *var_default y_label: Temperature @@ -157,7 +158,7 @@ diagnostics: short_name: tas <<: *var_default_ref2 y_label: Temperature - rlut: + rlut: <<: *var_default y_label: "LW radiation out" @@ -174,21 +175,22 @@ diagnostics: short_name: clt split: "ESACCI" y_label: "Cloud cover" - additional_datasets: - - {reference_for_metric: true, dataset: ESACCI-CLOUD, project: OBS, type: sat, version: AVHRR-fv3.0, tier: 2, timerange: '1990/1992'} + additional_datasets: + - {reference_for_metric: true, dataset: ESACCI-CLOUD, project: OBS, + type: sat, version: AVHRR-fv3.0, tier: 2, timerange: '1990/1992'} psl: <<: *var_default y_label: "Sea level pressure" additional_datasets: - # - {dataset: ERA5, project: native6, type: reanaly, + # - {dataset: ERA5, project: native6, type: reanaly, # version: v1, tier: 3, reference_for_metric: true} - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} scripts: perfmetrics: - script: perfmetrics/main.py + script: perfmetrics/portrait_plot.py x_by: dataset y_by: y_label # split_by is the 'split' extra facet by default (set in variables) @@ -196,11 +198,11 @@ diagnostics: additional_datasets: - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} plot_kwargs: - vmin: 0.0 - vmax: 0.6 + vmin: 0.5 + vmax: 1.0 cbar_kwargs: label: "Pearson correlation coefficient" - ticks: [0, 0.3, 0.6] + ticks: [0.5, 0.6, 0.7, 0.8, 0.9, 1.0] extend: both cmap: "Reds" metrics: @@ -234,8 +236,8 @@ diagnostics: preprocessor: rmse scripts: perfmetrics: - script: perfmetrics/main.py + script: perfmetrics/portrait_plot.py y_by: y_label plot_kwargs: vmin: 0 - vmax: 1.0 \ No newline at end of file + vmax: 1.0 From ffee698e13441ac80004c6db398a618bcd897a46 Mon Sep 17 00:00:00 2001 From: Diego Cammarano Date: Wed, 3 Apr 2024 17:46:22 +0200 Subject: [PATCH 17/56] adding myself to list of authors --- esmvaltool/config-references.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esmvaltool/config-references.yml b/esmvaltool/config-references.yml index 285ace740e..76491b1611 100644 --- a/esmvaltool/config-references.yml +++ b/esmvaltool/config-references.yml @@ -24,6 +24,11 @@ authors: institute: DLR, Germany email: bjoern.broetz@dlr.de orcid: + cammarano_diego: + name: Cammarano, Diego + institute: DLR, Germany + email: diego.cammarano@dlr.de + github: diegokam debeire_kevin: name: Debeire, Kevin institute: DLR, Germany From 54ad3d8c7a9898f0e7d5b4cd16ffd3fa2e51d85e Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 5 Apr 2024 14:21:59 +0200 Subject: [PATCH 18/56] number of local vars, alphabetical sorting by x labels --- .../diag_scripts/perfmetrics/portrait_plot.py | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 99060c4cf3..585593f65a 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -220,10 +220,9 @@ def split_legend(cfg, grid, data): NOTE: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies this a bit by using `append_axes`. """ - fig = grid[0].get_figure() - fig.canvas.draw() # set axes position in figure (dont call tight_layout()) + grid[0].get_figure().canvas.draw() # set axes position in figure size = cfg["legend"].get("size", 0.5) # rect width in physical size (inch) - fig_size = fig.get_size_inches() # physical size of figure + fig_size = grid[0].get_figure().get_size_inches() # physical figure size ax_size = (size / fig_size[0], size / fig_size[1]) # legend (fig coords) gaps = [0.3 / fig_size[0], 0.3 / fig_size[1]] # margins (fig coords) # anchor legend on origin of first plot or colorbar @@ -237,32 +236,28 @@ def split_legend(cfg, grid, data): anchor = (anchor[0] - gaps[0] - ax_size[0] + cfg["legend"]["x_offset"], anchor[1] - gaps[1] - ax_size[1] + cfg["legend"]["y_offset"]) # create legend as empty imshow like axes in figure coordinates - legend = fig.add_axes([anchor[0], anchor[1], ax_size[0], ax_size[1]]) - legend.imshow(np.zeros((1, 1))) # same axes properties as main plot - legend.set_xticks([]) - legend.set_yticks([]) - twins = [legend.twiny(), legend.twinx()] - twins[1].set_yticks([]) - twins[0].set_xticks([]) - labels = data.coords[cfg["split_by"]].values + axes = {"main": grid[0].get_figure().add_axes([*anchor, *ax_size])} + axes["main"].imshow(np.zeros((1, 1))) # same axes properties as main plot + axes["main"].set_xticks([]) + axes["main"].set_yticks([]) + axes["twiny"], axes["twinx"] = [axes["main"].twiny(), axes["main"].twinx()] + axes["twinx"].set_yticks([]) + axes["twiny"].set_xticks([]) label_at = [ # order matches get_triangle_nodes (halves and quarters) - legend.set_ylabel, # left - twins[1].set_ylabel, # right - legend.set_xlabel, # bottom - twins[0].set_xlabel, # top + axes["main"].set_ylabel, # left + axes["twinx"].set_ylabel, # right + axes["main"].set_xlabel, # bottom + axes["twiny"].set_xlabel, # top ] - for i, label in enumerate(labels): - nodes = get_triangle_nodes(i, len(labels)) - colors = ["#bbb", "#ccc", "#ddd", "#eee"] - patch = patches.Polygon( - nodes, - closed=True, - facecolor=colors[i], - edgecolor="black", - linewidth=0.5, - fill=True, - ) - legend.add_patch(patch) + for i, label in enumerate(data.coords[cfg["split_by"]].values): + axes["main"].add_patch( + patches.Polygon(get_triangle_nodes( + i, len(data.coords[cfg["split_by"]].values)), + closed=True, + facecolor=["#bbb", "#ccc", "#ddd", "#eee"][i], + edgecolor="black", + linewidth=0.5, + fill=True)) label_at[i](label) @@ -290,7 +285,7 @@ def overlay_reference(cfg, axe, data, triangle): def plot_group(cfg, axe, data, title=None): - """Create matrix for one subplot in ax using plt.imshow() + """Create matrix for one subplot in ax using plt.imshow. by default split None is used, if all splits are named the first is used. Other splits will be added by overlaying triangles. @@ -311,7 +306,7 @@ def plot_group(cfg, axe, data, title=None): def get_triangle_nodes(position, total_count=2): - """Returns list of nodes with relative x, y coordinates. + """Return list of nodes with relative x, y coordinates. The nodes of the triangle are given as list of three tuples. Each tuples contains relative coordinates (-0.5 to +0.5). For total of <= 2 a top left @@ -357,9 +352,11 @@ def plot_overlays(cfg, grid, data): def plot(cfg, data): - """Creates figure with subplots for each group, sets same color range and - overlays additional references based on the content of data - (xr.DataArray)""" + """Create figure with subplots for each group. + + sets same color range and overlays additional references based on + the content of data (xr.DataArray) + """ fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) grid = ImageGrid( @@ -440,6 +437,7 @@ def main(cfg): remove_reference(metas) add_split_none(metas) dataset = load_data(cfg, metas) + dataset = dataset.sortby(dataset[cfg["x_by"]].str.lower()) if cfg["normalize"] is not None: dataset["var"] = normalize(dataset["var"], cfg["normalize"], [cfg["x_by"], cfg["group_by"]]) From 7cbcf86f28eb9670346f7f3bd0bedb9b08e70a19 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 16 Apr 2024 12:18:39 +0200 Subject: [PATCH 19/56] add dpi option and plot legend only for multiple splits --- .../diag_scripts/perfmetrics/portrait_plot.py | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 585593f65a..94e5e94cbc 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -44,6 +44,9 @@ splits will be plotted as overlays. This can be used to choose the base reference, while all references are labeled for the legend. By default None. +plot_legend: bool, optional + If True, a legend is plotted, when multiple splits are given. + By default True. legend: dict, optional Customize, if, how and where the legend is plotted. The 'best' position and size of the legend depends on multiple parameters of the figure @@ -86,6 +89,8 @@ [width, height] of the figure in inches. The final figure will be saved with bbox_inches="tight", which can change the resulting aspect ratio. By default [5, 3]. +dpi: int, optional + Dots per inch for the figure. By default 300. """ import itertools @@ -385,11 +390,11 @@ def plot(cfg, data): grid.cbar_axes[0].colorbar(grid[0].get_images()[0], **cfg["cbar_kwargs"]) if data.shape[3] > 1: plot_overlays(cfg, grid, data) - if cfg.get("plot_legend", True): + if cfg["plot_legend"] and data.shape[3] > 1: split_legend(cfg, grid, data) - basename = "performance_metrics" + basename = "portrait_plot" fname = get_plot_filename(basename, cfg) - plt.savefig(fname, bbox_inches="tight") + plt.savefig(fname, bbox_inches="tight", dpi=cfg["dpi"]) log.info("Figure saved:") log.info(fname) @@ -419,17 +424,39 @@ def set_defaults(cfg): cfg.setdefault("cbar_kwargs", {}) cfg.setdefault("axes_properties", {}) cfg.setdefault("nan_color", 'white') + cfg.setdefault("figsize", (7.5, 3.5)) + cfg.setdefault("dpi", 300) + cfg.setdefault("plot_legend", True) cfg.setdefault("plot_kwargs", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") cfg["plot_kwargs"].setdefault("vmin", 0) cfg["plot_kwargs"].setdefault("vmax", 1) - cfg.setdefault("figsize", (7.5, 3.5)) cfg.setdefault("legend", {}) cfg["legend"].setdefault("x_offset", 0) cfg["legend"].setdefault("y_offset", 0) cfg["legend"].setdefault("size", 0.3) +def sort_data(cfg, dataset): + """Sort the dataset along by custom or alphabetical order.""" + # custom order: dsimport xarray as xr + # import pandas as pd + # order = ['value3', 'value1', 'value2'] # replace by custom order + # ds[cfg['y_by']] = pd.Categorical(ds[cfg['y_by']], categories=order, + # ordered=True) + # ds = ds.sortby('y_by') + # sort alphabetically (caseinsensitive) + dataset = dataset.sortby([ + dataset[cfg["x_by"]].str.lower(), dataset[cfg["y_by"]].str.lower(), + dataset[cfg["group_by"]].str.lower(), + dataset[cfg["split_by"]].str.lower() + ]) + # apply custom orders if given: + # if cfg.get("x_order"): + # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) + return dataset + + def main(cfg): """Run the diagnostic.""" set_defaults(cfg) @@ -437,11 +464,10 @@ def main(cfg): remove_reference(metas) add_split_none(metas) dataset = load_data(cfg, metas) - dataset = dataset.sortby(dataset[cfg["x_by"]].str.lower()) + dataset = sort_data(cfg, dataset) if cfg["normalize"] is not None: dataset["var"] = normalize(dataset["var"], cfg["normalize"], [cfg["x_by"], cfg["group_by"]]) - plot(cfg, dataset["var"]) From f7b6c87c2d001158a85ed571dce8858c31dc9521 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 16 Apr 2024 12:20:24 +0200 Subject: [PATCH 20/56] WIP recreate example perfmetric recipe --- .../recipe_perfmetrics_CMIP5_python.yml | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml new file mode 100644 index 0000000000..6ad21ee5f5 --- /dev/null +++ b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml @@ -0,0 +1,276 @@ +# ESMValTool +# recipe_perfmetrics_CMIP5.yml +--- +documentation: + title: Performance metrics for essential climate variables in CMIP5 + + description: | + Recipe for plotting the performance metrics for the CMIP5 datasets, + including the standard ECVs as in Gleckler et al., and some additional + variables (like ozone, sea-ice, aerosol...) + + authors: + - winterstein_franziska + - righi_mattia + - eyring_veronika + - ruhe_lukas + + maintainer: + - ruhe_lukas + + references: + - gleckler08jgr + + projects: + - esmval + - embrace + - crescendo + - c3s-magic + - cmug + +preprocessors: + ppNOLEV1: + regrid: + target_grid: reference_dataset + scheme: linear + mask_fillvalues: + threshold_fraction: 0.95 + # multi_model_statistics: + # span: overlap + # statistics: [mean, median] + # exclude: [reference_dataset] + + rmse: &rmse + custom_order: true + regrid: + # target_grid: reference_dataset + scheme: linear + target_grid: 3x3 + # scheme: nearest + regrid_time: + calendar: standard + frequency: mon + mask_fillvalues: + threshold_fraction: 0.95 + # multi_model_statistics: + # span: overlap + # statistics: [mean, median] + # exclude: [reference_dataset, alternative_dataset] + distance_metric: + metric: rmse + + pp500: + # <<: *rmse + custom_order: true + regrid: + target_grid: 3x3 + scheme: linear + extract_levels: + levels: 50000 + scheme: linear + regrid_time: + calendar: standard + frequency: mon + mask_fillvalues: + threshold_fraction: 0.95 + distance_metric: + metric: rmse + +diagnostics: + perfmetrics: + description: Near-surface air temperature + themes: + - phys + realms: + - atmos + variables: + tas: &var_default + preprocessor: rmse + reference_dataset: ERA-Interim + # alternative_dataset: NCEP-NCAR-R1 + mip: Amon + split: Ref + y_label: tas_Glob + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + tas_alt: + <<: *var_default + short_name: tas + y_label: tas_Glob + split: Alt + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + pr: + <<: *var_default + y_label: pr_Glob + reference_dataset: GPCP-V2.2 + additional_datasets: + - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} + + # swcre: + # <<: *var_default + # derive: true + # force_derivation: false + # y_label: swcre_Glob + # additional_datasets: + # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, + # tier: 1, reference_for_metric: true} + + # rlut: + # <<: *var_default + # y_label: rlut_Glob + # additional_datasets: + # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, + # tier: 1, reference_for_metric: true, start_year: 2000, end_year: 2002} 34 vs 36 month dataa.. 2 months missing in obs?? + + zg: + <<: *var_default + y_label: zg_Glob-500 + preprocessor: pp500 + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + # - {dataset: CNRM-CM5-2} # not in example plot + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + # - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + # - {dataset: GISS-E2-H-CC} # not in example plot + - {dataset: GISS-E2-R, ensemble: r1i1p2} + # - {dataset: GISS-E2-R-CC} # not in example plot + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + # - {dataset: MPI-ESM-MR} # not in example plot + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + # - {dataset: ERA-Interim, project: OBS6, type: reanaly, + # version: 1, tier: 3, reference_for_metric: true} + # - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + # version: 1, tier: 2} + scripts: + portrait: + script: perfmetrics/portrait_plot.py + y_by: y_label + x_by: dataset + plot_kwargs: + vmin: -0.5 + vmax: 0.5 + normalize: centered_median # default + default_split: Alt + + ### pr: PRECIPITATION ####################################################### + # pr: + # description: Precipitation + # themes: + # - phys + # realms: + # - atmos + # variables: + # pr: + # preprocessor: ppNOLEV1 + # reference_dataset: GPCP-V2.2 + # mip: Amon + # project: CMIP5 + # exp: historical + # ensemble: r1i1p1 + # start_year: 2000 + # end_year: 2002 + # additional_datasets: + # - {dataset: ACCESS1-0} + # - {dataset: ACCESS1-3} + # - {dataset: bcc-csm1-1} + # - {dataset: bcc-csm1-1-m} + # - {dataset: BNU-ESM} + # - {dataset: CanCM4} + # - {dataset: CanESM2} + # - {dataset: CCSM4} + # - {dataset: CESM1-BGC} + # - {dataset: CESM1-CAM5} + # - {dataset: CESM1-CAM5-1-FV2} + # - {dataset: CESM1-FASTCHEM} + # - {dataset: CESM1-WACCM} + # - {dataset: CMCC-CESM} + # - {dataset: CMCC-CM} + # - {dataset: CMCC-CMS} + # - {dataset: CNRM-CM5} + # - {dataset: CNRM-CM5-2} + # - {dataset: CSIRO-Mk3-6-0} + # - {dataset: EC-EARTH, ensemble: r6i1p1} + # - {dataset: FGOALS-g2} + # - {dataset: FIO-ESM} + # - {dataset: GFDL-CM2p1} + # - {dataset: GFDL-CM3} + # - {dataset: GFDL-ESM2G} + # - {dataset: GFDL-ESM2M} + # - {dataset: GISS-E2-H, ensemble: r1i1p2} + # - {dataset: GISS-E2-H-CC} + # - {dataset: GISS-E2-R, ensemble: r1i1p2} + # - {dataset: GISS-E2-R-CC} + # - {dataset: HadCM3} + # - {dataset: HadGEM2-AO} + # - {dataset: HadGEM2-CC} + # - {dataset: HadGEM2-ES} + # - {dataset: inmcm4} + # - {dataset: IPSL-CM5A-LR} + # - {dataset: IPSL-CM5A-MR} + # - {dataset: IPSL-CM5B-LR} + # - {dataset: MIROC4h} + # - {dataset: MIROC5} + # - {dataset: MIROC-ESM} + # - {dataset: MIROC-ESM-CHEM} + # - {dataset: MPI-ESM-LR} + # - {dataset: MPI-ESM-MR} + # - {dataset: MPI-ESM-P} + # - {dataset: MRI-CGCM3} + # - {dataset: MRI-ESM1} + # - {dataset: NorESM1-M} + # - {dataset: NorESM1-ME} + # - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1} + # scripts: + # grading: + # <<: *grading_settings From 39df74e713f5f9bb2c244c5e7c4b24dbf3c85453 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 2 May 2024 11:49:49 +0200 Subject: [PATCH 21/56] add optional calculation of distance metrics --- .../diag_scripts/perfmetrics/portrait_plot.py | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 94e5e94cbc..2215099e0e 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -19,7 +19,11 @@ normalize: str or None, optional ('mean', 'median', 'centered_mean', 'centered_median', None). Subtract median/mean if centered. Divide by median/mean if not None. - By default None. + By default 'centered_median'. +distance_metric: str or None, optional + A method for the distance_metric preprocessor can be set, to apply it to + the input data along all axis before plotting. If set to None, the input + is expected to contain scalar values for each input file. By default, None. x_by: str, optional Metadata key for x coordinate. By default 'alias'. @@ -67,8 +71,8 @@ plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. Colormaps will be converted to 11 discrete steps automatically. Default - colormap is RdYlBu_r but can be changed with cmap. - Other common keywords: vmin, vmax + colormap RdYlBu_r and limits vmin=-0.5, vmax=0.5 can be changed using + keywords like: cmap, vmin, vmax. By default {}. cbar_kwargs: dict, optional Dictionary that gets passed to `matplotlib.pyplot.colorbar()`. @@ -95,15 +99,19 @@ import itertools import logging +from pathlib import Path +import iris import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import xarray as xr +from esmvalcore import preprocessor as pp from matplotlib import patches from mpl_toolkits.axes_grid1 import ImageGrid from esmvaltool.diag_scripts.shared import ( + get_diagnostic_filename, get_plot_filename, group_metadata, run_diagnostic, @@ -413,6 +421,35 @@ def normalize(array, method, dims): return normalized +def apply_distance_metric(cfg, metas): + """Optionally apply preproc method. + + reference_for_metric facet required. + """ + if not cfg["distance_metric"]: + return + for y_metas in group_metadata(metas, cfg["y_by"]).values(): + try: # TODO: add select_single_metadata to shared? + reference = select_metadata(y_metas, reference_for_metric=True)[0] + except IndexError as exc: + raise IndexError("No reference found for metric.") from exc + ref_cube = iris.load_cube(reference["filename"]) + for meta in y_metas: + if meta.get("reference_for_metric", False): + continue # skip distance to itself + cube = iris.load_cube(meta["filename"]) + distance = pp.distance_metric([cube], + reference=ref_cube, + metric=cfg["distance_metric"]) + basename = f"{Path(meta['filename']).stem}" + basename += f"{cfg['distance_metric']}" + fname = get_diagnostic_filename(basename, cfg) + iris.save(distance, fname) + log.info("Distance metric saved: %s", fname) + # TODO: adjust all relevant meta data + meta["filename"] = fname + + def set_defaults(cfg): """Set default values for most important config parameters.""" cfg.setdefault("normalize", "centered_median") @@ -429,8 +466,8 @@ def set_defaults(cfg): cfg.setdefault("plot_legend", True) cfg.setdefault("plot_kwargs", {}) cfg["plot_kwargs"].setdefault("cmap", "RdYlBu_r") - cfg["plot_kwargs"].setdefault("vmin", 0) - cfg["plot_kwargs"].setdefault("vmax", 1) + cfg["plot_kwargs"].setdefault("vmin", -0.5) + cfg["plot_kwargs"].setdefault("vmax", 0.5) cfg.setdefault("legend", {}) cfg["legend"].setdefault("x_offset", 0) cfg["legend"].setdefault("y_offset", 0) From a896ce60d62fe4cc8fcfd5dd2a111e8b9ee3e1e0 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 2 May 2024 12:35:04 +0200 Subject: [PATCH 22/56] fix bug for non default "split" facet --- esmvaltool/diag_scripts/perfmetrics/portrait_plot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 2215099e0e..c9f67e0cd6 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -165,11 +165,11 @@ def remove_reference(metas): metas.remove(meta) -def add_split_none(metas): +def add_split_none(cfg, metas): """List of metadata with split=None if no split is given.""" for meta in metas: - if "split" not in meta: - meta["split"] = None + if cfg["split_by"] not in meta: + meta[cfg["split_by"]] = None def open_file(metadata, **selection): @@ -499,7 +499,7 @@ def main(cfg): set_defaults(cfg) metas = list(cfg["input_data"].values()) remove_reference(metas) - add_split_none(metas) + add_split_none(cfg, metas) dataset = load_data(cfg, metas) dataset = sort_data(cfg, dataset) if cfg["normalize"] is not None: From 62086aae601856f5badb2eb00fdb3e07cd51d157 Mon Sep 17 00:00:00 2001 From: Diego Cammarano Date: Mon, 4 Nov 2024 16:22:07 +0100 Subject: [PATCH 23/56] recipe CMIP5 python, new python script creating netcdf file --- .../diag_scripts/perfmetrics/portrait_plot.py | 15 +- .../recipes/recipe_perfmetrics_CMIP5_py.yml | 2110 +++++++++++++++++ 2 files changed, 2124 insertions(+), 1 deletion(-) create mode 100644 esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index c9f67e0cd6..b67c9e3e7a 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -365,11 +365,14 @@ def plot_overlays(cfg, grid, data): def plot(cfg, data): - """Create figure with subplots for each group. + """Create figure with subplots for each group and save to NetCDF. sets same color range and overlays additional references based on the content of data (xr.DataArray) """ + # Save the dataset to NetCDF before plotting + save_to_netcdf(cfg, data) + fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) grid = ImageGrid( @@ -493,6 +496,16 @@ def sort_data(cfg, dataset): # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) return dataset +def save_to_netcdf(cfg, data): + """Save the final dataset to a NetCDF file.""" + # Define the output filename for the NetCDF file + basename = "performance_metrics" + fname = get_diagnostic_filename(basename, cfg, extension='nc') + + # Save the dataset to a NetCDF file + data.to_netcdf(fname) + log.info("NetCDF file saved:") + log.info(fname) def main(cfg): """Run the diagnostic.""" diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml new file mode 100644 index 0000000000..d8c5836432 --- /dev/null +++ b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml @@ -0,0 +1,2110 @@ +# ESMValTool +# +--- +documentation: + title: Performance metrics plots. + description: > + Compare performance of ICON simulations to a reference dataset. + authors: + - ruhe_lukas + maintainer: + - ruhe_lukas + references: + - eyring21ipcc + +preprocessors: + pp200_3x3: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + extract_levels: + levels: 19900 + scheme: linear + coordinate: air_pressure + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ERA-Interim, NCEP-NCAR-R1] + pp850_3x3: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + extract_levels: + levels: 85500 + scheme: linear + coordinate: air_pressure + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ERA-Interim, NCEP-NCAR-R1] + pp400_3x3: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + extract_levels: + levels: 85500 + scheme: linear + coordinate: air_pressure + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [AIRS-2-1, ERA-Interim] + pp500: + regrid: + target_grid: 3x3 + scheme: linear + extract_levels: + levels: 50000 + scheme: linear + coordinate: air_pressure + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ERA-Interim, NCEP-NCAR-R1] + ppNOLEV1: + regrid: + target_grid: 3x3 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.95 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [CERES-EBAF] + ppNOLEV1_3x3: + regrid: + target_grid: 3x3 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.95 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [GPCP-V2.2] + ppNOLEV1thr10: + regrid: + target_grid: 3x3 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.10 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-SOILMOISTURE] + ppNOLEV1thr10_1: + regrid: + target_grid: 3x3 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.10 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-AEROSOL] + ppNOLEV2thr10: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.10 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-OZONE, NIWA-BS] + ppNOLEV2thr10_1: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.10 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-AEROSOL, MODIS] + ppNOLEV2thr10_shpolar: + regrid: + target_grid: 3x3 + scheme: linear + extract_region: + start_longitude: 0 + start_latitude: -90 + end_longitude: 360 + end_latitude: -60 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.10 + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-OZONE, NIWA-BS] + ppNOLEV2: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + mask_fillvalues: + threshold_fraction: 0.95 + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [PATMOS-x, ESACCI-CLOUD] + ppNOLEV2_1: + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + mask_fillvalues: + threshold_fraction: 0.95 + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ERA-Interim, NCEP-NCAR-R1] + ppNOLEV1x1: + regrid: + target_grid: 1x1 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.95 + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + multi_model_statistics: + span: overlap + statistics: [mean, median] + exclude: [ESACCI-SST, HadISST] + +diagnostics: + perfmetrics_rmse: # seasonal normalized rmse + variables: + zg: + variable: zg500 + preprocessor: pp500 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + zg_2: + variable: zg500 + short_name: zg + preprocessor: pp500 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref2 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + sm: + variable: sm + preprocessor: ppNOLEV1thr10 + mip: Lmon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2002 + end_year: 2004 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: HadCM3} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-SOILMOISTURE, project: OBS, type: sat, + version: L3S-SSMV-COMBINED-v4.2, tier: 2, reference_for_metric: true} + clt: + variable: clt + preprocessor: ppNOLEV2 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-CLOUD, project: OBS, type: sat, + version: AVHRR-AMPM-fv3.0, tier: 2, reference_for_metric: true} + clt_2: + variable: clt + short_name: clt + preprocessor: ppNOLEV2 + mip: Amon + split: Ref2 + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, reference_for_metric: true} + ts: + variable: ts + preprocessor: ppNOLEV1x1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-SST, project: OBS, type: sat, + version: 2.2, tier: 2, reference_for_metric: true} + ts_2: + variable: ts + short_name: ts + preprocessor: ppNOLEV1x1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref2 + start_year: 2000 + end_year: 2002 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: HadISST, project: OBS, type: reanaly, version: 1, tier: 2, reference_for_metric: true} + toz: + variable: toz + preprocessor: ppNOLEV2thr10 + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2002 + end_year: 2004 + additional_datasets: + - {dataset: CESM1-WACCM, ensemble: r2i1p1} + - {dataset: CNRM-CM5} + - {dataset: GFDL-CM3} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: MIROC-ESM-CHEM} + - {dataset: ESACCI-OZONE, project: OBS, type: sat, version: L3, tier: 2, reference_for_metric: true} + toz_2: + variable: toz + short_name: toz + preprocessor: ppNOLEV2thr10 + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref2 + start_year: 2002 + end_year: 2004 + additional_datasets: + - {dataset: CESM1-WACCM, ensemble: r2i1p1} + - {dataset: CNRM-CM5} + - {dataset: GFDL-CM3} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: MIROC-ESM-CHEM} + - {dataset: NIWA-BS, project: OBS, type: sat, version: v3.3, tier: 3, reference_for_metric: true} + toz_shpolar: + variable: toz_shpolar + short_name: toz + preprocessor: ppNOLEV2thr10_shpolar + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref1 + start_year: 2002 + end_year: 2004 + additional_datasets: + - {dataset: CESM1-WACCM, ensemble: r2i1p1} + - {dataset: CNRM-CM5} + - {dataset: GFDL-CM3} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: MIROC-ESM-CHEM} + - {dataset: ESACCI-OZONE, project: OBS, type: sat, version: L3, tier: 2, reference_for_metric: true} + toz_2_shpolar: + variable: toz_shpolar + short_name: toz + preprocessor: ppNOLEV2thr10_shpolar + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + split: Ref2 + start_year: 2002 + end_year: 2004 + additional_datasets: + - {dataset: CESM1-WACCM, ensemble: r2i1p1} + - {dataset: CNRM-CM5} + - {dataset: GFDL-CM3} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: MIROC-ESM-CHEM} + - {dataset: NIWA-BS, project: OBS, type: sat, version: v3.3, tier: 3, reference_for_metric: true} + + tas: + variable: tas + preprocessor: ppNOLEV2_1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + # - {dataset: CESM1-CAM5-1-FV2} # data is missing on ESGF + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + tas_2: + variable: tas + short_name: tas + preprocessor: ppNOLEV2_1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + # - {dataset: CESM1-CAM5-1-FV2} # data is missing on ESGF + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + lwcre: + variable: lwcre + preprocessor: ppNOLEV1 + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2001 + end_year: 2003 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} + pr: + variable: pr + preprocessor: ppNOLEV1_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true } + rlut: + variable: rlut + preprocessor: ppNOLEV1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2001 + end_year: 2003 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: CERES-EBAF, project: OBS, type: sat, version: Ed4.2, + tier: 2, reference_for_metric: true} + rsut: + variable: rsut + preprocessor: ppNOLEV1 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2001 + end_year: 2003 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} + swcre: + variable: swcre + preprocessor: ppNOLEV1 + mip: Amon + derive: true + force_derivation: false + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2001 + end_year: 2003 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} + hus: + variable: hus400 + preprocessor: pp400_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2003 + end_year: 2004 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R-CC} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: AIRS-2-1, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} + hus_2: + variable: hus400 + short_name: hus + preprocessor: pp400_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2003 + end_year: 2004 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R-CC} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, + type: reanaly, version: 1, tier: 3, reference_for_metric: true} + ua200: + variable: ua200 + short_name: ua + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ua200_2: + variable: ua200 + short_name: ua + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + ua850: + variable: ua850 + short_name: ua + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ua850_2: + variable: ua850 + short_name: ua + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + ta200: + variable: ta200 + short_name: ta + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ta200_2: + variable: ta200 + short_name: ta + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + ta850: + variable: ta850 + short_name: ta + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ta850_2: + variable: ta850 + short_name: ta + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-CAM5-1-FV2} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FGOALS-s2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + va200: + variable: va200 + short_name: va + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + va200_2: + variable: va200 + short_name: va + preprocessor: pp200_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + va850: + variable: va850 + short_name: va + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + va850_2: + variable: va850 + short_name: va + preprocessor: pp850_3x3 + mip: Amon + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2000 + end_year: 2002 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: bcc-csm1-1} + - {dataset: bcc-csm1-1-m} + - {dataset: BNU-ESM} + - {dataset: CanCM4} + - {dataset: CanESM2} + - {dataset: CCSM4} + - {dataset: CESM1-BGC} + - {dataset: CESM1-CAM5} + - {dataset: CESM1-FASTCHEM} + - {dataset: CESM1-WACCM} + - {dataset: CMCC-CESM} + - {dataset: CMCC-CM} + - {dataset: CMCC-CMS} + - {dataset: CNRM-CM5} + - {dataset: CNRM-CM5-2} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: EC-EARTH, ensemble: r6i1p1} + - {dataset: FGOALS-g2} + - {dataset: FIO-ESM} + - {dataset: GFDL-CM2p1} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-H-CC} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: GISS-E2-R-CC} + - {dataset: HadCM3} + - {dataset: HadGEM2-AO} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: inmcm4} + - {dataset: IPSL-CM5A-LR} + - {dataset: IPSL-CM5A-MR} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MPI-ESM-LR} + - {dataset: MPI-ESM-MR} + - {dataset: MPI-ESM-P} + - {dataset: MRI-CGCM3} + - {dataset: MRI-ESM1} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + abs550aer: + variable: abs550aer + preprocessor: ppNOLEV1thr10_1 + mip: aero + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2002 + end_year: 2004 + split: Ref1 + additional_datasets: + - {dataset: CSIRO-Mk3-6-0} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, + version: SU-v4.21, tier: 2, reference_for_metric: true} + od870aer: + variable: od870aer + preprocessor: ppNOLEV1thr10_1 + mip: aero + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2002 + end_year: 2004 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MRI-CGCM3} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, + version: SU-v4.21, tier: 2, reference_for_metric: true} + od550aer: + variable: od550aer + preprocessor: ppNOLEV2thr10_1 + mip: aero + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2003 + end_year: 2004 + split: Ref1 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: BNU-ESM} + - {dataset: CESM1-CAM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, + version: SU-v4.21, tier: 2, reference_for_metric: true} + od550aer_2: + variable: od550aer + short_name: od550aer + preprocessor: ppNOLEV2thr10_1 + mip: aero + project: CMIP5 + exp: historical + ensemble: r1i1p1 + start_year: 2003 + end_year: 2004 + split: Ref2 + additional_datasets: + - {dataset: ACCESS1-0} + - {dataset: ACCESS1-3} + - {dataset: BNU-ESM} + - {dataset: CESM1-CAM5} + - {dataset: CSIRO-Mk3-6-0} + - {dataset: GFDL-CM3} + - {dataset: GFDL-ESM2G} + - {dataset: GFDL-ESM2M} + - {dataset: GISS-E2-H, ensemble: r1i1p2} + - {dataset: GISS-E2-R, ensemble: r1i1p2} + - {dataset: HadGEM2-CC} + - {dataset: HadGEM2-ES} + - {dataset: IPSL-CM5B-LR} + - {dataset: MIROC4h} + - {dataset: MIROC5} + - {dataset: MIROC-ESM} + - {dataset: MIROC-ESM-CHEM} + - {dataset: MRI-CGCM3} + - {dataset: NorESM1-M} + - {dataset: NorESM1-ME} + - {dataset: MODIS, project: OBS, type: sat, version: MYD08-M3, tier: 3, reference_for_metric: true} + + scripts: + portrait: + script: perfmetrics/portrait_plot.py + x_by: alias + y_by: variable # use extra_facet since group and short_name don't work + group_by: project + normalize: "centered_median" + default_split: Ref1 + nan_color: null + plot_kwargs: + vmin: -0.5 + vmax: +0.5 + cbar_kwargs: + label: Relative RMSE + extend: both \ No newline at end of file From 8c9aa90b7142b50d0faeb2ba85685a6dd0a9325d Mon Sep 17 00:00:00 2001 From: Diego Cammarano Date: Mon, 4 Nov 2024 16:25:14 +0100 Subject: [PATCH 24/56] comparison script NetCDF files --- .../perfmetrics/compare_netCDFs.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py diff --git a/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py b/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py new file mode 100644 index 0000000000..a3a6ff1020 --- /dev/null +++ b/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py @@ -0,0 +1,80 @@ + + + + +import xarray as xr +import numpy as np +import pandas as pd + + +# Load the NetCDF files +file_diagnostics_path = '/home/b/b309265/ta850-global_to_sm-global_RMSD.nc' +file_performance_path = '/home/b/b309265/performance_metrics.nc' + +# Load datasets +ds_diagnostics = xr.open_dataset(file_diagnostics_path) +ds_performance = xr.open_dataset(file_performance_path) + +# Strip the "CMIP5_" prefix from model names in the performance dataset +ds_performance['alias'] = [model.replace("CMIP5_", "").strip() for model in ds_performance['alias'].values] + +# Convert all model names and strip whitespace for consistency +models_diagnostics = ds_diagnostics['models'].astype(str).str.strip() +models_performance = ds_performance['alias'].astype(str).str.strip() + +# Display unique variables from each dataset +variables_diagnostics = set(ds_diagnostics['diagnostics'].astype(str).values) +variables_performance = set(ds_performance['variable'].astype(str).values) + +print("Variables in Diagnostics Dataset:", variables_diagnostics) +print("Variables in Performance Dataset:", variables_performance) + +# Standardize variable names by converting to lowercase and stripping whitespace +variables_diagnostics = {var.lower().strip() for var in variables_diagnostics} +variables_performance = {var.lower().strip() for var in variables_performance} + +# Find common models and standardized variables +common_models = np.intersect1d(models_diagnostics, models_performance) +common_variables = variables_diagnostics.intersection(variables_performance) + +print("Common Models:", common_models) +print("Common Variables after Standardization:", common_variables) + +# Initialize a dictionary to collect comparison results +comparison_results = { + "Model": [], + "Variable": [], + "Grade_Value": [], + "Var_Value": [], + "Difference": [] +} + +# Perform comparison for each common model-variable pair +for model in common_models: + for variable in common_variables: + try: + # Select values for each matched model-variable pair + # Align 'diagnostics' and 'variable' names by standardizing to lowercase + grade_val = ds_diagnostics.grade.sel(models=model, diagnostics=variable).values + var_val = ds_performance.var.sel(alias=model, variable=variable).values + + # Calculate the absolute difference + diff = np.abs(grade_val - var_val).flatten() + + # Store results + comparison_results["Model"].append(model) + comparison_results["Variable"].append(variable) + comparison_results["Grade_Value"].append(grade_val) + comparison_results["Var_Value"].append(var_val) + comparison_results["Difference"].append(diff) + + except KeyError: + # Skip if the pair is missing in one dataset + print(f"Skipped {model}-{variable} due to mismatch in one of the datasets") + +# Convert to DataFrame and check results +comparison_df = pd.DataFrame(comparison_results) +print("Comparison Results:\n", comparison_df) + +# Optionally, save results to CSV +# comparison_df.to_csv("comparison_results.csv", index=False) From 9b20ac3558ec923473ee054749ba15646538e40f Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 5 Nov 2024 17:07:36 +0100 Subject: [PATCH 25/56] fixed pp heights, name changed, add provenance --- esmvaltool/config-references.yml | 4 +-- .../diag_scripts/perfmetrics/portrait_plot.py | 30 +++++++++++++++---- .../recipes/recipe_perfmetrics_CMIP5_py.yml | 22 ++++++++------ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/esmvaltool/config-references.yml b/esmvaltool/config-references.yml index 76491b1611..732300a384 100644 --- a/esmvaltool/config-references.yml +++ b/esmvaltool/config-references.yml @@ -462,8 +462,8 @@ authors: rol_evert: name: Rol, Evert orcid: https://orcid.org/0000-0001-8357-4453 - ruhe_lukas: - name: Ruhe, Lukas + lindenlaub_lukas: + name: Lindenlaub, Lukas institute: University of Bremen, Germany orcid: https://orcid.org/0000-0001-6349-9118 github: lukruh diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index b67c9e3e7a..16036bae43 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -95,6 +95,8 @@ By default [5, 3]. dpi: int, optional Dots per inch for the figure. By default 300. +domain: str, optional + Domain for provenance. By default 'global'. """ import itertools @@ -111,6 +113,7 @@ from mpl_toolkits.axes_grid1 import ImageGrid from esmvaltool.diag_scripts.shared import ( + ProvenanceLogger, get_diagnostic_filename, get_plot_filename, group_metadata, @@ -184,7 +187,7 @@ def open_file(metadata, **selection): if len(metas) < 1: log.debug("No Metadata found for %s", selection) return np.nan - log.warning("Metadata found for %s", selection) + log.debug("Metadata found for %s", selection) das = xr.open_dataset(metas[0]["filename"]) varname = list(das.data_vars.keys())[0] return das[varname].values.item() @@ -371,7 +374,18 @@ def plot(cfg, data): the content of data (xr.DataArray) """ # Save the dataset to NetCDF before plotting - save_to_netcdf(cfg, data) + provenance = { + 'ancestors': list(cfg["input_data"].keys()), + 'authors': ["cammarano_diego", "lindenlaub_lukas"], + 'caption': 'RMSE performance metric', + 'domains': [cfg.get('domain', 'global')], + 'plot_types': ['portrait'], + 'references': [ + 'gleckler08jgr', + ], + 'statistics': ['rmsd'], + } + save_to_netcdf(cfg, data, provenance) fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) @@ -404,8 +418,12 @@ def plot(cfg, data): if cfg["plot_legend"] and data.shape[3] > 1: split_legend(cfg, grid, data) basename = "portrait_plot" + print("save prov") fname = get_plot_filename(basename, cfg) + print(fname) plt.savefig(fname, bbox_inches="tight", dpi=cfg["dpi"]) + with ProvenanceLogger(cfg) as prov_logger: + prov_logger.log(fname, provenance) log.info("Figure saved:") log.info(fname) @@ -496,16 +514,18 @@ def sort_data(cfg, dataset): # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) return dataset -def save_to_netcdf(cfg, data): + +def save_to_netcdf(cfg, data, provenance): """Save the final dataset to a NetCDF file.""" # Define the output filename for the NetCDF file basename = "performance_metrics" fname = get_diagnostic_filename(basename, cfg, extension='nc') - - # Save the dataset to a NetCDF file data.to_netcdf(fname) log.info("NetCDF file saved:") log.info(fname) + with ProvenanceLogger(cfg) as prov_logger: + prov_logger.log(fname, provenance) + def main(cfg): """Run the diagnostic.""" diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml index d8c5836432..29f84fc8d4 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml @@ -4,13 +4,15 @@ documentation: title: Performance metrics plots. description: > - Compare performance of ICON simulations to a reference dataset. + Compare performance of CMIP simulations to a reference dataset. authors: - - ruhe_lukas + - cammarano_diego + - lindenlaub_lukas maintainer: - - ruhe_lukas + - lindenlaub_lukas references: - - eyring21ipcc + - eyring21ipcc + - gleckler08jgr preprocessors: pp200_3x3: @@ -23,7 +25,7 @@ preprocessors: operator: mean period: month extract_levels: - levels: 19900 + levels: 20000 scheme: linear coordinate: air_pressure mask_fillvalues: @@ -42,7 +44,7 @@ preprocessors: operator: mean period: month extract_levels: - levels: 85500 + levels: 85000 scheme: linear coordinate: air_pressure mask_fillvalues: @@ -61,7 +63,7 @@ preprocessors: operator: mean period: month extract_levels: - levels: 85500 + levels: 40000 scheme: linear coordinate: air_pressure mask_fillvalues: @@ -246,7 +248,9 @@ preprocessors: exclude: [ESACCI-SST, HadISST] diagnostics: - perfmetrics_rmse: # seasonal normalized rmse + perfmetrics_rmse: + themes: [ aerosols, phys, clouds, atmDyn, chem, ghg ] + realms: [ atmos, land, atmosChem, ocean ] variables: zg: variable: zg500 @@ -2096,7 +2100,7 @@ diagnostics: scripts: portrait: script: perfmetrics/portrait_plot.py - x_by: alias + x_by: dataset y_by: variable # use extra_facet since group and short_name don't work group_by: project normalize: "centered_median" From db014413df9ec0ece97308478896ddf79d7a8394 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 5 Nov 2024 17:35:00 +0100 Subject: [PATCH 26/56] format recipe (pre commit) --- .../recipes/recipe_perfmetrics_CMIP5_py.yml | 175 +++++++++--------- 1 file changed, 83 insertions(+), 92 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml index 29f84fc8d4..afa177c0b2 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml @@ -6,24 +6,24 @@ documentation: description: > Compare performance of CMIP simulations to a reference dataset. authors: - - cammarano_diego - - lindenlaub_lukas + - cammarano_diego + - lindenlaub_lukas maintainer: - - lindenlaub_lukas + - lindenlaub_lukas references: - - eyring21ipcc - - gleckler08jgr + - eyring21ipcc + - gleckler08jgr preprocessors: pp200_3x3: regrid: target_grid: 3x3 - scheme: linear + scheme: linear distance_metric: metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month extract_levels: levels: 20000 scheme: linear @@ -33,16 +33,16 @@ preprocessors: multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] + exclude: [ERA-Interim, NCEP-NCAR-R1] pp850_3x3: regrid: target_grid: 3x3 - scheme: linear + scheme: linear distance_metric: metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month extract_levels: levels: 85000 scheme: linear @@ -52,16 +52,16 @@ preprocessors: multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] + exclude: [ERA-Interim, NCEP-NCAR-R1] pp400_3x3: regrid: target_grid: 3x3 - scheme: linear + scheme: linear distance_metric: metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month extract_levels: levels: 40000 scheme: linear @@ -71,11 +71,11 @@ preprocessors: multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [AIRS-2-1, ERA-Interim] + exclude: [AIRS-2-1, ERA-Interim] pp500: regrid: target_grid: 3x3 - scheme: linear + scheme: linear extract_levels: levels: 50000 scheme: linear @@ -84,7 +84,7 @@ preprocessors: metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month mask_fillvalues: threshold_fraction: 0.95 multi_model_statistics: @@ -98,10 +98,10 @@ preprocessors: mask_fillvalues: threshold_fraction: 0.95 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] @@ -113,10 +113,10 @@ preprocessors: mask_fillvalues: threshold_fraction: 0.95 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] @@ -124,14 +124,14 @@ preprocessors: ppNOLEV1thr10: regrid: target_grid: 3x3 - scheme: linear + scheme: linear mask_fillvalues: threshold_fraction: 0.10 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] @@ -139,14 +139,14 @@ preprocessors: ppNOLEV1thr10_1: regrid: target_grid: 3x3 - scheme: linear + scheme: linear mask_fillvalues: threshold_fraction: 0.10 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] @@ -156,10 +156,10 @@ preprocessors: target_grid: 3x3 scheme: linear distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month mask_fillvalues: threshold_fraction: 0.10 multi_model_statistics: @@ -171,10 +171,10 @@ preprocessors: target_grid: 3x3 scheme: linear distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month mask_fillvalues: threshold_fraction: 0.10 multi_model_statistics: @@ -186,32 +186,32 @@ preprocessors: target_grid: 3x3 scheme: linear extract_region: - start_longitude: 0 - start_latitude: -90 - end_longitude: 360 - end_latitude: -60 + start_longitude: 0 + start_latitude: -90 + end_longitude: 360 + end_latitude: -60 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month mask_fillvalues: threshold_fraction: 0.10 multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ESACCI-OZONE, NIWA-BS] + exclude: [ESACCI-OZONE, NIWA-BS] ppNOLEV2: regrid: target_grid: 3x3 scheme: linear distance_metric: - metric: weighted_rmse + metric: weighted_rmse mask_fillvalues: threshold_fraction: 0.95 climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] @@ -221,16 +221,16 @@ preprocessors: target_grid: 3x3 scheme: linear distance_metric: - metric: weighted_rmse + metric: weighted_rmse mask_fillvalues: threshold_fraction: 0.95 climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] + exclude: [ERA-Interim, NCEP-NCAR-R1] ppNOLEV1x1: regrid: target_grid: 1x1 @@ -238,19 +238,19 @@ preprocessors: mask_fillvalues: threshold_fraction: 0.95 distance_metric: - metric: weighted_rmse + metric: weighted_rmse climate_statistics: operator: mean - period: month + period: month multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ESACCI-SST, HadISST] + exclude: [ESACCI-SST, HadISST] diagnostics: perfmetrics_rmse: - themes: [ aerosols, phys, clouds, atmDyn, chem, ghg ] - realms: [ atmos, land, atmosChem, ocean ] + themes: [aerosols, phys, clouds, atmDyn, chem, ghg] + realms: [atmos, land, atmosChem, ocean] variables: zg: variable: zg500 @@ -372,7 +372,7 @@ diagnostics: - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} + version: 1, tier: 2, reference_for_metric: true} sm: variable: sm preprocessor: ppNOLEV1thr10 @@ -470,8 +470,12 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ESACCI-CLOUD, project: OBS, type: sat, - version: AVHRR-AMPM-fv3.0, tier: 2, reference_for_metric: true} + - dataset: ESACCI-CLOUD + project: OBS + type: sat + version: AVHRR-AMPM-fv3.0 + tier: 2 + reference_for_metric: true clt_2: variable: clt short_name: clt @@ -524,8 +528,8 @@ diagnostics: - {dataset: MPI-ESM-MR} - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, reference_for_metric: true} + - {dataset: NorESM1-ME} + - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, reference_for_metric: true} ts: variable: ts preprocessor: ppNOLEV1x1 @@ -578,12 +582,11 @@ diagnostics: - {dataset: MPI-ESM-MR} - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ESACCI-SST, project: OBS, type: sat, - version: 2.2, tier: 2, reference_for_metric: true} + - {dataset: NorESM1-ME} + - {dataset: ESACCI-SST, project: OBS, type: sat, version: 2.2, tier: 2, reference_for_metric: true} ts_2: variable: ts - short_name: ts + short_name: ts preprocessor: ppNOLEV1x1 mip: Amon project: CMIP5 @@ -634,8 +637,8 @@ diagnostics: - {dataset: MPI-ESM-MR} - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: HadISST, project: OBS, type: reanaly, version: 1, tier: 2, reference_for_metric: true} + - {dataset: NorESM1-ME} + - {dataset: HadISST, project: OBS, type: reanaly, version: 1, tier: 2, reference_for_metric: true} toz: variable: toz preprocessor: ppNOLEV2thr10 @@ -719,7 +722,6 @@ diagnostics: - {dataset: GISS-E2-R, ensemble: r1i1p2} - {dataset: MIROC-ESM-CHEM} - {dataset: NIWA-BS, project: OBS, type: sat, version: v3.3, tier: 3, reference_for_metric: true} - tas: variable: tas preprocessor: ppNOLEV2_1 @@ -909,7 +911,7 @@ diagnostics: ensemble: r1i1p1 start_year: 2000 end_year: 2002 - split: Ref1 + split: Ref1 additional_datasets: - {dataset: ACCESS1-0} - {dataset: ACCESS1-3} @@ -960,7 +962,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true } + - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} rlut: variable: rlut preprocessor: ppNOLEV1 @@ -1118,7 +1120,7 @@ diagnostics: - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} - hus: + hus: variable: hus400 preprocessor: pp400_3x3 mip: Amon @@ -1170,7 +1172,7 @@ diagnostics: - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - {dataset: AIRS-2-1, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} - hus_2: + hus_2: variable: hus400 short_name: hus preprocessor: pp400_3x3 @@ -1347,7 +1349,7 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} + - {dataset: NorESM1-ME} - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} ua850: @@ -1473,7 +1475,7 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} + - {dataset: NorESM1-ME} - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} ta200: @@ -1538,8 +1540,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} ta200_2: variable: ta200 short_name: ta @@ -1602,8 +1603,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} ta850: variable: ta850 short_name: ta @@ -1666,8 +1666,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} ta850_2: variable: ta850 short_name: ta @@ -1730,8 +1729,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} va200: variable: va200 short_name: va @@ -1792,8 +1790,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} va200_2: variable: va200 short_name: va @@ -1853,9 +1850,8 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} va850: variable: va850 short_name: va @@ -1916,8 +1912,7 @@ diagnostics: - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} + - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} va850_2: variable: va850 short_name: va @@ -1977,9 +1972,8 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: MRI-ESM1} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} + - {dataset: NorESM1-ME} + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} abs550aer: variable: abs550aer preprocessor: ppNOLEV1thr10_1 @@ -2003,8 +1997,7 @@ diagnostics: - {dataset: MIROC-ESM-CHEM} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, - version: SU-v4.21, tier: 2, reference_for_metric: true} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} od870aer: variable: od870aer preprocessor: ppNOLEV1thr10_1 @@ -2028,8 +2021,7 @@ diagnostics: - {dataset: MIROC-ESM} - {dataset: MIROC-ESM-CHEM} - {dataset: MRI-CGCM3} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, - version: SU-v4.21, tier: 2, reference_for_metric: true} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} od550aer: variable: od550aer preprocessor: ppNOLEV2thr10_1 @@ -2061,8 +2053,7 @@ diagnostics: - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - {dataset: NorESM1-ME} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, - version: SU-v4.21, tier: 2, reference_for_metric: true} + - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} od550aer_2: variable: od550aer short_name: od550aer @@ -2094,9 +2085,9 @@ diagnostics: - {dataset: MIROC-ESM-CHEM} - {dataset: MRI-CGCM3} - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} + - {dataset: NorESM1-ME} - {dataset: MODIS, project: OBS, type: sat, version: MYD08-M3, tier: 3, reference_for_metric: true} - + scripts: portrait: script: perfmetrics/portrait_plot.py @@ -2111,4 +2102,4 @@ diagnostics: vmax: +0.5 cbar_kwargs: label: Relative RMSE - extend: both \ No newline at end of file + extend: both From efc026a402690d3253263bd5b4d2776c19823efb Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 6 Nov 2024 16:12:08 +0100 Subject: [PATCH 27/56] simplify recipe p1 --- .../recipes/recipe_perfmetrics_CMIP5_py.yml | 214 ++++-------------- 1 file changed, 39 insertions(+), 175 deletions(-) diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml index afa177c0b2..b51ff0dffb 100644 --- a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml +++ b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml @@ -15,7 +15,7 @@ documentation: - gleckler08jgr preprocessors: - pp200_3x3: + default: &default # common preprocessor settings regrid: target_grid: 3x3 scheme: linear @@ -24,228 +24,92 @@ preprocessors: climate_statistics: operator: mean period: month - extract_levels: - levels: 20000 - scheme: linear - coordinate: air_pressure mask_fillvalues: threshold_fraction: 0.95 multi_model_statistics: span: overlap statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] - pp850_3x3: - regrid: - target_grid: 3x3 + # exclude all possible reference datasets + exclude: [ + AIRS-2-1, + CERES-EBAF, + ERA-Interim, + ESACCI-AEROSOL, + ESACCI-CLOUD, + ESACCI-OZONE, + ESACCI-SOILMOISTURE, + ESACCI-SST, + GPCP-V2.2, + HadISST, + NCEP-NCAR-R1, + MODIS, + NIWA-BS, + PATMOS-x] + pp200_3x3: # only add/overwrite var specific settings + <<: *default + extract_levels: + levels: 20000 scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month + coordinate: air_pressure + pp850_3x3: + <<: *default extract_levels: levels: 85000 scheme: linear coordinate: air_pressure - mask_fillvalues: - threshold_fraction: 0.95 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] pp400_3x3: - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month + <<: *default extract_levels: levels: 40000 scheme: linear coordinate: air_pressure - mask_fillvalues: - threshold_fraction: 0.95 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [AIRS-2-1, ERA-Interim] pp500: - regrid: - target_grid: 3x3 - scheme: linear + <<: *default extract_levels: levels: 50000 scheme: linear coordinate: air_pressure - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - mask_fillvalues: - threshold_fraction: 0.95 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] ppNOLEV1: - regrid: - target_grid: 3x3 - scheme: linear - mask_fillvalues: - threshold_fraction: 0.95 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [CERES-EBAF] + <<: *default ppNOLEV1_3x3: - regrid: - target_grid: 3x3 - scheme: linear - mask_fillvalues: - threshold_fraction: 0.95 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [GPCP-V2.2] + <<: *default ppNOLEV1thr10: - regrid: - target_grid: 3x3 - scheme: linear + <<: *default mask_fillvalues: threshold_fraction: 0.10 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-SOILMOISTURE] ppNOLEV1thr10_1: - regrid: - target_grid: 3x3 - scheme: linear + <<: *default mask_fillvalues: threshold_fraction: 0.10 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-AEROSOL] ppNOLEV2thr10: - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month + <<: *default mask_fillvalues: threshold_fraction: 0.10 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-OZONE, NIWA-BS] ppNOLEV2thr10_1: - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month + <<: *default mask_fillvalues: threshold_fraction: 0.10 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-AEROSOL, MODIS] ppNOLEV2thr10_shpolar: - regrid: - target_grid: 3x3 - scheme: linear + <<: *default extract_region: start_longitude: 0 start_latitude: -90 end_longitude: 360 end_latitude: -60 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month mask_fillvalues: threshold_fraction: 0.10 - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-OZONE, NIWA-BS] ppNOLEV2: - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - mask_fillvalues: - threshold_fraction: 0.95 - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [PATMOS-x, ESACCI-CLOUD] + <<: *default ppNOLEV2_1: - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - mask_fillvalues: - threshold_fraction: 0.95 - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ERA-Interim, NCEP-NCAR-R1] + <<: *default ppNOLEV1x1: + <<: *default regrid: target_grid: 1x1 scheme: linear - mask_fillvalues: - threshold_fraction: 0.95 - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - multi_model_statistics: - span: overlap - statistics: [mean, median] - exclude: [ESACCI-SST, HadISST] + +# datasets can't be defined here as there is one variable that do not use own datasets +# is there a way to overwrite datasets rather than just put ADDITIONAL ones? diagnostics: perfmetrics_rmse: From af917bc2db85e2ddaa76fdb6e3f4f4b6f755a3a0 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 6 Nov 2024 23:56:05 +0100 Subject: [PATCH 28/56] MM stats as first columns (wip) --- .../diag_scripts/perfmetrics/portrait_plot.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 16036bae43..1c25d46d67 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -307,8 +307,6 @@ def plot_group(cfg, axe, data, title=None): used. Other splits will be added by overlaying triangles. """ split = data.sel({cfg["split_by"]: cfg["default_split"]}) - print(f"Plotting group {title}") - print(split) plot_matrix( split.values.T, # 2d numpy array split.coords[cfg["y_by"]].values, # y_labels @@ -374,6 +372,7 @@ def plot(cfg, data): the content of data (xr.DataArray) """ # Save the dataset to NetCDF before plotting + # TODO: should we collect references and list them in the caption somehow? provenance = { 'ancestors': list(cfg["input_data"].keys()), 'authors': ["cammarano_diego", "lindenlaub_lukas"], @@ -497,6 +496,7 @@ def set_defaults(cfg): def sort_data(cfg, dataset): """Sort the dataset along by custom or alphabetical order.""" + # TODO: decide on (default) strategies and options. # custom order: dsimport xarray as xr # import pandas as pd # order = ['value3', 'value1', 'value2'] # replace by custom order @@ -512,6 +512,20 @@ def sort_data(cfg, dataset): # apply custom orders if given: # if cfg.get("x_order"): # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) + # move MMM to begin: + if cfg["x_by"] in ["alias", "dataset"]: + # TODO: not clean, but it works for many cases + mm_stats = [ + v for v in dataset[cfg["x_by"]].values + if "Mean" in v or "Median" in v + ] + others = [ + v for v in dataset[cfg["x_by"]].values + if "Mean" not in v and "Median" not in v + ] + new_order = mm_stats + others + dataset = dataset.reindex({cfg["x_by"]: new_order}) + dataset[cfg["x_by"]] return dataset From d25290ef28b84cff66328eeaa30c40826cb4bbad Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 6 Nov 2024 23:56:34 +0100 Subject: [PATCH 29/56] simpler cmip5 vs cmip6 plot --- esmvaltool/recipes/recipe_portrait_CMIP.yml | 218 ++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 esmvaltool/recipes/recipe_portrait_CMIP.yml diff --git a/esmvaltool/recipes/recipe_portrait_CMIP.yml b/esmvaltool/recipes/recipe_portrait_CMIP.yml new file mode 100644 index 0000000000..3d10d3e7df --- /dev/null +++ b/esmvaltool/recipes/recipe_portrait_CMIP.yml @@ -0,0 +1,218 @@ +# ESMValTool +# +--- +documentation: + title: Performance metrics plots. + description: > + Compare performance of CMIP simulations to a reference dataset. + authors: + - cammarano_diego + - lindenlaub_lukas + maintainer: + - lindenlaub_lukas + references: + - eyring21ipcc + - gleckler08jgr + +cmip5: &CMIP5 + project: CMIP5 + ensemble: r1i1p1 + +datasets: + # cmip5 + - {<<: *CMIP5, dataset: ACCESS1-0} + - {<<: *CMIP5, dataset: CESM1-BGC} + - {<<: *CMIP5, dataset: CNRM-CM5} + - {<<: *CMIP5, dataset: GFDL-ESM2M} + - {<<: *CMIP5, dataset: HadGEM2-CC} + - {<<: *CMIP5, dataset: IPSL-CM5B-LR} + - {<<: *CMIP5, dataset: MIROC-ESM} + - {<<: *CMIP5, dataset: MPI-ESM-LR} + - {<<: *CMIP5, dataset: MRI-CGCM3} + # cmip6 + - {dataset: ACCESS-ESM1-5, institute: CSIRO} + - {dataset: CESM2, institute: NCAR} + - {dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} + - {dataset: GFDL-CM4, grid: gr1} + # - {dataset: IPSL-CM6A-LR, grid: gr} + - {dataset: MIROC-ES2L, ensemble: r1i1p1f2} + - {dataset: MPI-ESM1-2-LR} + - {dataset: MRI-ESM2-0} + - {dataset: UKESM1-0-LL, ensemble: r1i1p1f2} + + +preprocessors: + default: &default # common preprocessor settings + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + groupby: ['project'] + # exclude all possible reference datasets + exclude: [ + AIRS-2-1, + CERES-EBAF, + ERA-Interim, + ESACCI-AEROSOL, + ESACCI-CLOUD, + ESACCI-OZONE, + ESACCI-SOILMOISTURE, + ESACCI-SST, + GPCP-V2.2, + HadISST, + NCEP-NCAR-R1, + MODIS, + NIWA-BS, + PATMOS-x] + pp200: # only add/overwrite var specific settings + <<: *default + extract_levels: + levels: 20000 + scheme: linear + coordinate: air_pressure + pp500: # only add/overwrite var specific settings + <<: *default + extract_levels: + levels: 50000 + scheme: linear + coordinate: air_pressure + thr10: # only add/overwrite var specific settings + <<: *default + mask_fillvalues: + threshold_fraction: 0.10 + +# datasets can't be defined here as there is one variable that do not use own datasets +# is there a way to overwrite datasets rather than just put ADDITIONAL ones? + +var_default: &var_default + mip: Amon + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + preprocessor: default + grid: gn + start_year: 2000 + end_year: 2002 + split: Ref1 # first triangle + +diagnostics: + portrait_rmse: + themes: [aerosols, phys, clouds, atmDyn, chem, ghg] + realms: [atmos, land, atmosChem, ocean] + variables: + zg: &zg + <<: *var_default + short_name: zg # find cmor table for custom var name + variable: zg500 + preprocessor: pp500 + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + zg_2: + <<: *zg + split: Ref2 + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + clt: &clt + <<: *var_default + short_name: clt + variable: clt + additional_datasets: + - dataset: ESACCI-CLOUD + project: OBS + type: sat + version: AVHRR-AMPM-fv3.0 + tier: 2 + reference_for_metric: true + clt_2: + <<: *clt + split: Ref2 + additional_datasets: + - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, reference_for_metric: true} + ts: &ts + <<: *var_default + short_name: ts + variable: ts + preprocessor: default + additional_datasets: + - {dataset: ESACCI-SST, project: OBS, type: sat, version: 2.2, tier: 2, reference_for_metric: true} + ts_2: + <<: *ts + split: Ref2 + additional_datasets: + - {dataset: HadISST, project: OBS, type: reanaly, version: 1, tier: 2, reference_for_metric: true} + tas: &tas + <<: *var_default + short_name: tas + variable: tas + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + tas_2: + <<: *tas + split: Ref2 + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + pr: + <<: *var_default + variable: pr + split: Ref1 + additional_datasets: + - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} + rlut: + <<: *var_default + variable: rlut + start_year: 2001 + end_year: 2003 + additional_datasets: + - {dataset: CERES-EBAF, project: OBS, type: sat, version: Ed4.2, + tier: 2, reference_for_metric: true} + rsut: + <<: *var_default + variable: rsut + start_year: 2001 + end_year: 2003 + additional_datasets: + - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} + ua200: &ua200 + <<: *var_default + variable: ua200 + short_name: ua + preprocessor: pp200 + split: Ref1 + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ua200_2: + <<: *ua200 + split: Ref2 + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + + scripts: + portrait: + script: perfmetrics/portrait_plot.py + x_by: dataset + y_by: variable # extra_facet + group_by: project + normalize: "centered_median" + default_split: Ref1 + nan_color: null + plot_kwargs: + vmin: -0.5 + vmax: +0.5 + cbar_kwargs: + label: Relative RMSE + extend: both From 9ae502c374f6c2077631b8ad2c15e062b17195e9 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 14:16:54 +0100 Subject: [PATCH 30/56] provenance update, non-scalar error --- .../diag_scripts/perfmetrics/portrait_plot.py | 71 +++++++++---------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 1c25d46d67..f8895e553c 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -124,8 +124,23 @@ log = logging.getLogger(__name__) +def get_provenance(cfg): + # TODO: should we collect references and list them in the caption somehow? + return { + 'ancestors': list(cfg["input_data"].keys()), + 'authors': ["lindenlaub_lukas", "cammarano_diego"], + 'caption': 'RMSE performance metric', + 'domains': [cfg.get('domain', 'global')], + 'plot_types': ['portrait'], + 'references': [ + 'gleckler08jgr', + ], + 'statistics': ['rmsd'], + } + + def unify_limits(grid): - """Set same limits for all subplots.""" + """Ensure same limits for all subplots.""" vmin, vmax = np.inf, -np.inf images = [ax.get_images()[0] for ax in grid] for img in images: @@ -153,17 +168,14 @@ def plot_matrix(data, row_labels, col_labels, axe, plot_kwargs): # ax.spines[:].set_visible(False) axe.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) axe.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) - axe.grid(which="minor", color="black", linestyle="-", linewidth=1) + axe.grid(which="minor", color="black", linestyle="-", linewidth=0.8) axe.tick_params(which="both", bottom=False, left=False) return img def remove_reference(metas): - """Remove reference for metric from list of metadata. - - NOTE: list() creates a copy with same references to allow removing in place - """ - for meta in list(metas): + """Remove reference for metric from list of metadata.""" + for meta in list(metas): # list() creates a copy to allow remove in place if meta.get("reference_for_metric", False): metas.remove(meta) @@ -190,7 +202,11 @@ def open_file(metadata, **selection): log.debug("Metadata found for %s", selection) das = xr.open_dataset(metas[0]["filename"]) varname = list(das.data_vars.keys())[0] - return das[varname].values.item() + try: + return das[varname].values.item() + except ValueError: + msg = f"Expected scalar in input file {metas[0]['filename']}." + raise ValueError(msg) # iris.load_cube(metas[0]["filename"]).data @@ -215,11 +231,11 @@ def load_data(cfg, metas): for coord_tuple in itertools.product(*coords.values()): selection = dict(zip(coords.keys(), coord_tuple)) data['var'].loc[selection] = open_file(metas, **selection) - # data[coord_tuple] = (list(coords.keys(), value)) if None in data.coords[cfg["split_by"]].values: - cfg.update({"default_split": None}) + cfg.setdefault({"default_split": None}) else: - cfg.update({"default_split": data.coords[cfg["split_by"]].values[0]}) + cfg.setdefault( + {"default_split": data.coords[cfg["split_by"]].values[0]}) log.debug("using %s as default split", cfg["default_split"]) log.debug("Loaded Data:") log.debug(data) @@ -371,20 +387,7 @@ def plot(cfg, data): sets same color range and overlays additional references based on the content of data (xr.DataArray) """ - # Save the dataset to NetCDF before plotting - # TODO: should we collect references and list them in the caption somehow? - provenance = { - 'ancestors': list(cfg["input_data"].keys()), - 'authors': ["cammarano_diego", "lindenlaub_lukas"], - 'caption': 'RMSE performance metric', - 'domains': [cfg.get('domain', 'global')], - 'plot_types': ['portrait'], - 'references': [ - 'gleckler08jgr', - ], - 'statistics': ['rmsd'], - } - save_to_netcdf(cfg, data, provenance) + save_to_netcdf(cfg, data) fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) @@ -417,12 +420,10 @@ def plot(cfg, data): if cfg["plot_legend"] and data.shape[3] > 1: split_legend(cfg, grid, data) basename = "portrait_plot" - print("save prov") fname = get_plot_filename(basename, cfg) - print(fname) plt.savefig(fname, bbox_inches="tight", dpi=cfg["dpi"]) with ProvenanceLogger(cfg) as prov_logger: - prov_logger.log(fname, provenance) + prov_logger.log(fname, get_provenance(cfg)) log.info("Figure saved:") log.info(fname) @@ -497,12 +498,6 @@ def set_defaults(cfg): def sort_data(cfg, dataset): """Sort the dataset along by custom or alphabetical order.""" # TODO: decide on (default) strategies and options. - # custom order: dsimport xarray as xr - # import pandas as pd - # order = ['value3', 'value1', 'value2'] # replace by custom order - # ds[cfg['y_by']] = pd.Categorical(ds[cfg['y_by']], categories=order, - # ordered=True) - # ds = ds.sortby('y_by') # sort alphabetically (caseinsensitive) dataset = dataset.sortby([ dataset[cfg["x_by"]].str.lower(), dataset[cfg["y_by"]].str.lower(), @@ -514,7 +509,7 @@ def sort_data(cfg, dataset): # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) # move MMM to begin: if cfg["x_by"] in ["alias", "dataset"]: - # TODO: not clean, but it works for many cases + # NOTE: not clean, but it works for many cases mm_stats = [ v for v in dataset[cfg["x_by"]].values if "Mean" in v or "Median" in v @@ -529,16 +524,16 @@ def sort_data(cfg, dataset): return dataset -def save_to_netcdf(cfg, data, provenance): +def save_to_netcdf(cfg, data): """Save the final dataset to a NetCDF file.""" # Define the output filename for the NetCDF file - basename = "performance_metrics" + basename = "portrait" fname = get_diagnostic_filename(basename, cfg, extension='nc') data.to_netcdf(fname) log.info("NetCDF file saved:") log.info(fname) with ProvenanceLogger(cfg) as prov_logger: - prov_logger.log(fname, provenance) + prov_logger.log(fname, get_provenance(cfg)) def main(cfg): From 666837cb5075cfc3fafc99cff5d5fed8a4352fac Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 14:29:05 +0100 Subject: [PATCH 31/56] don't overwrite users default_split --- esmvaltool/diag_scripts/perfmetrics/portrait_plot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index f8895e553c..1b18ec1aab 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -232,10 +232,9 @@ def load_data(cfg, metas): selection = dict(zip(coords.keys(), coord_tuple)) data['var'].loc[selection] = open_file(metas, **selection) if None in data.coords[cfg["split_by"]].values: - cfg.setdefault({"default_split": None}) + cfg.setdefault("default_split", None) else: - cfg.setdefault( - {"default_split": data.coords[cfg["split_by"]].values[0]}) + cfg.setdefault("default_split", data.coords[cfg["split_by"]].values[0]) log.debug("using %s as default split", cfg["default_split"]) log.debug("Loaded Data:") log.debug(data) From ac0e95e36bb11423377c02b1dac405f4195baab9 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 14:51:11 +0100 Subject: [PATCH 32/56] remove experimental/dev recipes --- .../recipes/recipe_perfmetrics_CMIP5_py.yml | 1969 ----------------- .../recipe_perfmetrics_CMIP5_python.yml | 276 --- .../recipes/recipe_perfmetrics_python.yml | 243 -- 3 files changed, 2488 deletions(-) delete mode 100644 esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml delete mode 100644 esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml delete mode 100644 esmvaltool/recipes/recipe_perfmetrics_python.yml diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml deleted file mode 100644 index b51ff0dffb..0000000000 --- a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_py.yml +++ /dev/null @@ -1,1969 +0,0 @@ -# ESMValTool -# ---- -documentation: - title: Performance metrics plots. - description: > - Compare performance of CMIP simulations to a reference dataset. - authors: - - cammarano_diego - - lindenlaub_lukas - maintainer: - - lindenlaub_lukas - references: - - eyring21ipcc - - gleckler08jgr - -preprocessors: - default: &default # common preprocessor settings - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - mask_fillvalues: - threshold_fraction: 0.95 - multi_model_statistics: - span: overlap - statistics: [mean, median] - # exclude all possible reference datasets - exclude: [ - AIRS-2-1, - CERES-EBAF, - ERA-Interim, - ESACCI-AEROSOL, - ESACCI-CLOUD, - ESACCI-OZONE, - ESACCI-SOILMOISTURE, - ESACCI-SST, - GPCP-V2.2, - HadISST, - NCEP-NCAR-R1, - MODIS, - NIWA-BS, - PATMOS-x] - pp200_3x3: # only add/overwrite var specific settings - <<: *default - extract_levels: - levels: 20000 - scheme: linear - coordinate: air_pressure - pp850_3x3: - <<: *default - extract_levels: - levels: 85000 - scheme: linear - coordinate: air_pressure - pp400_3x3: - <<: *default - extract_levels: - levels: 40000 - scheme: linear - coordinate: air_pressure - pp500: - <<: *default - extract_levels: - levels: 50000 - scheme: linear - coordinate: air_pressure - ppNOLEV1: - <<: *default - ppNOLEV1_3x3: - <<: *default - ppNOLEV1thr10: - <<: *default - mask_fillvalues: - threshold_fraction: 0.10 - ppNOLEV1thr10_1: - <<: *default - mask_fillvalues: - threshold_fraction: 0.10 - ppNOLEV2thr10: - <<: *default - mask_fillvalues: - threshold_fraction: 0.10 - ppNOLEV2thr10_1: - <<: *default - mask_fillvalues: - threshold_fraction: 0.10 - ppNOLEV2thr10_shpolar: - <<: *default - extract_region: - start_longitude: 0 - start_latitude: -90 - end_longitude: 360 - end_latitude: -60 - mask_fillvalues: - threshold_fraction: 0.10 - ppNOLEV2: - <<: *default - ppNOLEV2_1: - <<: *default - ppNOLEV1x1: - <<: *default - regrid: - target_grid: 1x1 - scheme: linear - -# datasets can't be defined here as there is one variable that do not use own datasets -# is there a way to overwrite datasets rather than just put ADDITIONAL ones? - -diagnostics: - perfmetrics_rmse: - themes: [aerosols, phys, clouds, atmDyn, chem, ghg] - realms: [atmos, land, atmosChem, ocean] - variables: - zg: - variable: zg500 - preprocessor: pp500 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - zg_2: - variable: zg500 - short_name: zg - preprocessor: pp500 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref2 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - sm: - variable: sm - preprocessor: ppNOLEV1thr10 - mip: Lmon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2002 - end_year: 2004 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: HadCM3} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ESACCI-SOILMOISTURE, project: OBS, type: sat, - version: L3S-SSMV-COMBINED-v4.2, tier: 2, reference_for_metric: true} - clt: - variable: clt - preprocessor: ppNOLEV2 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - dataset: ESACCI-CLOUD - project: OBS - type: sat - version: AVHRR-AMPM-fv3.0 - tier: 2 - reference_for_metric: true - clt_2: - variable: clt - short_name: clt - preprocessor: ppNOLEV2 - mip: Amon - split: Ref2 - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: PATMOS-x, project: OBS, type: sat, version: NOAA, tier: 2, reference_for_metric: true} - ts: - variable: ts - preprocessor: ppNOLEV1x1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ESACCI-SST, project: OBS, type: sat, version: 2.2, tier: 2, reference_for_metric: true} - ts_2: - variable: ts - short_name: ts - preprocessor: ppNOLEV1x1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref2 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: HadISST, project: OBS, type: reanaly, version: 1, tier: 2, reference_for_metric: true} - toz: - variable: toz - preprocessor: ppNOLEV2thr10 - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2002 - end_year: 2004 - additional_datasets: - - {dataset: CESM1-WACCM, ensemble: r2i1p1} - - {dataset: CNRM-CM5} - - {dataset: GFDL-CM3} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: MIROC-ESM-CHEM} - - {dataset: ESACCI-OZONE, project: OBS, type: sat, version: L3, tier: 2, reference_for_metric: true} - toz_2: - variable: toz - short_name: toz - preprocessor: ppNOLEV2thr10 - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref2 - start_year: 2002 - end_year: 2004 - additional_datasets: - - {dataset: CESM1-WACCM, ensemble: r2i1p1} - - {dataset: CNRM-CM5} - - {dataset: GFDL-CM3} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: MIROC-ESM-CHEM} - - {dataset: NIWA-BS, project: OBS, type: sat, version: v3.3, tier: 3, reference_for_metric: true} - toz_shpolar: - variable: toz_shpolar - short_name: toz - preprocessor: ppNOLEV2thr10_shpolar - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref1 - start_year: 2002 - end_year: 2004 - additional_datasets: - - {dataset: CESM1-WACCM, ensemble: r2i1p1} - - {dataset: CNRM-CM5} - - {dataset: GFDL-CM3} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: MIROC-ESM-CHEM} - - {dataset: ESACCI-OZONE, project: OBS, type: sat, version: L3, tier: 2, reference_for_metric: true} - toz_2_shpolar: - variable: toz_shpolar - short_name: toz - preprocessor: ppNOLEV2thr10_shpolar - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - split: Ref2 - start_year: 2002 - end_year: 2004 - additional_datasets: - - {dataset: CESM1-WACCM, ensemble: r2i1p1} - - {dataset: CNRM-CM5} - - {dataset: GFDL-CM3} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: MIROC-ESM-CHEM} - - {dataset: NIWA-BS, project: OBS, type: sat, version: v3.3, tier: 3, reference_for_metric: true} - tas: - variable: tas - preprocessor: ppNOLEV2_1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - # - {dataset: CESM1-CAM5-1-FV2} # data is missing on ESGF - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - tas_2: - variable: tas - short_name: tas - preprocessor: ppNOLEV2_1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - # - {dataset: CESM1-CAM5-1-FV2} # data is missing on ESGF - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - lwcre: - variable: lwcre - preprocessor: ppNOLEV1 - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2001 - end_year: 2003 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} - pr: - variable: pr - preprocessor: ppNOLEV1_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} - rlut: - variable: rlut - preprocessor: ppNOLEV1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2001 - end_year: 2003 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: CERES-EBAF, project: OBS, type: sat, version: Ed4.2, - tier: 2, reference_for_metric: true} - rsut: - variable: rsut - preprocessor: ppNOLEV1 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2001 - end_year: 2003 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} - swcre: - variable: swcre - preprocessor: ppNOLEV1 - mip: Amon - derive: true - force_derivation: false - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2001 - end_year: 2003 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} - hus: - variable: hus400 - preprocessor: pp400_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2003 - end_year: 2004 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R-CC} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: AIRS-2-1, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} - hus_2: - variable: hus400 - short_name: hus - preprocessor: pp400_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2003 - end_year: 2004 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R-CC} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, - type: reanaly, version: 1, tier: 3, reference_for_metric: true} - ua200: - variable: ua200 - short_name: ua - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - ua200_2: - variable: ua200 - short_name: ua - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - ua850: - variable: ua850 - short_name: ua - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - ua850_2: - variable: ua850 - short_name: ua - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - ta200: - variable: ta200 - short_name: ta - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} - ta200_2: - variable: ta200 - short_name: ta - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} - ta850: - variable: ta850 - short_name: ta - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} - ta850_2: - variable: ta850 - short_name: ta - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-CAM5-1-FV2} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} - va200: - variable: va200 - short_name: va - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} - va200_2: - variable: va200 - short_name: va - preprocessor: pp200_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} - va850: - variable: va850 - short_name: va - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ERA-Interim, project: OBS6, type: reanaly, version: 1, tier: 3, reference_for_metric: true} - va850_2: - variable: va850 - short_name: va - preprocessor: pp850_3x3 - mip: Amon - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - - {dataset: CNRM-CM5-2} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-H-CC} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: GISS-E2-R-CC} - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - - {dataset: MPI-ESM-MR} - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2, reference_for_metric: true} - abs550aer: - variable: abs550aer - preprocessor: ppNOLEV1thr10_1 - mip: aero - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2002 - end_year: 2004 - split: Ref1 - additional_datasets: - - {dataset: CSIRO-Mk3-6-0} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} - od870aer: - variable: od870aer - preprocessor: ppNOLEV1thr10_1 - mip: aero - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2002 - end_year: 2004 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MRI-CGCM3} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} - od550aer: - variable: od550aer - preprocessor: ppNOLEV2thr10_1 - mip: aero - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2003 - end_year: 2004 - split: Ref1 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: BNU-ESM} - - {dataset: CESM1-CAM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: ESACCI-AEROSOL, project: OBS, type: sat, version: SU-v4.21, tier: 2, reference_for_metric: true} - od550aer_2: - variable: od550aer - short_name: od550aer - preprocessor: ppNOLEV2thr10_1 - mip: aero - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2003 - end_year: 2004 - split: Ref2 - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: BNU-ESM} - - {dataset: CESM1-CAM5} - - {dataset: CSIRO-Mk3-6-0} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - - {dataset: GISS-E2-R, ensemble: r1i1p2} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MRI-CGCM3} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - - {dataset: MODIS, project: OBS, type: sat, version: MYD08-M3, tier: 3, reference_for_metric: true} - - scripts: - portrait: - script: perfmetrics/portrait_plot.py - x_by: dataset - y_by: variable # use extra_facet since group and short_name don't work - group_by: project - normalize: "centered_median" - default_split: Ref1 - nan_color: null - plot_kwargs: - vmin: -0.5 - vmax: +0.5 - cbar_kwargs: - label: Relative RMSE - extend: both diff --git a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml b/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml deleted file mode 100644 index 6ad21ee5f5..0000000000 --- a/esmvaltool/recipes/recipe_perfmetrics_CMIP5_python.yml +++ /dev/null @@ -1,276 +0,0 @@ -# ESMValTool -# recipe_perfmetrics_CMIP5.yml ---- -documentation: - title: Performance metrics for essential climate variables in CMIP5 - - description: | - Recipe for plotting the performance metrics for the CMIP5 datasets, - including the standard ECVs as in Gleckler et al., and some additional - variables (like ozone, sea-ice, aerosol...) - - authors: - - winterstein_franziska - - righi_mattia - - eyring_veronika - - ruhe_lukas - - maintainer: - - ruhe_lukas - - references: - - gleckler08jgr - - projects: - - esmval - - embrace - - crescendo - - c3s-magic - - cmug - -preprocessors: - ppNOLEV1: - regrid: - target_grid: reference_dataset - scheme: linear - mask_fillvalues: - threshold_fraction: 0.95 - # multi_model_statistics: - # span: overlap - # statistics: [mean, median] - # exclude: [reference_dataset] - - rmse: &rmse - custom_order: true - regrid: - # target_grid: reference_dataset - scheme: linear - target_grid: 3x3 - # scheme: nearest - regrid_time: - calendar: standard - frequency: mon - mask_fillvalues: - threshold_fraction: 0.95 - # multi_model_statistics: - # span: overlap - # statistics: [mean, median] - # exclude: [reference_dataset, alternative_dataset] - distance_metric: - metric: rmse - - pp500: - # <<: *rmse - custom_order: true - regrid: - target_grid: 3x3 - scheme: linear - extract_levels: - levels: 50000 - scheme: linear - regrid_time: - calendar: standard - frequency: mon - mask_fillvalues: - threshold_fraction: 0.95 - distance_metric: - metric: rmse - -diagnostics: - perfmetrics: - description: Near-surface air temperature - themes: - - phys - realms: - - atmos - variables: - tas: &var_default - preprocessor: rmse - reference_dataset: ERA-Interim - # alternative_dataset: NCEP-NCAR-R1 - mip: Amon - split: Ref - y_label: tas_Glob - project: CMIP5 - exp: historical - ensemble: r1i1p1 - start_year: 2000 - end_year: 2002 - additional_datasets: - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - tas_alt: - <<: *var_default - short_name: tas - y_label: tas_Glob - split: Alt - additional_datasets: - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - pr: - <<: *var_default - y_label: pr_Glob - reference_dataset: GPCP-V2.2 - additional_datasets: - - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} - - # swcre: - # <<: *var_default - # derive: true - # force_derivation: false - # y_label: swcre_Glob - # additional_datasets: - # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, - # tier: 1, reference_for_metric: true} - - # rlut: - # <<: *var_default - # y_label: rlut_Glob - # additional_datasets: - # - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, - # tier: 1, reference_for_metric: true, start_year: 2000, end_year: 2002} 34 vs 36 month dataa.. 2 months missing in obs?? - - zg: - <<: *var_default - y_label: zg_Glob-500 - preprocessor: pp500 - additional_datasets: - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - - additional_datasets: - - {dataset: ACCESS1-0} - - {dataset: ACCESS1-3} - - {dataset: bcc-csm1-1} - - {dataset: bcc-csm1-1-m} - - {dataset: BNU-ESM} - - {dataset: CanCM4} - - {dataset: CanESM2} - - {dataset: CCSM4} - - {dataset: CESM1-BGC} - - {dataset: CESM1-CAM5} - - {dataset: CESM1-FASTCHEM} - - {dataset: CESM1-WACCM} - - {dataset: CMCC-CESM} - - {dataset: CMCC-CM} - - {dataset: CMCC-CMS} - - {dataset: CNRM-CM5} - # - {dataset: CNRM-CM5-2} # not in example plot - - {dataset: CSIRO-Mk3-6-0} - - {dataset: EC-EARTH, ensemble: r6i1p1} - - {dataset: FGOALS-g2} - # - {dataset: FGOALS-s2} - - {dataset: FIO-ESM} - - {dataset: GFDL-CM2p1} - - {dataset: GFDL-CM3} - - {dataset: GFDL-ESM2G} - - {dataset: GFDL-ESM2M} - - {dataset: GISS-E2-H, ensemble: r1i1p2} - # - {dataset: GISS-E2-H-CC} # not in example plot - - {dataset: GISS-E2-R, ensemble: r1i1p2} - # - {dataset: GISS-E2-R-CC} # not in example plot - - {dataset: HadCM3} - - {dataset: HadGEM2-AO} - - {dataset: HadGEM2-CC} - - {dataset: HadGEM2-ES} - - {dataset: inmcm4} - - {dataset: IPSL-CM5A-LR} - - {dataset: IPSL-CM5A-MR} - - {dataset: IPSL-CM5B-LR} - - {dataset: MIROC4h} - - {dataset: MIROC5} - - {dataset: MIROC-ESM} - - {dataset: MIROC-ESM-CHEM} - - {dataset: MPI-ESM-LR} - # - {dataset: MPI-ESM-MR} # not in example plot - - {dataset: MPI-ESM-P} - - {dataset: MRI-CGCM3} - - {dataset: MRI-ESM1} - - {dataset: NorESM1-M} - - {dataset: NorESM1-ME} - # - {dataset: ERA-Interim, project: OBS6, type: reanaly, - # version: 1, tier: 3, reference_for_metric: true} - # - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - # version: 1, tier: 2} - scripts: - portrait: - script: perfmetrics/portrait_plot.py - y_by: y_label - x_by: dataset - plot_kwargs: - vmin: -0.5 - vmax: 0.5 - normalize: centered_median # default - default_split: Alt - - ### pr: PRECIPITATION ####################################################### - # pr: - # description: Precipitation - # themes: - # - phys - # realms: - # - atmos - # variables: - # pr: - # preprocessor: ppNOLEV1 - # reference_dataset: GPCP-V2.2 - # mip: Amon - # project: CMIP5 - # exp: historical - # ensemble: r1i1p1 - # start_year: 2000 - # end_year: 2002 - # additional_datasets: - # - {dataset: ACCESS1-0} - # - {dataset: ACCESS1-3} - # - {dataset: bcc-csm1-1} - # - {dataset: bcc-csm1-1-m} - # - {dataset: BNU-ESM} - # - {dataset: CanCM4} - # - {dataset: CanESM2} - # - {dataset: CCSM4} - # - {dataset: CESM1-BGC} - # - {dataset: CESM1-CAM5} - # - {dataset: CESM1-CAM5-1-FV2} - # - {dataset: CESM1-FASTCHEM} - # - {dataset: CESM1-WACCM} - # - {dataset: CMCC-CESM} - # - {dataset: CMCC-CM} - # - {dataset: CMCC-CMS} - # - {dataset: CNRM-CM5} - # - {dataset: CNRM-CM5-2} - # - {dataset: CSIRO-Mk3-6-0} - # - {dataset: EC-EARTH, ensemble: r6i1p1} - # - {dataset: FGOALS-g2} - # - {dataset: FIO-ESM} - # - {dataset: GFDL-CM2p1} - # - {dataset: GFDL-CM3} - # - {dataset: GFDL-ESM2G} - # - {dataset: GFDL-ESM2M} - # - {dataset: GISS-E2-H, ensemble: r1i1p2} - # - {dataset: GISS-E2-H-CC} - # - {dataset: GISS-E2-R, ensemble: r1i1p2} - # - {dataset: GISS-E2-R-CC} - # - {dataset: HadCM3} - # - {dataset: HadGEM2-AO} - # - {dataset: HadGEM2-CC} - # - {dataset: HadGEM2-ES} - # - {dataset: inmcm4} - # - {dataset: IPSL-CM5A-LR} - # - {dataset: IPSL-CM5A-MR} - # - {dataset: IPSL-CM5B-LR} - # - {dataset: MIROC4h} - # - {dataset: MIROC5} - # - {dataset: MIROC-ESM} - # - {dataset: MIROC-ESM-CHEM} - # - {dataset: MPI-ESM-LR} - # - {dataset: MPI-ESM-MR} - # - {dataset: MPI-ESM-P} - # - {dataset: MRI-CGCM3} - # - {dataset: MRI-ESM1} - # - {dataset: NorESM1-M} - # - {dataset: NorESM1-ME} - # - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1} - # scripts: - # grading: - # <<: *grading_settings diff --git a/esmvaltool/recipes/recipe_perfmetrics_python.yml b/esmvaltool/recipes/recipe_perfmetrics_python.yml deleted file mode 100644 index 1d044a2829..0000000000 --- a/esmvaltool/recipes/recipe_perfmetrics_python.yml +++ /dev/null @@ -1,243 +0,0 @@ -# ESMValTool -# ---- -documentation: - title: Performance metrics plots. - description: > - Compare performance of model simulations to a reference dataset. - authors: - - ruhe_lukas - # - cammarano_diego - maintainer: - - ruhe_lukas - references: - - eyring21ipcc - - -preprocessors: - default: &default_preproc - custom_order: true - regrid: - target_grid: 3x3 - scheme: nearest - # for icon: - # scheme: - # reference: esmf_regrid.schemes:ESMFAreaWeighted - regrid_time: - calendar: standard - frequency: mon - distance_metric: - metric: pearsonr - rmse: - <<: *default_preproc - distance_metric: - metric: rmse - - -cmip6_default: &cmip6 - grid: gn - ensemble: r1i1p1f1 - project: CMIP6 - timerange: '1990/1992' - - -cmip6_examples: &cmip6_examples - - {<<: *cmip6, dataset: MRI-ESM2-0} - - {<<: *cmip6, dataset: NESM3} - - {<<: *cmip6, dataset: NorCPM1, institute: NCC, ensemble: r10i1p1f1} - - {<<: *cmip6, dataset: NorESM2-LM, institute: NCC} - - {<<: *cmip6, dataset: NorESM2-MM, institute: NCC} - - {<<: *cmip6, dataset: SAM0-UNICON} - - {<<: *cmip6, dataset: TaiESM1} - - {<<: *cmip6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2} - -cmip6_remaining: &cmip6_remaining - - {<<: *cmip6, dataset: ACCESS-CM2} - - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO} - - {<<: *cmip6, dataset: AWI-CM-1-1-MR} - - {<<: *cmip6, dataset: AWI-ESM-1-1-LR} - - {<<: *cmip6, dataset: CESM2-FV2, institute: NCAR} - - {<<: *cmip6, dataset: CESM2-WACCM-FV2, institute: NCAR} - - {<<: *cmip6, dataset: CESM2-WACCM, institute: NCAR} - - {<<: *cmip6, dataset: CIESM, grid: gr} - - {<<: *cmip6, dataset: CMCC-CM2-HR4} - - {<<: *cmip6, dataset: CMCC-CM2-SR5} - - {<<: *cmip6, dataset: CMCC-ESM2} - - {<<: *cmip6, dataset: CNRM-CM6-1-HR, ensemble: r1i1p1f2, grid: gr} - - {<<: *cmip6, dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} - - {<<: *cmip6, dataset: CNRM-ESM2-1, ensemble: r1i1p1f2, grid: gr} - - {<<: *cmip6, dataset: E3SM-1-0, grid: gr} - - {<<: *cmip6, dataset: E3SM-1-1-ECA, institute: E3SM-Project, grid: gr} - - {<<: *cmip6, dataset: E3SM-1-1, institute: E3SM-Project, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-AerChem, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-CC, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-Veg-LR, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3-Veg, grid: gr} - - {<<: *cmip6, dataset: EC-Earth3, grid: gr} - - {<<: *cmip6, dataset: FGOALS-f3-L, grid: gr} - - {<<: *cmip6, dataset: FGOALS-g3} - - {<<: *cmip6, dataset: GFDL-CM4, grid: gr1} - - {<<: *cmip6, dataset: GFDL-ESM4, grid: gr1} - - {<<: *cmip6, dataset: GISS-E2-1-G} - - {<<: *cmip6, dataset: GISS-E2-1-H} - - {<<: *cmip6, dataset: HadGEM3-GC31-LL, ensemble: r1i1p1f3} - - {<<: *cmip6, dataset: HadGEM3-GC31-MM, ensemble: r1i1p1f3} - - {<<: *cmip6, dataset: IITM-ESM} - - {<<: *cmip6, dataset: INM-CM4-8, grid: gr1} - - {<<: *cmip6, dataset: INM-CM5-0, grid: gr1} - - {<<: *cmip6, dataset: IPSL-CM6A-LR, grid: gr} - - {<<: *cmip6, dataset: KIOST-ESM, grid: gr1} - - {<<: *cmip6, dataset: MIROC-ES2L, ensemble: r1i1p1f2} - - {<<: *cmip6, dataset: MIROC6} - - {<<: *cmip6, dataset: MPI-ESM-1-2-HAM} - - {<<: *cmip6, dataset: MPI-ESM1-2-HR} - - {<<: *cmip6, dataset: MPI-ESM1-2-LR} - -datasets: - *cmip6_examples - # ICON - # - {project: ICON, dataset: ICON, exp: icon-2.6.1_atm_amip_R2B4_r1v1i1p1l1f1, timerange: '19900101/19930101'} - - # # grid wrong? missing data - # # - {<<: *cmip6, dataset: MCM-UA-1-0} - # # - {<<: *cmip6, dataset: GISS-E2-1-G-CC} - # # - {<<: *cmip6, dataset: IPSL-CM5A2-INCA, grid: gr} - # # - {<<: *cmip6, dataset: KACE-1-0-G, grid: gr} - - -diagnostics: - simple: - variables: &simple_variables - pr: &var_default - preprocessor: default - mip: Amon - exp: historical - additional_datasets: &ref - # - {dataset: ERA5, project: native6, type: reanaly, version: v1, tier: 3, reference_for_metric: true} - - {<<: *cmip6, dataset: BCC-ESM1, reference_for_metric: true, timerange: 1990/1992} - tas: - <<: *var_default - ps: - <<: *var_default - clt: - <<: *var_default - scripts: - perfmetrics: - script: perfmetrics/portrait_plot.py - y_by: variable_group - - complex: - description: > - A more complex example with extra variables to support different reference datasets, - groups for datasets and some plot customization. - variables: - pr: - <<: *var_default - y_label: Precipitation - pr_vs_access: &var_default_ref2 - <<: *var_default - short_name: pr - split: "ACCESS" - y_label: Precipitation - additional_datasets: &ref_access - - {<<: *cmip6, dataset: ACCESS-ESM1-5, institute: CSIRO, reference_for_metric: true} - pr_vs_era: - <<: *var_default - short_name: pr - split: "ERA5" - y_label: Precipitation - additional_datasets: &ref_era - - {dataset: ERA5, project: native6, type: reanaly, version: v1, - tier: 3, timerange: 1990/1992, reference_for_metric: true} - tas: - <<: *var_default - y_label: Temperature - # additional_datasets: - # - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, version: 1, tier: 2} - tas_vs_access: - short_name: tas - <<: *var_default_ref2 - y_label: Temperature - rlut: - <<: *var_default - y_label: "LW radiation out" - - ps: - <<: *var_default - y_label: "Surface pressure" - # sfcWind: - # <<: *var_default - clt: - <<: *var_default - y_label: "Cloud cover" - clt_vs_esacci: - <<: *var_default - short_name: clt - split: "ESACCI" - y_label: "Cloud cover" - additional_datasets: - - {reference_for_metric: true, dataset: ESACCI-CLOUD, project: OBS, - type: sat, version: AVHRR-fv3.0, tier: 2, timerange: '1990/1992'} - psl: - <<: *var_default - y_label: "Sea level pressure" - - - additional_datasets: - # - {dataset: ERA5, project: native6, type: reanaly, - # version: v1, tier: 3, reference_for_metric: true} - - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} - - scripts: - perfmetrics: - script: perfmetrics/portrait_plot.py - x_by: dataset - y_by: y_label - # split_by is the 'split' extra facet by default (set in variables) - group_by: project - additional_datasets: - - {<<: *cmip6, dataset: MPI-ESM-MR, type: exp, project: CMIP5, exp: historical, ensemble: r1i1p1} - plot_kwargs: - vmin: 0.5 - vmax: 1.0 - cbar_kwargs: - label: "Pearson correlation coefficient" - ticks: [0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - extend: both - cmap: "Reds" - metrics: - variables: - pr: - <<: *var_default - y_label: "Precipitation" - pr_rmse: - short_name: ps - split: "RMSE" - y_label: "Precipitation" - <<: *var_default - preprocessor: rmse - tas: - <<: *var_default - y_label: "Temperature" - tas_rmse: - short_name: tas - split: "RMSE" - y_label: "Temperature" # extra_facet to share y tick - <<: *var_default - preprocessor: rmse - clt: - <<: *var_default - y_label: "Cloud cover" - clt_rmse: - short_name: clt - y_label: "Cloud cover" - split: "RMSE" - <<: *var_default - preprocessor: rmse - scripts: - perfmetrics: - script: perfmetrics/portrait_plot.py - y_by: y_label - plot_kwargs: - vmin: 0 - vmax: 1.0 From 0492849499f86aafe7fc01d5ad473b9c33cd978b Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 14:57:31 +0100 Subject: [PATCH 33/56] remove first compare script --- .../perfmetrics/compare_netCDFs.py | 80 ------------------- 1 file changed, 80 deletions(-) delete mode 100644 esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py diff --git a/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py b/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py deleted file mode 100644 index a3a6ff1020..0000000000 --- a/esmvaltool/diag_scripts/perfmetrics/compare_netCDFs.py +++ /dev/null @@ -1,80 +0,0 @@ - - - - -import xarray as xr -import numpy as np -import pandas as pd - - -# Load the NetCDF files -file_diagnostics_path = '/home/b/b309265/ta850-global_to_sm-global_RMSD.nc' -file_performance_path = '/home/b/b309265/performance_metrics.nc' - -# Load datasets -ds_diagnostics = xr.open_dataset(file_diagnostics_path) -ds_performance = xr.open_dataset(file_performance_path) - -# Strip the "CMIP5_" prefix from model names in the performance dataset -ds_performance['alias'] = [model.replace("CMIP5_", "").strip() for model in ds_performance['alias'].values] - -# Convert all model names and strip whitespace for consistency -models_diagnostics = ds_diagnostics['models'].astype(str).str.strip() -models_performance = ds_performance['alias'].astype(str).str.strip() - -# Display unique variables from each dataset -variables_diagnostics = set(ds_diagnostics['diagnostics'].astype(str).values) -variables_performance = set(ds_performance['variable'].astype(str).values) - -print("Variables in Diagnostics Dataset:", variables_diagnostics) -print("Variables in Performance Dataset:", variables_performance) - -# Standardize variable names by converting to lowercase and stripping whitespace -variables_diagnostics = {var.lower().strip() for var in variables_diagnostics} -variables_performance = {var.lower().strip() for var in variables_performance} - -# Find common models and standardized variables -common_models = np.intersect1d(models_diagnostics, models_performance) -common_variables = variables_diagnostics.intersection(variables_performance) - -print("Common Models:", common_models) -print("Common Variables after Standardization:", common_variables) - -# Initialize a dictionary to collect comparison results -comparison_results = { - "Model": [], - "Variable": [], - "Grade_Value": [], - "Var_Value": [], - "Difference": [] -} - -# Perform comparison for each common model-variable pair -for model in common_models: - for variable in common_variables: - try: - # Select values for each matched model-variable pair - # Align 'diagnostics' and 'variable' names by standardizing to lowercase - grade_val = ds_diagnostics.grade.sel(models=model, diagnostics=variable).values - var_val = ds_performance.var.sel(alias=model, variable=variable).values - - # Calculate the absolute difference - diff = np.abs(grade_val - var_val).flatten() - - # Store results - comparison_results["Model"].append(model) - comparison_results["Variable"].append(variable) - comparison_results["Grade_Value"].append(grade_val) - comparison_results["Var_Value"].append(var_val) - comparison_results["Difference"].append(diff) - - except KeyError: - # Skip if the pair is missing in one dataset - print(f"Skipped {model}-{variable} due to mismatch in one of the datasets") - -# Convert to DataFrame and check results -comparison_df = pd.DataFrame(comparison_results) -print("Comparison Results:\n", comparison_df) - -# Optionally, save results to CSV -# comparison_df.to_csv("comparison_results.csv", index=False) From 946cb008408695c1476b06607889924d24fb83c3 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 15:09:29 +0100 Subject: [PATCH 34/56] fix minor codacy issues --- esmvaltool/diag_scripts/perfmetrics/portrait_plot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 1b18ec1aab..cbf0a8072a 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -125,7 +125,7 @@ def get_provenance(cfg): - # TODO: should we collect references and list them in the caption somehow? + """Default provenance for this diagnostic.""" return { 'ancestors': list(cfg["input_data"].keys()), 'authors': ["lindenlaub_lukas", "cammarano_diego"], @@ -204,9 +204,9 @@ def open_file(metadata, **selection): varname = list(das.data_vars.keys())[0] try: return das[varname].values.item() - except ValueError: + except ValueError as exc: msg = f"Expected scalar in input file {metas[0]['filename']}." - raise ValueError(msg) + raise ValueError(msg) from exc # iris.load_cube(metas[0]["filename"]).data @@ -281,9 +281,9 @@ def split_legend(cfg, grid, data): axes["twiny"].set_xlabel, # top ] for i, label in enumerate(data.coords[cfg["split_by"]].values): + nodes = get_triangle_nodes(i, len(data.coords[cfg["split_by"]].values)) axes["main"].add_patch( - patches.Polygon(get_triangle_nodes( - i, len(data.coords[cfg["split_by"]].values)), + patches.Polygon(nodes, closed=True, facecolor=["#bbb", "#ccc", "#ddd", "#eee"][i], edgecolor="black", @@ -519,7 +519,6 @@ def sort_data(cfg, dataset): ] new_order = mm_stats + others dataset = dataset.reindex({cfg["x_by"]: new_order}) - dataset[cfg["x_by"]] return dataset From 7242b23f6af7090e26e44c2398bbd1453a102a01 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 15:15:34 +0100 Subject: [PATCH 35/56] setting trivial doc string to imperative to make codacy finally happy --- esmvaltool/diag_scripts/perfmetrics/portrait_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index cbf0a8072a..7dc868d666 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -125,7 +125,7 @@ def get_provenance(cfg): - """Default provenance for this diagnostic.""" + """Return default provenance for this diagnostic.""" return { 'ancestors': list(cfg["input_data"].keys()), 'authors': ["lindenlaub_lukas", "cammarano_diego"], From bc6d2379a5cfe2d6519c0e1a55ed84d118c289c1 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Tue, 19 Nov 2024 15:17:38 +0100 Subject: [PATCH 36/56] simplify docstring --- esmvaltool/diag_scripts/perfmetrics/portrait_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 7dc868d666..54c259baa3 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -125,7 +125,7 @@ def get_provenance(cfg): - """Return default provenance for this diagnostic.""" + """Return provenance for this diagnostic.""" return { 'ancestors': list(cfg["input_data"].keys()), 'authors': ["lindenlaub_lukas", "cammarano_diego"], From 4712895352dfabb5438131b0134d02fd5823a8e5 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Nov 2024 13:34:17 +0100 Subject: [PATCH 37/56] recipe documentation --- .../esmvaltool.diag_scripts.perfmetrics.rst | 2 +- .../figures/portrait/portrait_plot.png | Bin 0 -> 228922 bytes doc/sphinx/source/recipes/index.rst | 2 +- .../source/recipes/recipe_perfmetrics.rst | 7 +- .../recipes/recipe_perfmetrics_python.rst | 146 ------------------ doc/sphinx/source/recipes/recipe_portrait.rst | 107 +++++++++++++ 6 files changed, 113 insertions(+), 151 deletions(-) create mode 100644 doc/sphinx/source/recipes/figures/portrait/portrait_plot.png delete mode 100644 doc/sphinx/source/recipes/recipe_perfmetrics_python.rst create mode 100644 doc/sphinx/source/recipes/recipe_portrait.rst diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst index a4936689bc..7dbae3e390 100644 --- a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst +++ b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst @@ -9,7 +9,7 @@ This module contains various reusable diagnostics and plot scripts. Examples -------- -* :ref:`recipe_perfmetrics_python ` +* :ref:`recipe_portrait ` Diagnostic scripts diff --git a/doc/sphinx/source/recipes/figures/portrait/portrait_plot.png b/doc/sphinx/source/recipes/figures/portrait/portrait_plot.png new file mode 100644 index 0000000000000000000000000000000000000000..a6020c2d3736efb6bca7d25dc81ef69a9a4f90bf GIT binary patch literal 228922 zcmeEuWmr^e+cpNMC}JVf76z$=G^mJ3=YX^u1O%kJF)&a-kxr2wVx+qiL8N0CrIDT? zq#M3#0ekCy_V#_=@5lG=o8#CUVP@94*L`1cUgs6hamNU2GQh=|A}ZrxBMBHC+6 zM6_FJFA4mm_FU&N_#x_~eA`CJ+`z{E4=a5lB8PBmHc1&42^ob84J_627G(2eQnmZ4 z8yXZYa5i&twQ@E}Nw9KqaW~1zNnXCfBX>paG6_*22l;hv3(_LE+-2l)5~hC`>fg|} z*0VCQurV?>LoU02n#>M$;TEf84f_|FS}9r7XBpOuU!+NhbT97pv3bO2O0et`3Z}9H=<@d4|$f^8%IUN4~H{aTg|J!XR{-y9(9mj zFfr4xfr*b#?b8cN)zRiSio=JaNyw-Zo#ya%L4S#Xg#vP<9&!@5ZZ&4;*UZA#2_5F8 zC{0bxG=e{?h6+y0J_5cHr(NunbmRGv{IpEBHK%M7+{8N!Mk80&v?GW6qL9X9@j@gC zIlW60j+~bFl}X-Netvas<5mk3la_ezHjGXbVg1aRGu)=_Tyl~DAN%Cfbw4R*+?5D` z8+cQ5D`pzhwWezDU{lmuVodKU7dzRmtu6Fa67+{h6QbJA_zL=+@7M{CxJq?u{vkUX zo2*8z8MefA#b$Zdi1GY+Wv)5P z$kdhuIhs`G=>ZAXsZW&4-(RqE8PuMbo9NO|N{~gq&w}H$pXe0aXIL8{u;7>bS|?mK zp*~V58TJz2tkw5CePM%{rvp0 z?v(GWixSZZ<1trJS6BDp*0RN-jB#lZ1-68Ev#x?cH9Gjpk4U1z{qS>fnSik!3{W** zo*k_$e_4Cv=+SJr05(-awZGC^Nm0=+Zro+INt_=vnGu`ezTxJ&&}AQ#MW)kh!=G(X zN1bidaufcd<3qU*(b$2A$B%c*gm8$V(8^7(PKjo@ELr(Jq-x%{KKAhmnUW_d1=qdC zlWTK0ogig7ZO?|a%N3EA%Y@B3d9i6)8gv47tqVQw?$s-cy^qM13++t1Y;ju4eNP#z zgods7Bib_!eY+j{$!sUP*Zda9L4H=tc@SOJ5Sppwbb@sb+VE$(7Tb(A;~S$N;jzgH zR)b%XH1nIZ8z2yG?w{$tcUB3s{I(L z`K)3C7^IrrDQVf~t-mEn1~7R&d-l0`!iJxMk}@Gi+!y4VuLGy!G|CvkXMGbk=W@3> z$T}+97$wE6WA{xzunjRet5}?ld+I6CCnG=lJ_Z$~?msmTT46Ff(&RM%%@7rP>u~?( z_J#^M=ZN=wu)IvnT2W#*u8tgYgU~|w&Rx5_b6)%Tz3cn<)THl&r#AWsT(#seV*cLO z>G=x9Cqz0ib>R4I@v&9(&eE*vS)b>2;_|JBvvfa^7BAzIk@nUP`DF5;CcyF1aD zUoaGE!sRr3FJ`UViID2$<&`0S85Sa+cSyIveThz2PF7YW*Q_hcpe~dJQJ^>mwKv3ZkxxBT{bJ@xrX_PG*nbc0+U5b zpS&nClO*9W-rhM~VdlGfFulidI700qRdjT80bFkG^BLcPXdk0@a2y&Rh6x1l$5XV@`awaa0I>Lh&S|b z;eq32WMs_GhV!~N-ku#;qRx{E`&SprE%o9!tKzL&&zcHAhpjcV-`{zlsHEhbV~KCQ z$f{E!ST(-<1~dz;5OY(lT@|BkD#{H{a<%-%vo3c{$!BYb{fd=%)7kDt$)09QXdA^R zW3Hrq>a#e3L6pS#5aPX;j0e92EKd|pW@MGFFB1w@r)y?xbk>&VIJ<{ZbGy;tBnkq3 z=>*~fm{qAw^*|XUB_u3abzJAL#%(DfO;=p%Omn*e zV3u-os5lK0-PV>J)fwPz7jneyzrtv{OnqW81*578qIN&=TrxZ~*@;VKsdclM4;-aE zQS7|h@)+%67E=)6_gh!)JykIgB`rGkl$@dMgXjLPc03M)ah}LO-(UhykH4-389iJ3NG@Wdagda>x$MENnF1DgYw-f2qbGY(G;-ObGze{}6vvL;73Jou zSyk1HTt(dmA0Z6pnl`r|V>i(m5|;|*_GZT-0aH2)3X}-vjaJV_fB5j>7(IO&8kLNe zcyo@LNIJvck zy9;AoCYa*3Cz4jUQg?TXi;5N5MJ6~l({CA!Rkp(Gi+Z`GerV++KeK|-bLIiT7~!RX zH!P+ypMwX@WH{tD(7azp;3sJ2_q^4DO~#dhno1zSW>I7^Atd zZ%UZr`OtSUF*7M{@cjYy&86$a#>S4oo~pHjtpLRXDilM zCQEf@i`_TYa`|mW-m0Xk8_kb>YcH@({{H>De`qNB&3QTFhDf31*%mo&yRqAds|90! zDi%@yr(5c8Ino$?-#kM#QNH|%vEZ8}O#X3p?Uag&iu*k@baWAe?A=ce(&4k)r)%H{ zZ2+9<8?l2`ahe_Zqo~-!g9w!nXM7md`_L+PEzE_c~=m`X}X zaduO}e)JMRpxHs7*Vk4uvp@hPqFm=gsIS~l1aRk+)svOQg}@hY8lE<|Q>_wL4L(BKB*m=QtVWNxQ zZ8c9efIu@S+ql^m*~3rH-d3B~B`c8V8u=PkZ-Wmy#eE$P&^J;85RZaqsi1Q)Z^$ z)=;5%D#E1B^j=(SSDt7mB7TX(-7mVKGGOUWW=d z@>&l$+ihSb3nzsU9}uXKbuSuua!X4~FCU+_NFiqg@tXFOXz5k?pycIWbu1eh8Ns0i z!&ciq7eOH9^sY^N%AQeJ&AvwI@k3wq=|S6MOHd}Y61&AP^O7d;1LNv!+C_Oz3lkCM z+@XjTx`9t-*~l^PNgElt2awvZF!EzxWAJ4?F8c{h-}O>&D$biAB$v_S;56C*I3(Wm zrq*{fogSzb>_(J;nDmrgPv1DKe^`y`5!IV>GR>9VGyq_EJfA!%D8~Hp@d<+Pvcc}N zlq2ZW`^4oBKN5&RtmwAH>hSU7fu_#)@84fPuA8WyZEW0AQf%^V)P{+P>C3oK2>Xrs z(H8N(zP=>|?Gg)KS_DgjkR`xT8!nlo7O50DSf<`iKP|-octn+1CFKXc1Qg3-72Kb_ zgFuB_`UbAqeT@%%fTl?ifFk5548Aph?Z^-mM&j6#rwE?=OnLb5;cq!BPQ0RDmT_7N z3JQ{PazV2_sx5BIqX~Q^JvzL!2Pv3Uq#?{u1aUxsbTN2IB)jbUfzy*KSn8hT#)v(Ln(r`0TqG2U7PQqH%C4?e{$1t ziGk2$tTDfX!~Q1k9nxV|ST~-mx3M(LkgAq>({C0GPT?9cDuUm(^_|#@Rko(jp2Nb1 z>p_u`Y5y6o(D2BDhGoUogSs0U6*f-7U9Mic`mAeV5Cs8Cx&O6CBuIRBliQn4;EGho z@*KfCXLI|+k)VCx1$bi=#8y%(Adlf2pH46-#y*1&4Yyq}?-rC$+n}!1Vdq_b2caT@ z^UX~6kke-+QNvwMPJ-d4>pqwsbW!PC$9$MsL3Lgm;yh9(8AnH&Vvv2C%7G8Qa^I4J zmK3#27E_V+#Tc@H;2PS3#ep|+S!P}Qrd(=f|ttWlk(=d-#|6U3gIbea>s*KuF7r9ofjvFI&( zq0P&zQ&MDBvNYt9mUu@BLU@sl)&4dWwDLvbJ$pKo4m)Lz{vP1 zm9=n~36aqCmxrAD4jn$+H_v*8l9tz^ta%<>ZDtmr!usaT(-he{O~e9Z8)KQIRuUCr zYz98R7VN_9VQ_ghh{<8Yep<)BC?%HQxc{R0Ds=G6hCa=9+snEP*16uYmFd49|?YRSkVV-VI{S(t3F z?E;hlcKrIr>Wr_3GXxGxfe7LkDciLH;$<4?GA-fx^r;Cr3AGYGTXC>(0 zdg9n*RXK#T`Ie#2D3P9?-d^fn>b5jUY1*D8FIf{0bi!ZEorgdTp4}K$wOqQfB0{`-VAbCK`Sa)M>AIfUu5&2UukXvl^gmt=yJEU& z+!wl?({g}pK*MrFA#~@rp9mj!0t43y<|}h8(AzN5@g@YT+Q|sr>G)`q6+j}b)+{3> zGMXza9mDih6&3e@n5gU^+rPhMajG8$3|P^Hqd{e(HsHv#+Oti9rV2oL+5kgh2x*1& z`vVY7D0zCwxtzy&{yqYY+KZiYlgB^5K7)kTp*-ekKqDoA8%9jA54{i?qIX#xx1~5B zk23F@bmm!7U%GSEEfB#vTU2X@ExuL4d0_&}*&rB$({`4DZ4b;vrQE(eH`${DPCbta zu%x}%X&qu>;w1Ntb*Is|AZ6ez3Tu$XOj7{+mg+F|eHqv)px>Mkve68)wmQ>*Y`#!? zJ*=&GJ`u6{1I6K`D+gikqmF^AloMWxU0++!0W&0*+iCSe(*HtVQ^8^YGJ9Ek9D)`XZeqRx^T$FGnA&?0rXgoFGj2se>OifKm0;Mwb{SyjyA>g9YO*l?v zFlS?I6nwYjq8I|7f%KWFf#?MX#&--?lQc4Hx?kx(57D|qvEJ| z<2Ta@TQ~b*Kl?q#krl8ul5+%{+|hH=c>`r|PDIlTh*jHs{WuB+C=EC`K-zB2 z-NnunM~|LKXi4JG|B?v49l^A)-MaSA;nR7$|05-EVUv^Dz7qT(;nqoy|26GOh&+u`@-jqQ)#Yd9;g zPRiiuF_S?vJn+t2i%f!M1)FSw5(2#oCrgsSz$U9@8rTKxK1t38Gv>3JZ$nd0{f#;=ZQw?xjISD5i?C^JAMusjI$(%2u5()TT+yU`oD2 zlRP71=pRpRD+&L<#OI&5Arw6pmu`y6cCkd>JFnYd^rh3A|9DuUU-SbnDOu~{%1Og{ z+3=Sw{hWcLU!QZnG$OI0Ziy^R_d)v~$V}(&v6=34l+h?vt!ju+t^RT76Ju?C}hw z#Q>ZY>v3I(NKW-07x9_qIk_p)PdH-Wh`H{yE?6L9`#-|r_ zTfLFNF2;-QAt$tjEfXrHl+bF$McUz)<-1lyCY_3HI|!(R?tm*QBhKq zO?nDS@VLH;vza0r{OI(fF*Z!+v~AHhNAjK32}9z-w#D67`_L(kFU;Q_;!m1!lH9<* zY*yATr%tqT(tfIi$u(CLn=|oGPru$X(~!=Ix7ZLz4wO31%ec+k7Q6CMM-wx%5`q=k zh*Vyg;ItX5psM8BmG5n2YT}P7wtAev!G9tsv3`H<$YTBs+R;&>m^a?LKu&wYDMGl0 zNiRy@p7u^;U~%a@U1P<**Is-S$;@f!xbsmFDZ*V(^RH&>+$WrBR%B{M# z&N&UuFqu~uMSZbNQ;9fEH?7~ZetM$fu$YkGjeV90$_3XcY^m3t9T)epEbbn7(uQ`^ z*P>ybmD!+Ql)?xLD;UaFCEgDREif9m!5YfZKF2>3k-|}bI2o6SrfQH;*w8+siD@xZ zYcg!R8?I_--)T_t231K=q)1l51t)RNe*LkYh8xXHu__D{kJ4w)b>yD$N%5guRwbV= zMx`gn-gRc_g*6XdPF;BBJd5{#o`#ztXr2`b7j#sN5U_7!FI}Bf*XJWP@?#=!c#d!o zn>>)cpimfr}nyyk#cabQpnKq(4 z_hL3Aq}ZKar9&gKHTp~f3ZwqGt>ml5;NlA`t=3)xTAVx20KJlafb!ce{iUq^0jI-4w z=#O#w?225O@m(tztaRhI8c9q;JA10Q%su$x&-gj`&C954rs#6@+qngjd#2Q8#3&z9 zCHT@SkCLLe2EJXOV6W5Z8y@P_amTL6Xri>OFXx zLDM36+&~`!VocNQubF$%y)o;$%K&swj z1NiGQ#=<){1HMSVxAyC*rgaT0>~bbg>PnRx{hWTAJ5k}LiFT1zkfkV__%~wCFF!s9 z(dtvyCTczm8|15I3oZ{16sY8~hPTi^Zp9;S_42+dEOwqxT9H|^8T)LBO?3|l#AwG( z2U%X`IuD;?dNK`L1OJc=OulEYCOkPQvVeaeoXa${csT&$=1J167O=SlttuK4k<>7tN1Qw9v0oa*yoHjycLQLKb+hMEHR zs6aECY9}?ObiDvF;VFG)p?sUa+|D$qqPLyox?1HMq5SdY-0W!3Wb%E!%Ixq%wLchk zbEbSNH4LI3>(EM&qgV9|wdEQa%_n6?T`D~-fH}l*Njyqzrl-j1i4UD`f8KcX2b-gu z7z@@0>LDz0BW7v6_CQvR)&g4-Iw7Z4Y}@TVr`rL_TtQWVSCsSU74#|UEKKYnvxDy{ z%msSBNQKj*NQ`&ag#q|-|K0k~9=p}lgF*%)71f!>PGXL(YcA#itkq;QusGx-Xv+_n zoM_oTmQcD5`(*K90o&<{i`>S|SEJmPO1}6HlH8ZwYp_WEkg9Dp+vF)d9;XG?2(3Hr zL_J%p=^4ag3bN<@>H*(!ZRkc_a)U+Zy!;;p&qGwV#R?9ZBL1=|p7;FG155KhC@5xaT;wNuRdUM&hdCb@~!zx-* z+&=>`1D8I!AF6#n)IirUcc*m1>jht24d&k4Yn(%g_ofu>f(BdmQU%8}q*U`_hVsTk z{^c+Xqg>-5B4s*tsvO&<&1mb(bm79s7)e0|nds1WOiVE+RXZR+#6r;gsiI;)t&XS& zVodp4+s{#YU*N5eP?SpTsDqY9!b24i`!Gq$;d2JC!H7)3`ng{kZ3|dkj*h!(1DieG zMmi-_yX6*xdpZP2L0P{%hHp=yl1I}*LZOcl}@GY_0Hh7++=M1#_ zwrm_MLp8WE%X2>^8j_9>U)W`Lr8HB=Pm^9cV4x|la4HndgbwagFGpBoU`IzFN(|EE zBto)t$F9RkT7quKm9EFpm^-T{=pCJPpJOt{rgSKyPw-r}zP>3>pxG&ZNONurm)qJB zPgk4mTV1)Tq$OLK*{R{lq`OH+Hn)09sK%DYRb|F{lnncIQTeiG$XxJRo)mhTpSLv} z&=GB}kW#Rj?hr-VpxVcxc92-N`=E-zEe4Q2|VGH&0S zU_P$nTt7bjB}oTA~#( z&nH}_=_vLMpYT6=on&Z)(}bfwna|8Q{!&*(Ewg4#<`c(kL}*ECf`@GQ=YTFSsSUnt zHZH2Hoy&B(VO~k`T4>6b0>zD${^+%pg=F9|K&Dh9$M;Y({XG4 zXx|wS`gz8quQ`ccw+&V#rNvDW~ou`>wFChdwQE8eNDO#c7Qh|bLc6nd!iC1 zgr6oV_KG(XVaCuY(V^+k@BBqgWfYb$RPH50gP~E^N??qtQ=0nzi@%@~F>U*}%Xw7O zr3lSkbaPjvgHp8EM+6ic-n6xUD#aiu#rK~|al0}L;m4@CC855_Dx0__dS8Ji>tM0@ zyL~7RdU-~z>22e?s=!&PzbIU)a5EXD6>rC=7RVQbfIqwUa^L`_v+& z%SYoMZki zQJka;f%YhKn+OuYsVCPzo}je4{<)0S)u7IvW+8EA984`DQtvw}YNh@rQh`wWvSSy? zNQ8a2Wj4iYQSv0e(CU>32`Bk%FidEfCxj#Ll*>#wpD2BK}_4< zN<5mBZ_eM1z2AI&?Tn7VfP{AH8^>)^&@fapOodA_-}jmi(Lt;z_WXI{EySAyGBM%Q=Xt;cB)nKyhUCE5#x zW)lnHM~*7BE*0RHXOlE2j#JS)GQFV9PjBco@Yec&8Xl7Y;zupQvE4ZGL@T{@K|72u;X(R$F@tDD`3Di|DbA- zjTB6pAMYRlO<|oKp!7aI(F=W{HK%~qDly^9y;ooI+qXju@QiIa1w@E2o!u6ux%Tk_ z+p_(VlKhyo4OLMF5z7}|k1dXJ{;M>N<9dv)VqQy1O18;pwVRYT^Ppm%xl$Ia?%0Nc zQ6*Y2RZr$rC9V9@@^ZYFh1by!MgLNZ-#-Dc<*|@`Bn0WWS!SaYpT#F;uqi$B4p@^r zAJBFW_3Dgh2b6H6h%>k<1M`wo;PK%lxa=|jNC@=MahnNq>Rfu~_ojgmNH8(}mK8<; z)xoq|NcQje$y^14izuE{w6gLl= zj7v!9xJ@O`3+26}hqf*33|0ec-Rn-N;gp3=je89igA82grbfSia=8xir{J^Iimkj! zx`oxNt1lMhJD;!9n5CwCWepg7QwL^3v7t`5s}XV%DiH(${Jgl%+2Ggv|K{EPsl)TY zkR#p;65@8*SXA)2qzo_S#*zSKcU!zMi<&KZ`0#%r|By3HL68uz-f21KNO3n% ziV6&52AYjm`$A*CFopKD{Xp^nK)o_wyjWfN#p8n~NDtAr^hCL=zi}dW6dP_x7@f~i zHE!zHqM#kd3PZv3j}O#tu?p6m^U{ChKMx7|1$zfxHid#T{MYoDKe2b{+Zu}R=W{hM z^1<{<6ak+PomB%8zrb!B4V+**NRPu})x90P@3w8JZ{S9v@JB;tZf+UZ4xKnpU)#QO z25kR^Sf}MmT7T?VVB+RR6;I@$%*BfCabc?#m45fn<*HPp&W@9)h_ivBqeQv2l{jXcM+*OM2dvPeQ49u zIo|g6Gja7MY##=>aC_qV8672c^#T%Gbb=$f{rBG-hPYIq$-vtws3KA-6>uKSwvX!H zX$8l&8Of!rjh(5>vyW|VX>Lu_BcIE2C;49-1^@?>#<$ntlGfRC3TksGKE741V9RS% z#3o+o#DeQurUG_j@=!l%+grAShItq1m9!gY$sTo$u5l5+OO zw}zLU7NTh=KL5s`#=Q{6jzljoL#}6SPgk01cW6@CReEm!KVf%|OObMe^oh9HSsU(> z^n?Pr;54F2Pul+m_1H@muS2E~WAg2kQQ{YZir80GWqRi5lQJs$T5%g%FS;PV3AGO9 zdZ?@AmDqb_4cpp0BeE#_Xwp$?bX1mXvhC_5xH8p$0Y*cQh#5vB`^YcAvUfe5Z(=qV z{`iX?7=POn>bTJbr7Ef|^~JW#u<`%nf+1i#dR{h+NyEGoHu}~sH*hmZXcMxso=a^vDF6lB)V2hy)6BXL>E7NB3&f0U+xnmJS+}n` z+Icec0_2RM&+HlbgfqBs(xavIPwryLs``4JlSK;n_)z=tn6OFjyPBVgx44(zL}rfC z`vXLI5+8TzJIEyNv?EU&+S z46PnD(E!=IHy#X#=Kyfk?WNvd@YQP4)I0zA=nk+BTjuO1sdIF${xptwps?~l-_DFZ z+gjtY?&i-NY^k*%u+PB3V~g_G3XA&A#q<4)PG1@ky|M>KCX6^KI1$J3E{yHMu9>qE zN1b@EFZ4RBID+>7%S~q{!X`%_I5WIC+a+X{7wKk3S{jvqavQ&WTeY8Sj#-fMTgMFF zv8CsJZh4CGSHpjR=W^XoOQR>f%cJO%o($;&h#*bCLh?h-3R3XLjvRUS?AhMU4u{UX z^ba2)HabiRUY_>6bz9N$yehTGsE|bSF4s__p+uGN+QEmLoC@sE?Vkio(b}?$Oo4mo zS)(ym=oBI0xnnK5&8)m}G$&+#7C$S2TcAeMJQex;9|!X!EO_XrbGjVA<7)SraD}T- z7bOW=`R+KIW@0H6lE0hWzi(d?^gLi7ZHv|^btB!+St@ZH%FvBk;`LY5WsiOUr+g18 zWzV7_x!`=sStH%jq0q}2jOVRk_x_0#%!@e0ePvuE{kYc4n?gBUPygKI z%&{)q+|caC`uBw=9pZ`ojVk~!ul)+nERMF&{uTc6RdNyiIQW5zxG|j1n(L1byOApL z;9jLtQb4mg{Qw+?D2r*HUgo9wZ0sQ$v8bJD?Y0R;0& zIp4VP7-}R_kg8KwRvDzv005>f)GN}UZy+HO!W@RMV;W=62j23dXRwKgKWJ25t@*D` zzowx$={&u&ir;KwO#kfX{-cX)wZz*C{bXL0Vjs0MNyFPQTdH9?Nn7(@S89K_{=(JG z0`u~h$63`!s?SF$L$NW%p+m3qBYNCcWTb-Fu#m&lb1a37cq25-oRvVzNC6#dVxXq9 z_Z5Kf^)4(k}@*3toEbLVwjR;bd`?D@#Dwi-(NBd{pPu8+G>IP zk#U~a6Pg-3?-xsT#=P<48YVlRsb`D+LxPOU@oHck5XS=v?*`$ej7V2Jg+#Oe=uz&z z=t9%9FAIk^d2-}Z#S6{StPZ247~RQBv%=X0da-q=?WJ0WY!ZPVWu&b&YdEWl+Nt3P zPqaYA5QsJrQe1Sm`UuYI>QxVgOS|T`Vrb-!N|D;o@v|ZA$7QX|Ec7)uAGzXRr|)yQ z4!I8Os5O6tlT-T(r}dC;ezBjaeD8Ix#oYhf;>3hRL?=s^dsNeVcvp9HH9F`0_r~ zP?WQhAD8RUU2iIp4@2lbB{aNcv^Cr?!DNcSF5zcJ&8L9txl+*yT9d6w||g%(v20kCo9fVq$4&{A$s&0em}HuvEB|Gu!N$ikqFCwCBPhmeJBIZtF^6JbU2n4)ZTKQ8C%&Rt!_ zuYWIVp{tEH#oPjtezTp-E!<;i0^_V-$hRV;O+}chh*@@+(C~{j(~BPo4O?l~446D~ z_l}P+XOQD&9<#H8t14U2x;8l#p%^cd5siuQ#t=antzD4Ei!GFch<`u5qa3DqCHw5C z_lm$snnqgA@1@JPQ$%L6WGeFPsOL91qtDOQuc!`!u|Wa`gN0s$h-?~87*YH{<2_FP~_&iAc$VF7M2Gy zsO`SMH^^CTmyVAI(VRoC5qExi0&=-cZ(u#KX(Zrz0n7ek zBziTFV}!Cb1^LE}NckD8s?I2swdJ%Pp{B;?golUUm!c~{aH8x`E!a7eiadKR`@US# zy%zV@W9z9Ml!SoIcl;LxMS65BW9ZvHB*u~M-P;&uts@_`Blp{;93Z4+uoTob%^aP? z21y>LMR{r zq5W&IDSN7n=bkcLIC)aO_*{K6cHAuIzc2PEELOKUR)X~4dXHEMQoaSqMjj4JoN)f! zIVLu?2|ogqou_q7Lwllab;qnY>?LI4*np9Na8zl3pqc!gx1y6GZ7A1v9sMoMKy(Op zP)|A&$#}spPzGe%@P1F0P1AYk<7pL`EKcb#8z#ALwyojKib~uwC53Z!RZ{Ew4Fox) z$l7hv|NG)8Ve#M5$!X2|*^7GH=Ao7nsfC7YPnVb|HMB{Dos)}9F~*{eAO;E26fd+J z?+j#c8aXAjGM0&S%_7Vu^s(Sdp=wuqdfjCU_=HHw$h5pI>}hj4urxxmZmb#^n%Ayf3td^iKR1G{`$h8AjnR?$=2>lSInp~GT*I%8>__*YGk;3H z86x@g{r`~sj)!AxM2?1IUpH%>0hWrXc0h$@D^%1XC8p3NrvOFK0U*Z^qThqwD6i+w zt6Qj%WYDz*Zj$B8MY(1vmY0`@=HAOeYPYsNjpQNoTW5y3^;Wf(CtfrhJ8>+f zgTeKht-8(opT}s5D2(1e6@~$hambTWTOJ}Y8>k?I%!zf{Jk$q{LK{+b5H-@x3pLC} z&=a?mA)>SG^PpV}>pozzyf_ne>*3@|1*8UwM9rlcf#`G7Ol_UKH^c{2b_|7!mk8O1)5bP@_Y zitdbtXL$>sH`S&)l0q!i^F42Thna3DR2}gR zVG#&b6-+|39@E>+%0BQVh#fN==`4#hvfGV)jfQ@sk&5R>ygr+Js`si{x>yxYl0EqT^X8~w zljJRnwH+1z;${a1?vfEiRFX?>$22w`VqN@UYFm`#{flh*58Rb0HB7wXnkjj${1l#Z z5T(^r6!715NC&*fPBLmPd7$~n*kZ13V{-|Ki+G^{{nXWE9*TYYS`*}Kao%;wTgLyN ztBZ}jOE&79TcVwsFKyR*NnawdriO;+IWO^Jg$4AyG|j zQ*|$O39sSlNzsHP4Iv4-g<-bbfrMmpYat0f!5nviH|yE5jH?WH*g##s{CO8s+BWV5wkO)9RboZ7Udv?rMdFII5ZkKG|)E45rse}RWmEBXPK-J#lmu1 zr*oW2@OLR^t1IO(BNv(lR720;n)C_U(r4IH#1z{UuSx52CY!nYp7Ic*`ApJn;T@ zD(bJe$$tKR(Q+rhvc{FQci$PkjJ|m-*wooVk;OLu)S17S8OpVIaD3JA#q}4*TO#C2 zVwh}cHaC4MMAG^|ek@V>b$@ZQ7IUzt@kU4pkt*9t@r>q`<^*eTKF*k6PxI|AHwgvx4NK zsK{heZo;M<{LjL~2Euf1TfCKP)w)^?yydT{|+ z^Sr45D*pas;WevhJWq{H9{USA|E?=*^xY$J*#vjcGl(hXnp^``AB*`3?AZDd9%cAHd6ZX$yY?Ks z2mC2Nlwf`DM|#=BUJcHO`%HQ;0r5Z+jd>9=&gn+<5}!_}}*#oZz>bzzgqsh)g?X(78@%~7)q&dNp7 zj$|QxI)ms|q^dN?(&w``V?(3i&{nx3Tfgl^GhL@ATfCe#{_AD<`iT~g7Z8;z!J%Y1 zOzGy%Lk?jyKFq9nJhvwen%TbeS>$O$7C5-TeliU@@CNMj*^t3QfLnDt*84gfQudJ+ zC&UY5j#kN8>gx7(7oNlah>n>Jc@VAFqc~gs2CbPE+*agJXspVlZ!h82XG*8^QZfIO z;hn`y9YW8Wsok6h4iqlXtvT~T=?W3}tQg^*G*QDhurqm$hO~PGdF%#6`4<;T3yKuQ zST~Gc8@Hj&kPCe5VD}3NcQNjSc-0SMF)j;$%l@K9h9%gw(%XE(4w!T zfAZT?oH2V1PMqAdo|LXX&Q_W3|A$>DhO{7IFi6MFWaY<4a%99w?wu!Rkw#yWwiE@J zUZiP?%#{HxoQ`XI8R_tS9~5v)`6M`*!?+=3}q5zh|@sx%KTGRW}`RxST zCaheV>x-Xu2G-m-7Smw$I=?}ZVn0@DAqP8}DY5L>pb&nVw1E^Z8;L&BE^F{jkX(vl*8oCz} zjp@O=kq^-Bsnr)g=+A<>E?0UW$I8-XZ*=m(Tk!4^>zqYjWSn z$C-xAba(0tPYy1$aMJDvGBiYBx?xA5lm{6oNjx%@DDde4s&~+it~0y##~**dfG|ZE z6&2t&-IMJHvu*BuuXv7D<8-GdEP^R9g8tLAL8Md)TFk41TtB+=0wKHEC75MBEQ`#z zn&3d1{L0Lcwg+ezU08PGbw}E6Un#gnytpAIW(3n}{9@(#xKvdgT%+ukRV9*3KH-1ikk#+|Ez) zLFj&%J=8ClI<5=jo(4$`Zrr#r0z)>w1g(95jLt9L{Ava%6SEo^12N~&1%rAnUbyf% zyU3QH0>Lp0MzW?2O0T*~==!0pS#@c8u(&{O5%adQXXml|S~rEz)ZUoP!UIvqtrrH9SLipszW$HKQNxz2a(0g~TTUqyYjzo3#Uqt>Usez)8etu@0z!#oX% zMFypY2kZrblrD?$h$Ydv=5}X8r-|BOP3T=c*}x8GW4jU zI2-P+r!<-8YZT7CJy)x3Zci0n>gX}UW5TgOGZWEi$)?LTV;LqqBR%96HebIJg^mTlJ$q2PA9U9I5Ki_RS=nM z*b3E-`UnBrZ~EO!LI?|tSeNi{771i5%aa3C-veBu@0<*Oa2<1Z$%{!R^^OQL3r0*+ zJ3yFIxlr*U7Nbx(!Lcek9vYP77V*U{s81E1G>v~QEvxWA>4)K3C^{yq+6f7tp~Y}D z(b7S)p6J_!Qm-nO!YIN^)jqmZ(N>uv%{T#QobntqphPrnEEY#84%i~Y*H z^MVF>ue04`_jrybj3|tmGr^(cA<@^_j#lAbIagLYxdDR-+4IxJZitH$n#37wS>+=5vAnk)et2sB zs|0<9#(fx5P)UHMeOc&aFO7Ars;n$vrG$BT(6WBBjVaUW$UH9MOG zgV~&VAM7d0U^vF|YiBGGOlVLBBI6}4jNcZ0`pg-2qaJ8IH>&|1uI%_+R1Y;$>`C!s zFuy~*$7RO*ER1!!I$1R1^YCHbxKHQQ#m!#c%_s7>zkByBoTk!y^CWt~!xzSL3%yeM zUc%Un6Xs{EdC$yTVaLJ@!#No1SP)p{O_N}fmVZue;lV@Zzqf6F{@iCVP{V(2B7c}3 z*RKlGN8-;%xilxQh?Xn9*VF!U_u!}RV^2?{Pn<&=1HIuvMU%Rpyz*IrVx~4fGs7Q} z>v$0vUKQD3>QZQg%a`k&kXzO_$r(XVDk->)_fPCZ+h7l0j_9G|LVFMYYY9twkFVfNqWY|V_j)hs&>9=k8thKy0#e%&bq zY?sGs-kw$6Z+rB}kt{G8w7Gk}KL2&w%Iu&;(@sMh$`Y#xZjjC2%&YSlHQxx*vQyIz zAkqAL@BQZZ2N0>WK_Jr+(v?1yhJs=+rxPkP1(T92`zxWkt8yb=HUgg$0W8n2ML3?>`cP=tPRo#$OwpK7|^3x0~|TNr_?(d_Wdov1uevycX&0#3S5OHR%Fonr47Ca;Y zCWm2XM_aHknuCH}8yRDQgO)v}gqBft_OoYiM7c~qXHrSIr#f0D_P1%qi!aN{u7cN6 zK6CSBBT&1V<5j*4I8clXtpt~)=}-~(5@f22j{eS@07ri=qAUR!fboKoz3nN3TjIt_ z=|UCT=U=H(W%U}6d~s0+INZ&&_Jg^2zi&u%7#gIIh5 zSyt2TA~j@_!RjD$R*^w-(EN&w>6^YU0&~sU^DJWyG6<)D4kzDwecBkZ8KGB9Lk(Ac zZXr=w5R_w_{c|KnAYe!|4A+DSX+~`+YECe)(dI{QISYZ9ganU74KRr<0p?H*7DU1H zKs+EFTULPxS&3$FXWWj{cVTXuWOa458Q~>^NaU~j^tKG(h3D6%7;$8p4@}Qx>;U)* z$;VL`BdQF^zPX_=bM^KHNM0l%iI~g=;hH10zpV4^TY#^mNkDod^LHRXesh+1_wGa( z!T{4c3Xm~)FvJMan6!l`5qE9-9@ku`Cx-JGoN|K+iARwca%tv`d1Dzht+5hE1K70G z5xfl9teM&2Uz}DM4)!A!M!^0b>fSsY%k_O1e;X7fr4SK?iikuZW12{jCS@$M%t>Y{ zh0Li$6H#a|CG$*%GNl0_GAFalLw@I7eb&lapYQM3$FcYRW4}jhRq}eD=YH<{8qVvy z&TGqkX=k;-4{_QuI3kMBPdMjTsVp<|2z?dmInzVl&!B~BMY`3Xi`if0g1%IKa5Awr zU4^T)KY%9B<5AG;AO*UX2i@8?ZlOf^^gpkrS0MU5K{NJZb~{$MOtvChR#m(6srf8aZH;#V+L#e3J;X z{C>$^+AS6lKr~Fj$(D3+B}#wTur)(i{#)pR?RG$6F5wuFEd9@mimKU1?%Nx_I1vX| zi1C4C)u*O}itdx6QRg({U=<=C|8ez)!^l&E^v#(}C5Rh;4${@pIj0!~J)NSdKfWz& z-THef^5SfCp|yXwND9?jeAdshZ4ED^$+T{fIQ;iZ_7Z3s0h3U<@o5Z&&Lk;)o;!Uy z3}$91b@P9|3%@U6&u6*%0Zskk2)F&4W^p#q+3-5a37S-VA|fAH|9!pS_s0O-v7i2d zhn2P^#q`s*IK@BqVwf!FJr&Qr`}UE(?V7O}WTXzz!TMZ>&DIn1;*SsKIgFZ(4%wFM zF5}H?O*fDnK%k3HTDtM$g@R?GoC# zU=ZPm>)Nd!|MBVLqU9V3TFaF$OM*Qjg-TAY1Kc@+r?R8 zJJN6(&epv&zJoXrT+~M-p+?|M6rL~)fOH%^!?t3#ut0{?$j9`@N6J84KkyM0KtEd+ zEF+rFJpzAh?u+y1h_+xb!T(*ceFi<-c_kc3uWr5XLcssmb>>=#&0laVP1h!)bl~Z` zoEg4e8B<#p*-OPg<-`B_w*Sjl_rE!ZVAqA!l-ezQ=Uw~?R^;|tlPVOY&z%4MH!!$4~*?VP+auT z$Jlsew=1pek585zE?4_V1a6S}+Mcb+JUG&Q3|EjN7a9VvANow<7+F0f!xs?Ci!5!o{|8+R=he$}wjR2|12(ri;Il=nu_1w83 z0fz8A+zQC*2v>(_%C%0#xbypQ$i%|yBDKUXp6%N|I#g-xn==xmEd=?M!t7Pyckka@ zJI_9L=LaHnYuk6mzqHB7Rh*V^r9O3%@&B>eCDXwL&6YE;{V|{0|5(c}Ma`DAnwORT zk+~d>*PzyKfGLOmrudzJJ9oY%Ya|*R@*ll!FQtt|v)}xC!p2XzYKYX1GX9Z1#$N?} z7A+4ZnMzP9DekuwV?%1^eyd$(Zf14?Embhmhe&pT8UqraXp&t(-cQxi(lWb-xM;w3 zCdd^@s|Ts|AXGzQO`IhvCTC)sm{gbtiKhOP13CjB8681} z8-TYta&`-p<)1-o|7Z*Z_KDCdB*a{+#@szToN$fu$@CAXHj;mZ z@aSRW{3-3M(mgLADVw1)}yOf)c|xcRxR^cG*##zS|a^JNpfBS6st)Q zW|O#Z5Q|bH+T;|027wM^sV7gKd|D)oP(Bvp2Ov3p9Cvvf>q&66(MF=AeNX(T>yL>j z5T1@0vD>_hJj6;qdLA6f->+OMg8Zv*@8Fp*4DAiW7BWCJuAjTkI;4D!s+ zAmO(2T$^Xj*`<)rChg#;o2->_(I6oW=5351G9vlOd*qIfp^$Kj2o92J>f6agJ{LEMc?L;KHs;Y{pzmew+T^L{ED#Y z3(S&1q+64-l=lf)CL}RHU#{K}F%HLc7b5$4N7DYqJ|E_X5R6uwdWI~@L>u)4Q8m$R z+up@g12ob;KlO9a40cd%3#B!@~&C&>k! zz8(Ww9-!5lo`eS@5clSC==yujT+sIWm}sa>6j0x}Dm;Zq@`)^mkt5KnJ&)`PJ#@2V z&d=}Hl1MB>H$m3h(b5u)jn5u)SA5|h(#uX`vlH8m-@P)MKeJ0`X@h;6TTws^a?5F28E?e z{D-k46P=ZdI_W3wcsu(llSx8Yr0FexCrcNbJ|WntKZgaE)Dp#PAe0P0Aaq72scqj& zse1`fj7@3BkdwM&3O^1&H5rduqZF5J<2Za!32C|rFR%?`Y%)7h*ZzpNk3h52 z1jT^#7B9Y`N9GnS;w;j7uz{fm(&rWPfX-Cyg^W34^Kleo` zR99mT-D-=2$UfQL>CkNb$&5r4#vUgDL&-u|_XrdO!Tq=L-!fSbBHthm|JjjqI| zymM`GP=R1N5eSVKK_Mst8}@KugB%U?!prud{cS-w%zk<{?Ra}+D;#d zobIpJ#mle%ml1GYA`yusDJ=;=-_y5F)C8rGLIW4GSD5ni_qSE2Te?)ZabJfGN~4Yn+cmvB*d64dyKS>&@#4kZ=%Px9 zc`$wR-o57lRnz5|Qa)d}yT6-gweEBnj^?v$wiFWq2AMUGztp@R<;;*q`_8pQpB8wa zWv}6n>4_`C0U3&i0{;$_N^~#LUA>U`rf+lRW}fZGC!I1r>+8`NGN6;pjy0r!B_t(8 zn@rz8``o9PE!(yc4rcP>2h+(os^r*A_TLr8KI*J;z`&mD`f~&Z#V31byJFW(f)qW1 z`h-FtAXW2(mT5A}ZCn{rwY>s=UjoG`|Ek6Afzh4zW+-?e>3pePPlg~s9{tlXl^@=I zenG=cv;G7oM2eagdjq6nY{`mVbzvabQMtLfY&DKwA9`raI%B{C#F4abA4yk5QTadH z0KZ@T|1SV^nUUQ`Mu?FDWt|!%;NWw{_gHf0ut_`Bjx^yYj2hjC9VGf!F`;BALAL`r zCqQ*0Ko(XAB&<~+PAn(x$xs`vQ^m-G3%CEifZS4eUbq*F((Dsnj&fcy7W?L=OFK|S{Pvmn^GuT|+_lD>rk51#^=H{o%-Lk9V~ z{6eZ3K@aQZ3F!yW_C2q4`wzrL0Jm^nT^gT(49O??`b1IYCja*(OY2iCYA|K!5u_1S z)gi?C(qZH*W<_vnpB>l$%rSPBj(UJUFkp~XokR;+qiX(RL>IiqHB`#6x^-lT6(A31 ztDrN~U#rrIQ1^J&)vj*4zi+n97uH#CBIqi@iDDyGB5GYf#0aPP+NRwALefN%0j5C1 zjNB2J{kJTdS7;x_Nw0A=q!l?bO1fgjO)Cbl|{{UDotIU`Qpw zOfZN8cI&)BIUi!&CQh?#8HDSDz5YggE_y%t*){+N?NbfF-_@_a)a18~I^Xh&MpYeb z?_q4RXs~XyKBowlG9Q5PR*voW$K)@>Y0cHuP}}z(yFI9|f`~iE-EIOfv=X7Bjk*>j zxc7HsqnGk%-V~p)L9dp7&EGi2VLMWlHnWFIXOFuZ5D*3r+?kq{Ni@KdESj_s71<y;pavBCjTmNwFqO0w?3>~R2=f!-`+<6w4@bn`OAHw*$|-21JrL;@Ean#(yH-1jNcD68H||%^nB~ z1OZbS+9w}1C+1_aAK2#;{HOLp?;1HuU~lp4+*vmw2;^;Y`QNJ>mirYb7bXO0ltNs1 z=xYLVAj5y^Lz(>L?eHin2$P8XuqwR!$CsFpc05+&6W`RZD9GZdWF#V8!!Ir9xO=-S zeFGCR0@q#&`cAe~tDi*QfSoouAt>Bw{4BFQX6(mE$t5%%1rMR5Ul?PCya7x6AU3UI z%}8&(Dvo3Z$oD%nG0BZgY6HC`mguHnJk^f6npCS+1_p+<TJ^_YnB@W=yXtNcDmiV#rsAuIFo1hnHx zZbJGhzt2o0!gnc5_DkDf#*#VOEys@YBIV*uyA+sQmr zjk_>o3k`h&{$)S;dumY}3L``|ojehw5Ro_)DzIvSkHG3u!RV61mX`5V zF2n_51?pSl?)+iV6})_W&S%K5Ffu+y*%?F+Bs*(%o{vV{qG5(X7q*U1WiPyk9z{C?0+v~r%IcO(?AS?h*qY@cWqq>aJ19bJ49K0t+g5o1Ec>XhAqDDTJM zsgTM-Y%)xr5r9~GwWF|)IiHAPM5fGD40ea45WS3g?F z%sSW<^5HW`Km%{K`%efC7ot@z2G56wpTDfNw-U4D%p9@(6xomULX2=p0?s^{y!2b) z`_}>sV4#&+`uQg$m4fR^t@FpeMZ~M0p@fKbyXy}poafCsZ>}uHGm9i4BOnE}mqyoj znAV0NDp#1DaW#kT`yx8FXbf6Pm#Ww(O0F8$uThGr0#$H9*E>f#H_XKsjsPvd^um)U z7*~T<5eNN$!>MdYMT}sS4_5dnDFdb*u*=8bB|!UR8*}q++_(=3H7U_!?BW))MAW4r zYf@RdEz9rEYF+LoGPaZaWYU_0W&MU^2hQs-hPDt!8NtEP6C~6a`#NGTp0T2zc}#PY{uq?tFg4E|SZNSd?H@T7nUOOWp_;4O{$Fr(QuzGib=Y#@qD383(rf8F z&@1=dIp@LLk-9@Z1R!4+x_d4T#_LgV-C}CUV=MKa*azhcBc9`uWy?k;U512db*Se2 zV?gIG>q{amSVN-nPV%P=e)WZO?}_Imz7ZAz_6l_$?I#vZ*-uXr!GId3DqEXYZOgoMZm#n&T|F&g1)gOi$&(ctuR0?uFWYzx6gG=`Ea z!78*cxkl5ve7kuZs2(AbJ*bcnr?!4+g3-IsF^l4|WSt40Fq<(gX+`+ps8qUtxX8ah(e?0QaP--(bZ zBM`nu8CndtjlIvFDemEY<*x^fpWk*kB(2Q(Rd7d0+03x&!900e$D@A+iDmpPn+4YNN-u+5nsQp(*n#fWuK_D(uVz@e$^kl zHe1rL->E{fz_45cKOpu-27f2#w2Q-)ZmN!LWAs3cU6-?0%p9UjlZi^e4Y-j~DUi{6 z$cta1-nW698*C?#aLN2otPi0TqKGbHTDz4XLU>bRh7%WE=hek&l&h~$;QAQ0Ye7co z&&QaPgo&Xaj|M;IWrR zRr9}7(~QRf^dS@+j5>J$uYa@M25|yXAuU2QU-t1-@f=3OMP=&c%WFL8xrWKJK(6b6 zyoF3i0Kk-?n87_~Ut!=ehT@=zKLSW>^Nt}E|@hM z9G-R7uX$<96n$jm4k7r_?d`AmaD9Ph=~8K@quWi6r^Kz3Ntlf@T9)bj(qy7_letRo zfyX_3CChsx7@pNfKDXQ=*!6i&5@){I^OCU{#v5VMrOD|*6DS3J@9eojHTL8}d8z61 z5}BTQ*Mm}vv%ji!B!cc2nlyDxg^@uAWt0BsBg5f-sdx)2A%?vpb+?YkP=_YnGxbt@ zdt-60>!`zL*8R^+ebKkl{C=elU3ZPXa%#}s(FfrJo~8Iu8NEKyie?ZAr+uYMH9Mqr zwR%IK+z_jXE3o11_?T8I8P@?u>9ot8(&;IM&~|fZQ+6elD8PfP3k8Ef1#?W^qlV7N z!6B${DtkNEy4=}pkIR;emShFC^F4J+;_DG8h&%RF>*&Wgv^=o5JT{2=_Vj4~0ThBH zlK5^NnOUPxzT<-zwQANmUT-u>E+Z#&-NjzGnK{?+uwO=?o!UKr}mI6-i&YUz4)Q_>9PY z6lYK*%TS~QD3dvYaKXyt`f+S|6@G;}_tW)3K&HBI%klLby3$3bK|-#@v|i#rk&AK9 z^s8?ZAZ!Jq3yy97lrVB`-HDp>HcFg02FMVpA`F-8LfChMyCy|XV+Ev@P%|+?#!pf^ z^^+0ne$Iyb%SH~}1k?|wErKy@7=I^+dVcJvPo6gz!Dop}MF%IwChH$fYWz9K?XUN> zg%E8@GPfJ6#L3&8q6f09QSQZgKUW0syTpMq+c{@O7V%ApVVk8`gB%Oo5FvVqU>z{T zLlZg7juXMN|F|J)1UUq30F~?+#ApH^VesH*zA4nVpQt9^UHy;m7DFF0A;CBgz22nt z*&jo~5nii~S@F*KzKfEbmSWtmdj~gOom&(N=%m&sjhBDjAXv(VXqoPvtfrm*IrZ(Y ze<=}$h*Aoi?yCQzC8Pm$$I%{@rflb40j~X|M*R0nns?n{zkTJsvH8!~Bv$nZ%!@b0 z_Ys~GAx9z^K#)}17bxKMXMjT)Utgy}3+;fKC#l;tZt8F}`!n^IGc9nfBHN z;FB-Al$v4XgOB(K!Ji#4Yi%1If@h`A`o;*9WN_dWH_zVhzB%)^989BKA z&woPnlnK+k%T%Nu^yv8|pQW?gvWZ$Y@KhtHpkg%naU@8Zc|JbfNR(BuU$Zb=k2QzJ zK;Fsv7r;F0pOr!9-_npz`C)z~V8vxBTa$z!Ipga=a=cm^UKtq-G)Eee7io7!Izw02$9!Cb<=+F<)?JXi>Nrz!-N<8NGzA7rZ z)0+X!BUqE^-Lt|+82;N2lW}3oy_=zrF_vFYha2xntPFIvKpJ5+)TZEs&r*kRt zfqG612XL!oCa!2twFSPGf5d-A0O@R^E-$8KC`0P;oxBH zhy3C@(O=-@?S{Y7Pft&;U}U_Cf=L9hkoc4o*X}Ir%2bgIZcy z?RqG8bpRq^VJ8!GHc#{Y4(g&}ZZHQfoA{5%aH#12(4^{FJ&Pb)!n+HT&IX}=2jcOmQ}Jw{j#8Ulx31vGRY z`k)TuZuFYoBN@cJ=Ze(c=H^Qvac9=#ynfd?TU=UNYF8i9cE0N!74U>xjEnzV6nc7N zZB$$bO;)d7eQIiYEPIyS>`3(ChsUlR+G1KUPQl=?tB9LhFuV3eql{q%8xHWIMQ*#Q zryz;vi#R*CsHo$9ISc&3pD&Kf+zLf;TV{lEHu(AZ@iO{tj2Gv+RajU^4r}W9u9cMB z>S`wBz8jV>@l#V!ouvZ6VBjCG%g}|-n+DeuuhX&mh#Lv=%)sA3_>L*L#h>TB6Jl=Qm&!>z( zKRFQfepev(r`fLaJ6l~jrxXGojmmu7Or8ejnwPlrLzLT{5c*SRhP#g5+*O)3ic95%aCD z0roi!V`>nWk&&@!$^|Ddu@^?~lMG*%=FaC;UFYTcH_ct<$>0QWF5hm{uRkSwc-hy9pCAck|K7j61yB3@dyP@i3>1I7kuUWKceOV#bP!i*=FglC3s9 zGjn~5`i_UL#VA`_B!9+`+i4hmvTq*9xY8*rE4R8LMwSabH1}!Ttgb!fy42YSgHj3+ zpb%G|nwlDh&Y6-B;Z5XYL4eA%8=McHxCac9PcDE<_n^MrIFSZdaWx}DLT>l&d6aqc z=0&T8bq3!8%}4d+(tYiZUb^|2PH+el_-Q^7sRQnYySh8C5@L!NU5)$v96k zAbWD|o&h6;II!maenE!BhzPER&!4NFpAsG)AE)kBoO|=b3FLg~VJr#UDiyM$OK`^O z&N+A&Fso}zR|}j5cOpJ9vC-rxD(A%9q9}A#DN1ov3J(1hf(@(h9fxj$7C+uJj{>@T zKK2y&4z6MFr#vd*3pF$}PMerK`}%$E$@F};xlx?!>c{7&Rt0kN)`{BoHGKV)xXEF( zhlVmRG?b4$`K7)6DhUA!3-eyTrhzfrk7=!g2xakCh4D^eJ*)3@2=*YEm z=NcU0eP}*Z$#Sqs+_(Jk>_!|aY_Y{45Yac@o_h^nb@MuCm+@(&Ze6lk;NIEL{%_yB z2zpi5xoWO4qUSJ?o4am%Za%HOx9gu@^ZH-?TF<{?u9B4qn;H;(z?*sc`0RmmR7tfw zbL`oPWuv2`Pm342<<)-%29Fd_r+ft^Bun6Y zdO0X$UO1g;Kn&F;Lt5B;@&AP-*2Fg+q zKuywdlop93mW@I|8R9^DNh8NU!%z96i1Vv}hnr#;c+1pfH@5@;R6UJZtccrp6S9m{ zGBF46$*C?g>4jrb*r@Q55ZYf15{ehH{znd`#@D$OYF`?!LS*QHi_aX9y03KG)Y=`V z3;7uBmqRxy;7erWGY=$)yUl%!BD>>iBCspP0409v#;)9hbqUy@$V9GQ)V$#$w*rOY z>gpy!Yv|L#3yJr(7CMF(&po6`(%`pp8)=3z|t&k7!x~wnKs<&?YhZyzM2<`X8 zYNQ?6lKiryre-ym4+YqV^K!>$qm-hAU1kJeF!rO6gIIC#eI*a9X#|QjPuLTW>^Qqv zcJB5V<`5MXO)R2ybadp`%4BYAY!q`C=1I0_@_^2r-vgVEb2wkGwQ%S-hXcxn#xd_Vie@cwCeF#P*U)uW!V;-r4@OQ z1#<`AJ>SK}!$S?}0i|2DFv3@0tXNV&QSu{q43BJ)mEtlp2@54j2xd;b*r2sYVxGPV zs{T6SfRZXTZnvoBTu0e8M59z-sO4kIw*K}Pi zr$_5B*!j#><|87>FNrQgs{QxK!9Hd*9^~Tkx1#mv11!*n12+Rk#8ESuH_)NPx}057 zR$gBIY2kGCzs~bh6yfu*5zDn$LEQK_L(q{#2_A6Fbbx5_uBpfNfUQ2$0D6rKid9B7 zIoM?DU4Kj$^&+@!#=`KfQCB0#P-_h~Pcdpt*Lja_Y_5DAw1ssE1JCIO7a#{-uWqmT z_h(CB17M=NZ#}9+dr`pD>#c{AOYB7$5@lrC*VkuIkC@awCMM?NjH#Jf)7cjJuG)8N zU|6t6UZrLkRdLqV@>pn&qdCw=8~$QAOGojLX~nv^j`<@Q%Wj$hlHiLm5R-QH^z``D zlXHY#odZ`#wGuWs_oH%tbY1w%Zdecs1v?+zcj4%Q=pV5sX_QNO@f{4J|MPXqa;#4? zu@{lTn?_L>B9fw(X@U8s<@Jhj_3E+ewJaE5%;`9blsUvq29^%pw!ACV)fiT+I6a+? zF3tWKdnF~MqPU3g@W!d{N0$D}4%y(rZLJqTapLU{qynP#*{-uc9E%rHkk4G!iW4>O zd~<85fPlb@FTxx#*oQwqhK0p=dM7RnAA<+TJ8A#q->2j$VylIV7vIe~ z5q)_5!%F$dA24j0wi=YibQ5g}$YJHV{(?+4# zv-@m!qnhgX>6=$Oqf+s4LWu&o#p=B)-mO&#WRP0IjX1pI7GI6yOb?uKqS>ZRo0>D$ zA*|!)@5$1cRDM@`b6Wi{JEO$>Sd3K; z9;C(g+?3e_Wj!I|caO#ZCw
y9w1qMai9~!cy*96Bk!hM1Syznw>vHG8j}MrR3s>KPa)^g z*_nK)q%jq_(uBv1pf3o!IFApUIe2%+A){GzlT8EVKnUp$=XMc1+DAq{yuP+r<+xp; z2|SZmFIZSa1VzWM=s6`ucjkw*Wl7uR)M(6T-56JUaeA-U6XyFnu2NB*=_|Q3;5g9| zBV^sQm3D(y`w5|)y$3s0;H;B}N{jCdcL>bb5#I-Zh1 zzsvIc>&w9dvoif5Y4Sq{t*<}oK3lyc&`^y}{!Y%In4{=>wO#MJchpF{2DdveSBq=# zT`6mYz^N6++oFV4D?E6tai#cE#wi>lwE#)Qn)&P|uZ8cv-)1PMx94Wv`p0}gXHw6# zdkwfCI&ta`?d|C?Oxmv+UEohV z8maMx&i-uTm+86ofdag^DDpPy2M}Rgp+dBH z=p})Cg5Sniyv>d-X2}W7kr!f^c=XbX{;YCIH|1bgO>g;pi4Y+^6@Jk-L7T&LuG9_Z zORPBLsg0xyohM(aP9NEqfriKqc(7bXl;>y!)<`vx)Z= z+bX?Y?6r<&emFY5c>lxln{ETcOZPvVFFw%iQ(;s!FR!x}wo*nb;EBLwry+(tS=r{bJB=?6PL=I#G`Qo9XE;QFg=E z+hQ2GZ9jK7behfk5myy6Bbzxqv@dp=YU}BDw6JlX#+^GjEfRyg2gDQ~B-w0_qq0c6 zu;oj_=#KZ|F|v71Uu9lyD*7S0w>SPBBRnVqr>8@SNez^TFddsEBqUIdSPL4xEX)|< zQj`SZou@94u27y6Wtq(usPEfNj|+k9M90CwAs@K#zS3faz<}*0CxEf-ee8P8i8yxQ z>koIG6it?xuPyE2ZNh#&ByEw<^)uDB2aDzAYwPMw8Q2XD3l$&CUD_}E;q1-@l~M~1 zKQdb1BauusZ{DS|9*>RZ#T6XRm^dhD*W>Lp@m;6?dO0->JdJ{^yvR47THnQ5c{~Av z5=wn*JpuxTr`Tk~%Zm7;gq6xHt_Dz#%pOte)^aZl`%$DgK<&FLfiun0p~-H1@s+1* zgCei4rx*URO#2m=M9)?_sueU@b!Uv#WRJ1AG&r#=Ul3vEvSh%e*)4KpXYTt}s!HXM zGQT9(xKLFKNmU8{1N$_N@d%i)yOreLF5A2C)fK8Q?~1BC=|*-ZOHx(YRH*H7-?}>C zmB{MmbT#((YV*E}P8>9$$}}(izH{A*a@Ox&7$^V4I_yFH>|JXC{*d z3?H2^HU0$7@HL?K;UJfL`ngyeO1tGzD4FALSu0dxMKJ+l5G1MZ?N*Ow(Td(1B>ujf~1U|#FL z?{3_eAKmf~Lgxqat@}RxvaIK1>&VgfEfEEl_TQQJWRL0yaIQ-$^5GZ%P99}Q2j{)j zZ9*}OTU~DmrWMg@rEFQcgm#~|`~87|AG&QN_g8;U>D`^px%VTT;`=!Z&6YUKhwhD% zpe=VlnFwlSu9&116ckMC4cu_Fqi;e%W>Z%s+KO}(yFrI0S;s>Jf<;B%vtJ1g8hMB% z_wGht)Gs^*6$Wnv}fOL7#5J1U(INn#QovTD{FTOvNUb#NLbh>h622cvptKtLuG~+?0%~xl3p9qtK`i z5PVspuCCsCK~|Q85_jn5W)uW{x3Wy#2-`5hKRfB-wji_lktLC_JjMY{=9s8nP{$=Y!kH zZ_)(ef(DRRV=5JC0O9dyWs0%OxR^4jgP*A1YcXjtzp-I^L0;bay*g7Tu?t_G{kQ~m z0yz}d1hg_eZwqR3B2z~0f{Mdl5Drw(`+@ZP>DpLM*y978I-L3OXEX%Ir5rCFJY^kL zTDHC9RKQXWMb3P==IVqE4;nMdg;tTxvHqb-bj`UP<_)8A6&m94oh1fNL&jEQ?E^ed ze&9%A6@8GsFZP}QmX!a1bAD?Xj`-Br>YT?*^kE>&$$Cb+dOv*N>9*8Vh>0q|AL}Wn zv+}LocgBXH<((6}7pe5D42svph29BTvFcLk-Qx8Pvn?^5C1yM!jsAfR0qNMT9ZP4Aioo(MuM{aF=NBJFLK)qZU;#^%h5olZFWvVGI0mcWy;}HoThGJxok98AN z=xYZQEixS(mMaN-dTZZtwrz8nfTJR3K|yo%6Grx?wAa5@>P&N0bko@#v7d)z>1ui` zWin|VANbB>7<6TC(Zf6*TMb7xN)20xRlYO-o12@%E}Xn$u*2XKo0XG`;!WOsIq&Vb z;ng!1LLU}bl}uX*y`+rao)>hbaAVl(AimV*)fL~5QkIsql=*GflG(mx{f;HKQ+)QP zHm&O5B<7<6E+%uGzj4gPiMvkZ2eC8xo$$^0QazGU5-7E}!^~>`b>#z3U^s+UtGIWc zwaR4j6u$pn%s>zFBMz3>HRjr>mh+Z#NZ(BAC$L5`x*#3|KFk`i2ixnSR$NMYs8(Dt z$|py+?>NAUSnsA%;lBMkt%6z2Q{!}OnlHjjpKmYGyn-zh&Y3S1Y0W2@ZgXFE<3(SV zmi&TZ;0Z) zdney6Y4?gOSI={nC-<~W%AjCzCH-Q<+SB|?!q#~|WV%|sA*`$VS?cI!pUx->Q$S(J zT9Ztdvubi046&wb{o_^r|A(V#XSYyI6jC{=76lvBFO=1%uih(jNTO0zb-_POoPxNx z_(R|v265#Nr0T!9fE)`fFvF_8qJj=6mU2}sNa=n+tk$GtWo0#;O_#60A~mNemU zilI(*m%-Qf-J%_aD<1E*s#2*PR^+gg$)qYSJM~r{TS?$eJ%`%LwJetsG+pxDSXF;} z15VCG%j(Mt3!f=p-{N?pM?8~W|IJ|BY5r@Te#QRgj+TSj94`#5i>^uO4xS46>7WgC zI|Av8J5%xv3&*{+O^8cbw5aUT@&}xwO*%%q7<7a_gnOp$5vg}zyzMM!@3Q3dyi@l+ zB@}f>MQ$kC+xo4VhI+)lL}L$yRruAPe$*ci5N5n8Qo&Xu>->U{M1z9NEslquGq4Kt zpAiVBt@+quw`GOl&M2$0ubLoyE(EUFI*KiXR=RB4Y?&Q9m|(JYqhK+Z1h!I`omHv- zYm}4>W=}a@@rsGr0%p1CXdS4Q&0Sp+70yUIguot5tU+yf5i9GDJJ~2@2g|rhL(3}W z0uKl6toEh@hAleb@y0qFnNF4qSng}SH7?rR##?mNe}j<9b>{k{Wqq0%Wxo#P_3H`a z3B4Nka;6uadXav5EN>aln7H?rpgl#eBC3L=#9aFsUl>^BCww;N(TbV_dk}h_{*DQ8tTF9 z%Ku?0xgy>s8{2AR+bu)hw$<>xIfLL)rT^j+6j2LzM?xK(RsHlhAELik&*v?muuIxq z=U^szM5;=;SFPgnM^D zrjHJo8keJ}vGwc+9tNJn6!?sM@Xp|Y>&`shEP*I6eEBBn9c%7x8ZGd4tZ5G572a8S z@~YE@)PV%@X6c1IdFq;!*a7d6HEA$Pp)LVaj{5u zw54aQww{$$zSUSl^vUe#k+tuI5`{`{&MWRn+Eb`}?%CR)4!Fq%*TT2^Jukh_Nm(x7 z)P6-{s*JS$mn~a+~G^h#IusNmdj~(KM%e( zZ2xzxCmg7YusPMXpB=b0Ed>~BZrCojtv26|L2ZaWMA)zhaIzH&9f{gs4NC&%0nseE zD{3=DVP$1a&B%yA(n|^xlrI2If0o)%LPgpn7s%a^VY?i)ukH0I(5tG=auhwkQxo9x z3D9J7KYw0Je$Z_}j&z%DQa%omlIzgTlYn7ZHJBh&D``y0?9;mHc0+ilvi3En4W+~m z78dPkz1`PqHTBNkFN6XYh_<}lH%6BZj7@ly^tAYVUM%{>+bfh9_Pxa$!@5$l#;1~( zYiyHfsWlgMnkXzzIjpFF7>9R(UxmpvL78?zmzXnf$zse~5u|!Di~X2VJ&T{;*W}#QUthW7!A-!BC5E7E}`V}ICswyF)rtsAjp;RfaS#DfA1?ov4_3#zdiwHda8q{ruJLVNk$ z)euqJj|xhemg`U(4bn_E)I_pB29p?(4LqYX>%tDs-av3eh;cY;;-XYESijnjv@o z3$a99-x}pZ?>D__{xy1nJaUhRWJ=7PxH`6`;-o5)mCIw12 zH*2f>%VJD!x_0iug(4Jnk>AmQfYg4!zx08W1e!!V0B?t*F2cc`tSmjT5Rw4-U?QnJ zKbc^g9fzG9jBYj(G@=6L6j1wa2};xlgSXDZ+Hc$OX}TsFbqh;0`=FDK9aOFY^dnnM zjhbyQI7ILCoTh5Oz+u(2{A@CD8rBJ0diM3WmB<{JHi6J@qNeQFb%1 zeNRnCSJYDIodD$qVvdO{KRcwy2!rP@cbSR-cprl z+SSrQuZoLwbo!N2M=M$Mz68H6!U0W47z6^Q$|hsII%J#pnW@I@(HghBMHF>fS5*P@ zzLz>WA7B6S)=wq+#KbBrsR+!pk-uBw-3T@D2Y0LEk;_QZo}|`p*ek*oR=~rMy9p+P z;coQ)yK8H#-f?;U6?-Wm_zIC0Vz041xHqhLS%<~X*i@3lrXI#I>yB>R)N}gB#-+ri z78O1GuIy~p(RNDj{F^Hw-50`4AHVr8YybErf~#;N6*Fn~e)R9^ak5-MlB!{mbIOthC+O>ON zxb9S~L?&+!qd>9u!M&z3Zky}R{Hr6)b3x6= zZP>QKIj@WHH1Yacm&~EWKDCr(?ftFRpLaNoj$~LgDPhyL{IO~2uK~?q1+N_BH}fy3 zKzyxF5rm?s&S>o_g}@bS*ZN{7N1(bk4g_BaRQBgA+F@~xP3|A zrumo^TUHH^8xO5#%Wk9DUrZ|&Z^NDa?1UPY1u!gIPrR#vu#;-`#osa32Dy9-yQ_CS z+-b68wi0 z56b7hyd-7W{EEI--%?h>FN7b9b2}iwGg*pBclJ7KNpRJ?yS6c}Qnrt)ZVlq*{#JG2 z&3d-f0X7!lI^Cv@P?hiX+sIABUFR`5Mv3k%7bLk5iIFrox%u4%W4V@$zHK&a+t0$Z zd&_~htZ$9S4zW3`%>6SnEDuwD<@;HfexPmTchR3Y^aGMZ7yOk&V}OmUzn_~`+ZP2r z%MwrHXQHj%m`(QY%y7@mBijthJOL24pnI59`HTG4dIR8M?T?(7Y5b0aO=vDY?(<VdsyjEk@#INj*rLXO)^2uVTd`2EI&N)z*EA=)F?NGq?dJ`Rl4xLb~!6_oU8Il1E343VvCam8(X+ z)^^(VNLx~|Y^&8R@RLz_R$)svz%}f8C7JnSt`9hOFOo>LV0KSWk_MP^2+@Y4de4LE zg*G{xpJQc-IdG64UmIshbv@pVt{`A}DAx_LlHqw5%^fYy^mDlaIz1`NWO(8E;N%nI zMQ-e(V-=TXBdp7s7Q(9Evlqy>-$jR_5>&fkUPv?tl;rDs8)mv5@g-BVEL-&p7ag07 z4f=1VhU2oY?=pbO9w3hAzvk@~Ia~K+ErmWyCQF(=qtmZwCUO56T|YGx)K)%iVd1UW?Gf(MJM`;_7#OSQ>uYCn{r_@M@K1O0!ir>E7(<9hnR^!Fk1wp#qecp&4iqm6qBFq>3 zVD#DGkTp-!aBX4od;c?7g8SX>Eyd}Kzt2R)B8iQn`Tobff1ina^H%=2$RV(<1qr|O zv5Sd#Xm=XdJ*>&DueF+GoDOU*5QHS?_^t=AO$cpiBV{5Js|!pMxckn9I?&$~EiwQ<(W6(ypl zg*=PNVr(Ugv9!3kBE&5|A*n!FcXiAA%}qbI#E3)LTA>wDqkUI=W7ewqYvv05jCzM1 z7NjqqUUk>@fye#(YpYGQE>Fq~0RXA$UiMYGQ!M$O(?SUebyIC&X5VhrW53RZI;-j3 zT58{CjK6_Lu?5`x9vl_3>a=TpoT5t*hda1kkzQ=BTc4%+m**w*1a3F>l;!+a`m==p z=hZkJr&60{#({mi^~|R=$DSP}{{5doZuo~Ut*vh0>FXdOL9j@nBu0X5y-s1}3y_px zO2biH1=xG<&U*DixD$~#r?;MN3fH!r`XK$))U@hsnbXaulWyd}fxdpZ%FRk=Y)7qS znI2$NF_dw%cAq$1Fq#;jlvMcC*SjiSXm&}_O1E8darT?y2geuG*43Tve!9!3E*{xD zK{7sljn|jkEix07GoIv*h=w*Ge*Ot=tR!H)+nqC|MW8_r4G-@DaniRQO161`kv=Z< z6e-3k&04Io`u^&H=vAqdG@kE=){?!56}f7@+v)^c_16fL0@Th^FNRqLLGRb^efW9Hq8E!lie zep}T=@+YZWr+qE4P@b9NOF{^UKpS{v-r|0pRx^${Mm10`_r6y}*n!+<1_dK5Ex+&l1)<#FrdhB*rohW4>Wr?go6HG8S?_9JVp37w4r@}< zExeQwpHOvXhGXV!C2$NI)UbvcII?&c1c)?~2AhqA^@cguBwK^yBKMb#)h{XwT-3 z9a1Cpy}x04MFOpQesyj~wu#<6N+T3Pm$3wLKLjoc8IHwh`k+-+RlOB)h=o*kxEe(a z4i272tcYvp7-aS}+=LRT1C)0HV)RN3d zu(nZQr8~d0o6P8EccrL;JCB{;W`7hVtJtBt=sCOQ>RyxEM}C!c#AsGlw{BOOR;TTC zlN?_p@N21?J9U@Gw@fbmeMDSH+@T-H-Tb3U-_B}g-LIJRHmO#e)_g|_;`YSb1ahY? z3-#IVYJM3X8a<&nxI6L0TxT0tVaz7Xd4L9Oj|R4RAb6oXm=8VkE9mM7Kl{-?uInDy z$Hw5;iVb!0qfiBE)7$g4A~$CrUL&>;3$R04`$qU5oWP59KWkELZzp;uI8_rS-t zv|GQ8TtEyeo~2Wl_Q~{4CUk6^*wS}Z$?RwN65`Jif9{y(`KRjpe&=@F^0l5dV(y2r z59G{`8UES_-^LeATYdEY9dmGyf=*|EXy&VP~mh}87zHYluQf1p8A$>%r^MR{-dp#O5~H0HwbPOsG{ zxs}#T4_&%Pzb$%f^>>{0`_U_)ed=p%0+++^*Gn(--|s(!mbxx*L1p@ya{S>|*NccS zpx#+>XFH#Ursg1oE5m_p@eTbp_%H42rcitr9g-__*z$Yryh3ioTetU9#0<_M6T8Q)z$|6Y$MnO6nG%2c_4)HX zlQ-U|7~kbqu$=nN&(xPnM;X`Mwfubl$)mrPZVT&=1?i&vQRj};k#Kz_x9P5RbX*H5 zZx^H;z5z3er!#)kf1rMYGDT9RU1#1Yjduha><#3m@mGp*FMj-)66{@4h_@CtZ}a}gX8&3^V{r8mRH{n&EcYsBS#t!+QFiwB zStFCcq2qx=+xDdxl&?A8TlXe4{I7d%OyTQ#N!Hte6Jv?J5trwsbk=ynT`>HhC2j8hMuB6mX9<6t6(BYe6^R)|wA>Yyy9JNziOJY#}uaQ84# zcrg+Lp(6K@zwpl*FzWD)zI)jwloF+v;)se|0&PuSWEcaa+1%b}G!)c=6yOn6x-p8Y zRpz1H*%A;Od7m?(W&tan)HywCBt{>glGnqwRJ(`*5D6*FC z-W~%M=`_#sOZZKE%H+#ZC*TNgBY)W2H|}`l9<-peALHh!X(6BypyocoVc*Z3JX$83-u<#qAaRN|GipZ|91k|V1`P2GI)uzY$P z;b16%*ak9gQh#ss5K15*+1t%@UH}X;_0ZJ!?}0#87=c_*h2DV&*qXDwc@499Ks`mE z+-H>%Q8cu)M8MAll@+lE3#csJvg<{jdLp(a9XEGMR+}o*JchV8K5~oRe5bxYV`-@& zMt^v|&hk1pTPr5)Pg0KZPTi1dDUG`mc6mEB<5R$QYEBH4M_dmMhD+xiVWv4=>RiAC zmTi*%z>36zj45W|cqLu2I)S6~^{?={zx}>1Rx@kgf?C*#3_fOLd^m)k`z#r|u+mS4 zYLp!}F!ba3D~twaEIjt97br)HF4qM{|=F=MU2$+{=gYbs!}YoVvqaN@lDAMTcoO0~2Xy8+Jdf%#st#QQfi z5o=95Q|C-}mQSHtZWO6Wsni~#k>Yo9DR1oeV0sk=Xd!>F=5~omrqRMo6~F)0pl2s?fD#C-g~ziqK-vZBMZj!}`B14ioIHb7VR4`c zgYF{-!iI;b@+f3YC@%nUjldM-HSh%9Z^IS;>=pS4symeX+b1un{~2Rbuq`&h{8m>71ZiTLGbzJ_IkN5OcFi9(bJ6J>_2%BKgX zt!OG<|HqIG9cBIUEjqHt&$tiqMrl*|qo0SUQC#E^@Q!|o{tMU*o`Wy!FPBA4uB#tM zO^Z7cYpl*9ds3$*L(}!`%fhb`qiq{<>E}K9knRAZPWv6+uc;2vD__v4kV@-7L$O@p@z~8=|HId5m*zqaVBp*R zpsAG%VX1ZwX2A`Ef(#4{{L)udo9A_WNGt}$)-zS6Dqip1;zYKX){~(22G=qs9GtlRAa)2)JdDgY({q8>5JqR3b zrcTO0>S@fE521Jn8K$~>$cfYQ-rFKHn!@BLrQ9);TUTZleCDCdI}O=xTbPG0R8nJ> zCrw2?PAnQKrH-Jc#To;NcVu(LVG*7?7^rBWkH>0v`xOVg+EKaOGaAPXMO9Q_ccHPt z<4@$>v`=sW_rQ*(0S1u=5H@bW>odRvk3k=F!SkQhV{{_D6f%}EyG4%edLZd ztPNH^J`EmkhJJ!Ie(z~Mm z9vXC135NG_B!3qgGBirj4E=6kp+3+H(P;aJBZ1uTA4?J-K?52JzK%cp1Ca!;z-9>8 zLpfjH@fp<p@e7_f}g4XtvzXqlRNU z#g)?=9vjg!UfOrk>(F4|GiE`B#UQVp$!70U3Zc8SD82fCQVKz5%y53T=ksNu4})RL zAB;~&Fm>I|Ds{PSg=oagt(ohiMhDcJS$0hZIWy}+PifBt6;im$vF&n7)eCF+# zz}!|?a|aW^4LDoMK;?wF+N%c5uhAbJjac#7ojW2l@?=%N{uBtorDwypQV-Xy!^b|l zJ{kSCRJ$HJ7$^#^53FYtX=@Cdf}UZ|C+}C@f^&mGWiaV0K*P({SUa1>$(_Cl!bMJ0=a9$q3q2_#Lmwa4i>CAiQ9SG!Qvm30 z2&kkm0Dn(q9sE+s_nd)p1Jb!9Ao_p_Jv(dyx?vDk5rTvd5s`q)MLA>v4MK1utp*7J z;Eh&F&KTN`bz9Hs?xZs3UCnKN-Dl52E}=rxQ;LIIS6uXps!kCKmb1Ag9lF!Jmi(y1 z`NW^mJofBxbUJBD{Y!Dd>&kECU3k&0-ovA#Ldun9<8X9zIDPd=chFF)5u|fa2i)Dg zV1p-ipZw_Dm>m4)&inx?u%y@l&k27Q>vxG+$46QU0X=q9qH%cXG_Umu`1k*r-L`uJ zahalw15$WP%g6y$>8!4*y1KeTZ3r;yVHffinkngH0M$cP25r4sz)c$So1nb1vgTcP zEdgtV1Q#F=BQ#q8RQil5Nohnv@$DA|NbAC>{ue@-gd4h>It4)rZ>NFm$ zt}kHHR93#T$1h}}f;^;UM1G_T{+Njz?~|GD!O*^@wr_CSX&9g7s00ugT$6*w`GZ|W zjA9%5x;Tuo-(9nw_xj|1D-J#^SIh?U1h=k5H~-5$`;ZKPv9mxHMig*tY;0;0#-JEL zV5O1@4c#x3E?(sfKw4iX$-rWd7PJhE=3s$>VX}XAt7tyC1gnRdx#qHZ`TB;Re`M3P6jy84uL>P>A2Xh4;>VgE>^^0sUq;OgJ z9uTw4x}=tCFt&(2%{@iIf-~KfB0$zO+sK_;Y1#PVd0m#z-FuB0FtXw?B?^<=zu?y? zM|K|&k#8?JxwV?^$JXM-CoE-qWi&n)m6*04#?MMH_iBl|RaoN5_`9TKE_Pi{XZDUCVfy^M(px} zI7SF~u=dQ8Cv|*pMQF)gU@Lob!qmJiju$(}v_ehl$%1iJLTloX5QkfaP@WU{C|7=J;0BvIjU1eoP{}GApR%cF z^$JMfo0(}m%L$0RH0r%M>T3*MllIMTCU@=iJ%-sl2a>_lAzi?D7%&y)c}o0l9klz_ z^rc#eU>%_QP?LBr_utlm>{WqqAhdBJfnk+92La@BFDEp~P5KV7+q!CM6ri7Wex~GV zPft&b6|0rhrrN;nHkV@y^~ukAQZ-xe7RJ1Af)b)4bzTY&hpAuME{8}9Xs9oiL-s^* zTO2+W3`w^N1foG56ylwwcZUGWF zvMAvA&K?wdXS7;R7U@;NVCmD(GU@e_0zG62&iq-#sIKqVHkH_;1^%YoYiahG?z0kt zIR;h!oG)X?)S_ChcToUPYM+h#qDwHnYu}3FxJhNIfAUK-H*?Uba|ZpT_a?q*I@Meb z1H}-Kv@!25wHw2N5Nl$W03=l~`6G*ukAFX=S?VSXY-q;qT@>;<9SFzZ&Lxlyh`OHo zG#TP;#5Ei&($r?w$^wUDW4l$Nk)od(>j3AXWYSgIFx|TCM1e=#>tr4JNWpXgE=t$X zjaIu}efY(X`B@Jqa9wQ=-s>&n#kz%F$XN~^Y#yfUQ&|jXuhGa?q^!vHGV6sUIJ<{frfnw%^hr{-+8l7K`ZR<~c3_ zC{JlWZH+>}g7IG>?^bt=#3O_Wd|u9g*3=*ru|o{i1^QSD-~d4qf%@lQ2`&)wAGo+s z?FWa1@Rb&Ta)n7k$43VXDEe@)d@5!gV0Zzd{YY*D)F=?fYK*z1VhPQzOJJF`t9;vJg%9Y30 z+e(b*nif9jr&6wCsJ3Qy-JD*<`Y_ju9{jv%JjdkGw&aaAk01KeCy7N0-i46{yFkd{ zqRqYTpfJ`#^eYiO!IX`erH4|Y<(NhiWLNsFq}=Um)2EN>hDD^IwY)jKg~tY;k1=pR!dz$$>AF@UNN z%JzHvX~ajeXFXFr7YlfI$51;U;jUKV!Ve-CXec1^Bb{qdFMJ5tVMc}L>~y+m3YB3&orWZ4fzU<~_dW(BDZx`l^Uj-ZYqI z8yFfI&qqNBq!F@30gg=`$dHQ;l9G~!%(6FWMedh_#AGXI!PG34x~zT~r9~76wKwMq zhFVkP32G$eXXvy=eCBxlD-v(W6upp4?pdH@pH;V899ITHGU=AgVR}AU-bQUUFeeCW z*0z*(M(yK0Iua5#e60sz31{eK0G;A^A>E*G1jr0U4oXkPJiVCzHK_jFN7#a%zKt#l(php8? z0ot1NaiW4>p0W9Z)|ux@uvc1Fvq9en##Yudi7SE*Z3g3(-MvdI@5jIBvU2A*nX**Z z**6*?eQEg{19HL5WlRzhM4_==bxZuX`le9pJxNg&Yr9v8R?Rxy_I^>dJG)qNc2?r2 z^NxE=n0oO1RlTaht!%@#0Vc?c?5UMENJ}sK`eqG>?A7CH0{&2BZU? z$-u`w3su+KUA%X$j$>l^*J&}`rNyq^Ne@-Aw%4~X&4OX7GI5aMxg7D21y;NS)F zI<*ydNLSaKPE_u2Q7hUL?Cx7u%N?)IBR5-Y@TfjL-Bzr8+EPcDaBWSCL(M?Mr0H!` z`J^sYWxT9YFmiOYW%SY9B2YNbhC?3KxnbGN4DDMBUD6i;z{-_G|V!!Jj z@wS+W<{SP)34^OtoH7yxYB>fJpoEEqNAdW)BGrqp-PH;`6QQp!Lor>>m=rii*Xf^vETMB%%1>`x4 zP{k_|X{z8Gp!J;S;I`0uN0EQTI*xG-s4a%M5@`gEj~ZUyj}TKr4XZN0ZW#<<$|Ftw z15ijGy132gkF_gYEVaDk5rO~GyhbqiA|GZSb$1KhPVWG;KF>cZ?LY2-ztxpBNJ?5u zT%AxhKfSm?pA;yqU$zuMasd8k)bxTBWbWiZp!FvE;Mdy5$Cnwy`=WB3?|HJ+)+LR1 z@wd7(N_Y2|BH+$X4vRFkPB#7KfeyS!vP?E_<+;h{T%$@ocp$gx z*p6Q)(3Z7K{`>v7s8e9)&*)R1abeBBG# z{}YSUfyOboSpv8qYWi#KFv{ioW|VY_K40+~}NJ0|cyL#7PpmS(sco-q3uC-Xv^HJqubK3=)DSZ-i z%%nbnjqTW@3Ez8_jl81ybLen+@+Vp3$TB8#lS>5n`!3m5Dvdu9k6Iv$b0rFy?8{_4 z)AlR#KY+sI1BVE{I3@-07$mlTK6L#>ZfIkBzlPdhw1iwKtzsSI6&2Y__eurMT+sE$ z6YFFLp8wx^*k|7Q=%C|YuND0G!LxS0HKnQHa5T|Poge!73nyNm;k^w zMEe%iGH~mWZh(A11EEPg(lBS(&2Os47&YhjFA4HQ=3>WT%YD5>iiBt1t{Mn^2v@I(*Qv= z{T&g|?K&^FG?IPu3d`_>Pi(hPk|42xp;CtQoe#Scg59RWz-HR+;xJT<>I?`C8$G#;xxmMv+uwTjm5oxSfFaH<)bwB<3=!mk6?CSfC z9vklo9dB59hd_>fu)R12T-bvFKEC&CE4jumVL^ENI8N)Gs+q-b{vY3*a2|VKz@(bz zuV4F6i2)f-X%#)7eHVNG(*4_f=RGtro~l|(`Grs(QuBUIM0W+#z@gI~uudrmIwSAW z#&F?VrYE1bM=n@W4B;+vx1H4}<>|eyhVF`e#{RijvGYl5Z4Kr#=YtH-AgN}N+Z-j; zhj^d**J^I6xl{skT4?g}$oKUa_VMVk|8A;s77P9rx~*nk^XBH$s~tC4__dID^Y zo(mr*sr}AqW-^FzL>xWkhcmXM(r`QHh6fNlVe0Z{)ONnVjV0&gatpzN0(L2~$Zk$R zga1cJz{2IotgQ>FnS;I|VF&b$e+Dh(-r8@Z5u~Lz<_)Vh;sb(%q>}FG?rC-#@|b>p zDq6tx@e;0#tnX3U(q-eEXx*TG95-Chl39`4lFPkfXW)OnP& z_`KnGu)_G?wzNAvt+3dbhIlObZ_$au;@HSY4gQmS>pM z&sHC8w+;h7??~ub3Z&9;DLR~_+wM=nQm_*n{B|h@oV}2x z;A6ovF5Nl7+iB(V)ZXOZwpzjwnvY0D3Z?*PJ{2gG^hru{9DwLP8JXY2oT*!rQk0=s z^4y{nxvm`+GW))ueyq(s#BvC4gt6l286LpqR0e4y%wzJ&qpA}*hVxW~h1u}UwZo;p zQsz)!9Ld@EXkbUv%YA}_qj)z7iCgCh@SE@AM=82_6dv5^)1zD%g8{y1H4#K9kLj;x z-$61Qa>9r7g1T}_is(nV*dgG830=$%#B*4xW&U=n|BTgQh3m&bn@HML>G)N5Wg_0L zIjPjf19;~@@3Z?X^j4KXj~9_8e!>E1T*@-jEr<5`n5zn*Gzc1%seA^fGxWRP^a9~H zn>s#i+B>OQMMy?QS2q~t@-6(bhZ&C2uf0q3JFS*y(0!PcMjBdjEcRUrjr2jdWjp@j z>)mAzOptH7`N%-%y#?R8L?#a`Sffg(l;k(WgQNZ$8Pz{%u@5389<8M8OK&*JwfQZCmj z4xvxb==1$Y6EZnm8z#e`RE!2fL55?GSgwFR8W*B03i5sE4?6w;W5AQ(KsN>?@rC7O z-co!N02o2tHA7_o_{Cy*%80-&&UpQS=$`@jxN?KU#;xXCCxk>`gz-5zEu1@9o7WfX z7?4Ad3XXt&d(?lDcR_@UW`ZrUDidQS;xrq?2fEVZa&rkzBMxSG=bv+Wr+n&v%#Uj` zZxz@R{jBj3v%Mw*TLQf`5ya?4SuvAx>XFuz<21v5==BKlKOCL8O|QeaI4tb%kDJkO zuqQQd<=?_!@q;Z2=H9Y5h_Dj!oQrv^;5E%SNe$01M6#>WAEc*%hV}30lhR36&si>0XgnkAWIAbofe8@{hf_PkfeIZas|3$av+jw z2c+Yd67CwIouUi?$ biSjf+q>>pyBIRRIAsG_C>9(k|; zRN&}`>E$F*iHQm#yAX8lRw`m#feh)PO%9eUVcdqbZm(NOfrLin#Xx@}!(k=})c3|Y zw$6#PXq`-JR7Vo6V)6Kqsr+k%JG}{q`I7TeBSGGaX0HTKiZ322Cw}@1FlnwGe6AbC zmlmp8J`I!8gtj!0fqnybeSBz(c0sjeV^Gm(ck%Q3pOr^3cW3W{pPzKNbiEh%S(GW{ zhb~k!kG7~et0d2d{VkbLu8=7FV8=I_Ed95W^}toED!}tVdJ4hJwZyw5nh&fUUvLdI zc-3IgY~oaC)Rg+S#OY_xEdsuGU37aMS;8hVk=7txJ__Qgs6jphFr&j&$DH?4f%K6^ z+~t&P&?z`g(Y&Vkpv;~1^|)XSyh-gP3&UM%Go2?RDzYNdFJHMIpL60m$H5YzySWdt zXJhOLoA8t@;lk%Z=PdD2O85--BvlHmI{O5)0`NUi&YES4@gfvy7vlq zxNw_dyuNuVQxje#S^MGco#*utH3&z)iIXWTl=1{7BU;CBsM56ZFd!F9V3n5eJv%VR z$tmmLi7rzGQ7JaP!<@@5qQZuXm&pfC(=H+AP-H zN;V$=X?I+ywi^nDKTVvBK0nv~`(wSXpbHBHqJZc^uuz#=AMfL!V9JL`$pilH+!hUV z3LE1@y9&iYTQ%}H=eKU)??v=p&(igc4&WMRzfBWonkko>p*WclBWl1Pt%Ggbh z1NvZT1#slPy0<1Q;2y6^d!=B|G7@Ur^qHmm1)Y6(6{irDu|!iiiEAxW8e4*bd8l3E z3Ddg8&3BJ^)1US-n(HKB6pWA%)uBI2oLG>jSg8-kqX6R$_a;T=IPURGQbt z68Va*I$-WGj3Ir}NpX=}{J`V9WBFbgkSt{i-`O)y_7~!Tp zQt${W(p+P(z*o(?)GqcXu3^9FfJk72j*gBhF0zmVMFV{ciy}Kd2?;t%9;3@^uF#j4 z&$sMBxaC03XAlyi=*hLn)hQP>Rr*LyLWI@jUDImZNhM@DQ7IqxxC@Pj_aYK7I2rIx zOI`>okr!YzU%(BMd-G(|_TIeE_?OR9H@}R)b-MZJ{aCfPO8v9YvW;gx5J`P6 zhP%~!*`4)bXZQOuN>2*qBq6jdH{@%Uw_@a2YUQ90f)vX;=#@g8$4@xZLIaHu0y@qD zflLzdX>OYt9V7duH;zK)&{jIopS5^ch}xG5qa=EzfC#^2EsuX*?p&v(p=r!FG%y(4 zR0lNV#*g$Y6bg_!g`hBSLN;cpkT~T&XG$*ak2R`C*AB4w-=XOsF4vKX?d3wL`-lT2C zjydf@yC@-NiF3y3sX=D?+_{(PG~8MHP7f#vA4Z`OotJ%OwJz?{m~U_7R112W!}NqF z_PvxGysu(Bp|jw~zKC7OM2K@-#vA^jSc1`}d}j|0xGpgTna7GfG*%m{bms_B@E$Rn zbBs@w5Quc2sOASDhiQ#wuj(7d!@mWEP?;Dk4&9djF08Kb8AsRxTplXe3;KWZvN&znVNYMTN&X| zsHl%;ruO!c4jJmp9wRu#rXy1_%=cyf4hn0P?wBrpSwU^1dlgsY+2@)y(;sGDIT3y& z*dp;j)31-II+o>A89LM_{)+qRc*|Q*+Fd<=mRNj}#@|Vg<(^P4@y_o|u|nA1m6lh8W#u_|tCVm^`y7nS3#RFt^A2I`J_jp|QS}8;bMj#Khc_ zMVvdt#1Y>*v68m-S=GfPpPIUOoQdl=QYQw9h}{zls}%s2AX*YZsoxuyQd4@o=z92b zBGUd8ulH|%jln%L{OQHFulSej6{n6xS@sao5nJxiO|?5MdwdT*j7JOaoFC5o7`FrF zY#zEX`?nK9)}IY8+<5tF`RU9Lr`!*jlR3}D6ohG}K8%Cxr@9sm5gDDKJ1f)GTQDc^ z0tbarn6d<7>YMeUC3+IMT zB_|`%G1^(~Ok27Y_nr0)_xJ=GUUA8%u^mg~=lqxwA`wO3rL_ls$5Q+m#vM>6o}CRW zEmN%O=CBGGn%dkH=vd|@pKU{_=iq03!^b^#VsfnbvG4cd#|276c+o-_W3C?rJreBA zm*T}&%UeARiH7N;XN$`nR{Ie3OO1y-Di+6R3JS8Hvo`l`es*wJ%6OOVoKe!GfnO&) zXLY5ErfN&DJW+WDgj)k{UId9PpxMMOZ-a1-4ZkVk=K#(qk6_n;Z2@=P4ve_JX6e!c zfTaSAFAz^Om_t!e^S&x73D*Rw7GUX|iKSsKit3-ef((I9;ohq>;oMdNh?) zsRY{+=J+gf)mpmMIFt`x zEw`eP_+Zt!>5e_7%)jd|`nrj7C9U^SP83FUeH*&?G%P(Mb#{O8m5G`ur(VoMe4*V3xG zI58)#l9#`6AA1}d7%2Oh-{h3-2&cz|D;zOwYF@G!3PNe;mdn{F^8Ds~ZIn)ZA#HjH zdbZ3?Yl;Vd7r)84LpCDTppf~xPPj=Vi#2?~KK9J0bxCaJjjJv9A|VoOFECwiZknFmEa*J-BsjDNafC9Odj zKu;Xn5)64iWVJCMnGc5p*k@{AZ`IB$UJ5ZA!eh+BP|CQk-Z(HDmLaT#)EeE<5spaT z+@Bym(RmmD_yWQBe}5x)WcvWrkWp#_j(*@kavo#>L%`Ok#GwF5R`uKexN=q{L+!3# zEE8f#p*VR;k0-4WS|>mijd?&z5ymOO7Gn6RuKZa6YhD0+zUaolv6rchNE2nxnOw)e zb~u<=*zUfLLgA9~88Yal(geGFc_8*NJYwzzBj4Gh(#&!Bc>{F(ndgw%GQm%A>D(<^ zq79LN-yg>?3w)+0De(G(?F%8eV!*SAxCI$CJUE1(>Np!`$io`;`6W-m zPnV?Y!Rp8Hf~0DeWNj_iDU4bs59r`P@l%|vw!407vNN~qwWxt0dX<4vju8hFeEKkf%O!65b2nH?>TK0s>E;Yzf1spDwR?!|_UTllZWwTRnZ=Zjd@@W8?FrY9A z=bz@Ou-(g!$fBiI0tdNS>B}-SZzDekBYl~45FKDgJ~P>Br1OKBt=O_zTwK}jLC?5c z$E<}+KPMw4Z*9m2q(smvbHRmvoQ&CJim78U$O{Ar5%8Mg*wgd~lgV$iV@ zelH}7(9eGzv_d=56w!Ps=v064*yJ~{>Bcc7C;#g~`!Nw~wrObU`^e8;=W!qi%OVyS z8V`j!m#Ps94ETk{k!ri9i!$Hi04Pn^=3fP|<*Bgoui=y!6V|tMk05AmGufL6P^v-O z{?{$26xZdu-{-UN`oZmeG0C+SOTWhJln77%XfuE6e!?r@ct=D_^suEgGlXF$jmsnDfk*)}G(ghbzSSHV0fz)<-{%qLY2 z1I4HWLK4!zr8}h+EBuQC>|{3_CN@HYjC*F`&thI{H!LKV9Yn9%IV`C>nWG22qK(+ zS;>SxiH$ZZf@A}d1@}kZ-J&Juc!Zatq7RNJL_uT(Yxr@;w|n}moANu6)OUk;@~jf+ zgYXp&qe*2?!^RtPCmDI!|K8-t(ko9^E_#GPPRerSEx&KQG4Nz%B0s`uGlXcZJ4Htk zd?C*wvTE-Fvzld&_n%=}x1bmv6-hVsFE6LWx^XTJ~;nl5<;4Djx#- z!I}xLub(zF!~)gw|F{m{`~2@VZnR3yxC72yhMxOqLx0S>n)Q1~fYzh(`gY53shtpw zbDt-GIT#^43pJa6dHHdV_*(6ptfCgvkK5<**3GXCPnR3#LRXNmE;*h@uz5FhaiEWp zLjdbp={DUh6>Hy1aypV1^bO^D5iIbYggf1w=86nVu>bqIe(wX2+E@IGPt}aiY+Xiz zD#SJt)1S##{5JzauSxmzPUKkZDj7A_L&NFM^LAa!^?#BXoH?6|~I;U6d5tKNaQle9oaA1Ty`s+ws7{P6A(K3P}$!fcHkeU8KOn0zNxF2>4fl_>jj1tEY}cP&?}4-`VZ4>xdD)a~SjPdh;y8!{g(cS!Ry4RUG@ zZu;Q?+HMxVlpcspn2;!sz|Uqd1x(Zc@NCEa|Ow5&D6ionk5|f_Vy4OFgPtQgSRefVRf|zAk0doZq8bO)crQPxU?kw z5dGM~fYXpZ;P=3>^9ATGpb)GspeKVs;@1EOI&a9R)oV$4+T^_szf~c9p?AJ`6a(YM z@Prq4YW*g5^=(=d0{?Pj`}duL9B&aYGPZej@=I3u*{$l^KvMGS#x~OldzoXnGH^4Z z@l*3xMawqmo3{cqdK2UDhd?6`=5y!Bt(rhWVNDJlWW0C>t`OLP*Y&=9}vdcenn zQiv}!ktoa7XM1=L+ypta%E6`K#VEI|j0_@TY*S)wVG#`9j|on4J&49uR#w`=u0m^_ zYKR1QCg4(wkbu>I7adgh99&t6Z8w*U5iNs5S4NOsiRRMx?oBEY%zP5qj$~r+cIH|P z8Go3`I~5q{^1(u#bmsZ053OgQ`Cl%qwx{I9?Kl1ruLTpW-rdNtwsi0ASxUY4>tWh) zH=|D__<`eYz;u`23a8*vgPuj**Kfa8#w1}|f8_@sW+1%oP$~WfMG>LNlQyFjD<^%0+8MIzGBG`E*Nelns=lkxNYE# zqmh$G6%5*gV=x}A$SKGZxJ%diY|2K!R_hXwLT_CVP)1;$KpF<=ZEmm%Em_?if58hv zBi_I*$9S+j{T%SzuUuZv`;eL?^BrZ`2<5`cmE)p8G99hwmR}7S<)5z*!n~MErI7}d zG5$mCfM^sN3Ux*1;%$ekkegv0y^p-u{oGc11RyX76XYB03=QQ+Ipb`PGoiRBz8urk zv7X;Qk&$#S#a+q3=914E(YXdYL_}t`k%||Uh!iwVKWPAdZS6J(<@6gzryyhjVMd4C z8?XTa2EdpRHT)<5f)S5zSS9_nkDz=DHk`LkOAsk5D?=%V0l)kZ$hC*ka51~<9L5dL zYy3`9LrX-2U8|4~jM}!mB7zM`MdCpE%NOh{_qt%e@-8u5gldR8=9WoA<}yON0EWPQ zb5&Z4=WE7)dv`RiOUTrj`zHxrp4Q63>kP?+s|BIqAS7O&8+b(N<64#2m13RlNRg*^ z4!2(A*F*Rzf9?_xDU$v>Y_tbMe2cLPrxhgwS6P>-ZTs>1htJqlUz7*{X%7ax;QI)$ zUq5K)0P!XW5WWq)b!}}#0FQW>o143s2dWh`AVn){NJ0umC}1n74}PSd1@xh98@x@< zu7)^1^Zon;$m!dGysxr}uSH@VJ+`5JEy=frj9WqSuD29RWv`Puf9G`SUs6v2I!_Qt z441Zf+?wa1oLh{N;a@T8}QATJY-0Ok>4<;A;UmDOSIDc`-y6XS; z?Qvncy#uY3TEu$@k4~)nhX)7*RD(#2RX_g8lQp2`4W5>8G8(MEa6J>W(Sg2X&&tZW8i7JQZ9%a)#HkMHQy@;Luy;wII&35EH=zw?;shw5aKX85 zuTIeze8gs&%;$UfRAlt5kbgY615U$9Hwx&p-Soo?;uGX1|Ge}W!fpfuru056XznlE z!X*8={6a$~`bJ(k9c`+7zD;F+0ya=>`F?hvtTMSU{-O#}u!Ah!1R-R0Td+(WL;FqV z3|bXeZ6G8Wn;TMn}QR4-za4H3o|2#UdoHh>ou4gkQ%r_k&|BASOtcx42zw4I@M7pd3}? zxl6;^D=%GvZ@FTjL8su~37Cdo$J9SRqpmo@2j&bf?*DuUd@s8u{a*!nuKub3-cQ~a z%igHpr>U>x!7PZsFB42_XWbF^`nB!|7^DEPDfPD5`~>7DCVdck@0<=p=nyc#sGWnp zOchu+FHXUQhH(hk$qur9xLw1>r%Y9TSe-u45)T}_q>2VJicJ60_8+NiSY06&4@|-UmjH=LkFk^Q?*K;@Te2x|vwxFQf zrtvhNv4VO_;4e`6J^HJtvlJE46uAa(;Aur`iXV8a>8OVR1o3*F^;S>c< zMWm(amjJ$PyZYnKSU%a$$!v=*4!{;w7Ft1m0<({F;^2gYgvA-?nwPCl35$wSmxYCd zxXrgHTmUb7Q(yw3b8==C=;#($(*nx{1a!dY!3WCb7bTPP^Wkt93_+#5m$IaE;-v}lyGRj*Xnt6J5kg7zd0cAkumWh=|I%ph!jvl3KN0AX@GXt{iX4ZoybSL^n-lEC?d;qD6=NOEc)zT<(S&-1=IX8E}VS z6^#hHacK8G;CbZlUsY9w0el-oxPU4tRsASh?X_Q`5Z32K=`}8-Mhj%}KNUXMM~Kl- z{j+y_oMhhLz(#SUR&_=9rcnL4%Z@ummnxzs+1$4{TfZ8s_d2}Ip&9?-J1^FEmX z&Edlm{b{GW+~7^@50*?+1-PeLt*->5%IGzb@P+q!n0(C3} z&;J&^c}*KgL)7i?Q40!FU!uG7&i}a}`5cx9m$ zfWg;lIZR%HPOA$mE7dS0QwBXT*#wxUb3px#0UE`#N}$hjDwy|GjxckavufJo++1FS z80_E)Qfi+29%E~)32xq({PCe+8Hw9QvU`xegL3evrc)n-{n5#&^)8{1bjL>NjZTw+ zV$G{VWu9NxA|{r@aOdqZ$}3PiT~9jd?V)1PLADhe#a{lS$c7F~ox08L(+Oxy1bhGu z`h!-7I+yFY)Eh4e{yXk(g#&XLJl>g#6TZh0wu`{HD%a|C*^uJL8|^JSgpPB)k0ji; zzzql(x;s#N(2G6)13`mps!Sk(qXNHz@t}in`J1XPBhaD{YhMF< zEF2s>Jek|KBkevxPD@73`{d@5V(Q9Kb*>mU6$X4LG*oJc)uTn`7wMp7gXC6+!7m0B zzd&jZg^F%3p{jLwdmylLSe)nDwP|H&FHH8l8juT-t*c;0#EpK{=FYrihLe$x{y)dy z7Vrz<_}&r4yW@`6LY547;w{fAI;6Wq%1et z1od8jy!Z<3@W>-7rj0TA$L&+gHfPv@vW>`OLmbQ?JzI6q$~_q89kg;U2fq-o+G?aKt(ZbSPl7in**DAFB)ZfBN)kjr%<4VS+pF82I7T%GXDpRYnIDr(MD% zHW=^_kN)^^0$^W|4WOeCc^jA>Cc)o*UjDh`VK4xJ%V;ip*3_0ETvW998(KMh>pzaP zE8&vZT_<4$Q(({>Jdn^cS$`8oWX1M|=Gnnf33fq5DSJ10v&|P{<;hm)L~*RP@DEq; zAQQI(E#ArTN70fNIDfu}|GGlyKfb>40nwD#Ys)~FHa}P#c*!)aD7Yp7c(TZQs{yj0 zMQ_S$cQfnGW(5B~qqRL) zkP*NP3OUYFaA*~x4ivgk3_zFfes3I{qkJjd67VeTWO3I`JMB)9kobT#1ReaD8j$-T zq@;XfaNnHZAY(v|gk48Ib?}AdR7brW*@L?LpqqspHIHx3hK9Ap9z%at?6A6O;!ZDe zy!zLp-uzjwBg^&yXE`L468uY%K@+4(YHUh=2a~6R2JPYGsU$Wy74@!2sgLdlnKl_Cdyb#cMYr9H!*EA7+Zdh3F&aNEz$1H5mmsQo))^@7_BIqIX zBn+X$bzld(HrHne(^VnQJuy&gTyzB~7PY*)qyYHYs2C%MNf)tzc{>Y;eD)*qpgBNu z=d&540ogoRh`3y4tqhiZ`6sh5R;B1DHK9Z-zulErM?aABnicL*Mzk=F=5r2A)JlR7#yeN@ZOR0Vqt?bR z+0&VR<3EE+O^!+;-ob1$2!RY7&NkmeG`(liN6w-3xMUvQ45Cw~jQSn#-IE4+@muWq z;|Lz|Bo!am_N6UQRU8I%Mm^-iZ}(=`p(VzN;76h4@n*SV;#3BrytVs#pA>36vlO0S z!u>&h7gUaJG2hn*WU{{Sz43Br0<=O4t)MimYsLQwz_FhUZM<$P@n)Q-T0E1vbM-Rov~hxIP9 z42Y`l_EHU^!)cNNDmp6y zVG+G%fHsKC>1^ca&BV)X!`2n2r5-XpGN|F_yL{ZPD{h{kcek7H{r2^TF^{I4@Ta(9 z2jZd{N^;i+-QRyYj>^J)K3c31wtQFkC+X{|5;5Vh8^X=$-$tmehuPyZHGF+*EFv%B zvhunn{F?I+OgSn$*P2Q$?1Ld^G7R$rKyPAD1$3Yg!y|a9JPoJZCT8ZVL@q(KdlKn!%q4I-yN495#M6IOow!eY@AsrsVJH ze2*sK-b>Z=D*l%=Q#3NqEg%{JRaZ}Ju+$xCxDJ2+ZoZZS2`{I3SL{7dl=%8u83U9f zYWpO%rkN3)13Ef7o7rL;@APywq{R+~r6W+}AVR%6ySqd*G$CMRV)P{vhu&=g4X$qh zn0-}`lL%xStD;;c`me&$F1&qOO_6|X;ojQN}@(hoTsK+~; z5&`5-spp=l4W|z`tP2}~0~sw~_2SBTHbDLY55F_}>YOb%D|}Z(M?b4#FuGuz0i^4* zA2_O7=c~aNK@Qb}YfpJ8POZedV=m|GdWW9OHdAuI;$<(_^P>@59S5BLWC&FVEgKoc zU@VNZ=1Hvg$0!=J>0pfzD}|qeSPEnr;|>?S;7T>8PZX*q;;%1_3rS1}WvTdE2nja} z2YiizomQv=X=c}n-AmHatwr{xPy0ah_3j!8`_9e|2*?{Q)wQ&6Hr0V~Fg3W9)qnw2i&llNC6utvSc`@f2@OVMs9zvIQ26n-U*kz+2BVKQTm1$ z?36b#;$|XSKe7)Pa$O3Gb-9P-g9e%})y>};;1l{>?3hY+49?exdgY2|qDFwli>wXS zDrM+IaOc8DhVtKEm>=#iJTq(mBXCiOkr55~dRwbAx-MDEP|P6J0Vvv76kh5$!vqK&B_k(?7&8D;&)IP5D+%m-VKnSJ-gKwu zx##cpPUhzes7Yjr_6lVE@y{jt*dpd@?3!8G*@@NQ*D2LURr7&EVI2uNsqt{I$}{wb z4Cnp6A=Vy`DVLQ>?Cjq)w2Nc$W?=hJB5Szo8diRHKD3zQxb1H*Nt@>ssD^>t-zaFz zp}YbDj(@Q1A-?2?cM-B5;C6cf_;>^)B+i5{FD+HV0CQw?bg9=u#BGBMBFHssS1@@x zfBQ9qM`nMKee7-|eo0VBNHt`mTo#?|P?$?8M;0uh9ovV1GJg(S0-uht-iR-5Tkh$L zb|iaJf3fb`+wS=L9iS&z%D+!lzhXYutEc z(Hb-Y)SyO#)O|pW$2)stW22bJZ&eN)Ga}JJ#H_fg|Hs;+7tmb`-IlECRk+STX^|yL zy7`e5c~J%i2AK3$9qh~ps(oOnA05??HJcj{>ne7j0hW?eL_~zxXrApFJF`-rhCm}S zLVo7HHOZdMP&D~?D8ibh?7cUGik6n=x8h~d1@U=}p$*qxmf~xR(*wI%zDYg()F(sS z+K*gtNKN<7hN6iHdpzE_w}=9FNoylMz2=Tb$^_S!-TPH;%^G|vC_LP&N6AfP@Vox+ z568qGh+d9>zS1+;OZ9r5$CoA#8I015?ry1tn*g`Nvz?i%vHwlj)M?+q| zE{FgM9^w%1D6hNhkW^Mz55g3~T~9(usR!JI4IMJ@0hb-o42OH8iDAvs;I@$01+ieE z0FyxqLQ%xjV)bhH4wsvAwVCLx+tV5UbXF;9B}fF zz7D;7i8sb-CEU8!;~bHM2UQ{O_&;!Tlme#_wfmOfu*{DN%M&0Fi%S0AjCU17~@7P(msQCAo4NKaSa#@cN%Mekt$c9 zO3)0d6fnsY4-u1vRLtMc?->9zNNd3AWO|c|uI>f;1EYajKs2K3qwqiWrLe}2k$Wv6p9cvT1bU^x`YR~RCopj zTeA(7k+Aam_3Nl-;LxL>t@vG!1=KjtI&DG*2yToVtQsa378cQk`?wIKK7%UN%YXo| ziO+CIm}vzBx`wO(C3y~9#lkd)0P6iSd;2!~ce`xVMD}af!a}N6mMJIk-oS5vUteu* zW135pxfnLE1r&EEyo;FL?i)SucrOE9vgSGvP|kH6<+ri%dYw)5N735-z59&>N{Y8{ zA1|8VK`C%g4gA*a_4q(5!5aX<<%3kY5FZWMe<9IjD>lVLD0_Q*q@aI4<&if4`{Vd3jnlze*%CRTa|%pC;Xw)&mz9W{pi#t*ciD z3jX#26aHw1Aa|~uKp6F`Hv1Bz$NcSfG|TOiDiQn^;z056KXa29po0_0l%;6Kvv0h> z38+I5son80?w&LIcBN63O)^9{Z*RvPy`QZ30tt2iQ$K=-CnhN=`JN{74lq{ry7Nv# z8%HS;7ec7P8QBW~3qssP#%_SHTPSX_1>U}uKYjYmy*8`%ya^^s%E`3ae43)9r5CM_ ze^7(R0xvKZdN#-}*z5j@Wwp*ZexqwzZr;KzrP(LR>RH{u_#sCA?CHJ)AtfWVUglU+ zqmp0@fcXjo_r9UksT4OC0$MZuHiG=DnbF@Kl4~jJ%TqWdi(i?cLB$!2NEs;BLZXW# zwba7)dIi#YdQ9<>A!rH;3Y7d$aUqm}YwH-zh)frNHE@GN@nCqP5m)ONatGn%70i@L zNJu#K+L7|?VH8X%7zAm4gjw<2$%zoc2q=dFk{Sa>Z*?rj*N<&5U-(epo_TV1KFvA9 zNy5Lpjg5t6qo?$ZCNt6D*MQ*6>9a9wtKG9}Thx?aI#lac4P`gd3l1C_Xa*5p9(NAF zTBeO~La(|g?=pYyz3#64QWXTxfQ)fWXthhV7`^ga`{cej4uZrvHY>?&5Bf(&Tw(ew z24tmn=N=%Est^Luv$7sTrGf?(SZb6d^%zuPDp%(Qn-ipI+B+2tD@v^|Feo8yBt@I^ zH*MF3gXDvv$@T&tPhV-#^joH>ro9+EeDUR3_qJpc<#qPL79=nlaWO0@v5zW{bUDW0{)Z;pM->$a>3Ez^KxJRx*Q}lG%yCmTD{jm z`9I!M4~#JCwc7>2f>TXLZ`s25xpzI;fNvoqv_1u1^lWc`0Ofj$;!mfcB+#CzO9Jre z`lUJt`VXTx58Tb%_p(D|=xfeaQ+w_Vj9dOvsx@`_mmuHs=C!MXY5A!WErL9}?W)xk z>k!5wIRJ8;D=Pclrl*X?KMV2j@Tr!4$?UqEg>oX>J<8BjmSMO}wLG;v{QILk1EZLa z^sUK-wj|2U{88%VfCTUW>^em=R&|C)L;yd;(Sk1c5Nr6ASn|D_Sc&LYFK@DWFT<4u&mn{l$EBx^mLyX5hLZeKhl< zO*`LpzaA}ab-sqNe}s@e$4eZ>vtuW7a(L86^Bh<6ZX5slz>c5wIw1>LHt@+>PLmE=hXyfg$GCP|(#s)bo3MD3sWLVIxSNP%!XSork zKWguIh)GoUci!hPTC6b7$>mdRia&+0-+vk#NNsgP70vYM#=W~7V@PPq8hELmO6$0@ z+!#5K#ig+|lYUD(!yF-WNK1}eQa+9FOr2f)p1)sGVHIKKjHJ%*8i?uK<>yZ+XO|w1q>)|+m=}<3zucLYp&8b39^kqpMcaJme znj;VwBT*i7GL5U<0GFrKfV`Mv9H&Ub`UfU_ST zX=pHkT#M_81jACnLXKs9YLBK*#6QpEUi?X?eq6?4^=bI(1Y4I9RjS1kdfWR z&lesQ?)VMH!;?BWvC_Wh5--KHGd#}RO$m2l0Wop$H)312c*3 z(HVSK4)^+f2juQfYWr1>{8mIE9)Sf3^sM6!3pJ$tl97suR>v%Vp5if|!WK}ZN;Q;M ze{~`B-e38bn1r~BWqy3l(`_cj?rt{oC1Rqru{(QfznhX068dZ3PoZyoEz~se90ed3 zl7gjNrkWnEHMad^{3r>(&B!F&^cZ2G0x1ZApzj?;7m13#u=qR1>s?fq);Tb;xu0>t zSm@U+6*y0w>84-)L-)|EnQ6uL4{w=xRwuHbTl&`h2(Dq<;Wn0Vr?7mi_~Z_cvN+)b z(G&?GwPXAe-#}Lg&hNLh5C=dU6bdvbT%o66b?yoE3d{`iXj;eu%er`U5B?w_ruGi? zPUxF-v8m}Bc12fk>ks&1wYN?CaA9uhTb>s30z554Au&t~hRMj*BauYwmk%tZi<`Q3 zXH|~X4}mW>)~e&7NsMb%FCU@eKG%Ah!)-{YNMKd0i&(d4FPpt(PJvR3A zTy76QdG5DM0f>VI2JxJ?9hOJ83(RQ9ICA*gW}r zKhVPk*II^1%xuTuI<*!aatQVT*ykd2vN3ZSiaDF85cc@$H&gU7?=6{sJp?=g#9Ua0 zl`i~EezoOfSX!Lo?>2EYgUk2uv@#J9)(ms{u3T#}AnmJt&zq`I&d52iq<(!eoXKDD zgIFH@-rJ5TEvfYlNC`HQKc|G?ukhJk5b*pty=-CgX~n#G?vENztKaC;%S5-k{amFz ziz#Et^aS_&^P3hOO2ssehSYBgw%`kA!Z%Z!WxT7hloQ-FNLvk>(p(J`P zp4mC@X?Ws44yrCRW5)5lnoCyWaT=r9JD4j>jA-A;Z)d(%%pq*{7~wCnw$>E0a7l=_FPcHK{h&fVPy zV0II<&*>_hoAG$bO1ZVaz2MFHHBC*@6UWWmz(N|aC1AoqltwR?P3k|bSU7jArH~O z>1Qs)r}X+LVjSJe-*@9*9NTqVykZ?siZ+9}-?~ZB83KhQ9NI76)1!=-N1V}KA2wlQ zAR;1qaem36>>CAvzhD@N&#KperYFYh!xDfFES{!)<;i;rFofrHYBJ`1#EMHe;>(iB zJOYX5oXCieNJ^@)ii}mx)#`>mE^~5XOBE{|rKq9NDEIOff_SgQb1kmdF5XyLIICZM;lZ5L7Yl-(Y~KS>$8pFVnu`HI@5EU6OKaNu6=Vc zsH~97)jh>!O~OZb)3r)eBmEiuSQ1;$JalQiLolE!L*zt;7O~D$RbG8cu^xLi^lY2X zrz@)%qO@9?+^daGJkWRrGMQ`7;})-w9(r*dSM$gwzmSzwdRIniAz|MRH@jAy4;h86 zAHB2?0A5FTe55fs77YgoIA)nxgG+H8dz7%scS=PbkvRYL`0x$z_`kqx@c61cPCWqX z)&XV~UMC*{MTX;l8&#p|?!I?}o&|mP+F~+iSx)yN>pDg`zcW*!DQuJJw(Z-T(0S~z5Z=`) zNisr8BE8Lr4jLl&d9)uyfOtN(=Pc2wYtJe42r-=Iv=(rNL=70S9^P<#ah=qDh7Ij_ zfM0V5b!po5sz_toB6<120K1FuC%6bC_?@d6bd`=EY1a!q!|5UNVfEhHqQM5TP7iPi+zd`E<3u*r0&)=j|0_J+X8lr z$Ao>gk;Q+P_0Z9lCd$%3oc#V<>FgQIsEL)E$BvQu&|##jS2?%7{c0OJH>#H%w6(>0 zWC@G88Z1vu05{3mTQpAhz4J*OUb8q*6P>$%t^DZHXn(ISYw8LkWM_U)!qOrOCoAm( zS)_^UL)(d$6dG8!!(n4AX1c0Z1SM!^b&ee+ZuG^zBN1)2b*m~;%LUz}OcuU{;mC^z zu9ux@u^;;3>-&>&%(kyCblt*d92EiEytRBQMA@xub&Z4g;6*(@($G1HxPDcvZ#l>Q#VL?wh$2%q5xa7zz$oJYcncT zv@^X{z?V4}BJ6a*7v`7E9+hr zJzIuGqh6Wna|CbJO|F%oS@k{l7n{cb?JL==kDbEIeCx*Gu#whq*r3PWD`jHKSwi}% z><1%!$`0P6Asi-ifq}&|m~U=uy~r6QaTl;MEgdc4)2I!72$AO$EW1pxrtWI@PME&t z8hW*dE)2eCGCA5;CIwRQ$v7TE2BU;?mCn9Qnx!@cZq~%iF9B zy>D4ZN4)vpD#x5bqZSrx9&qVTcewb`wxr>6Ml5lCjk_5MqsE9D z^TmBfiKk(1pjhmU$)(V*Q^G;S^6)VKa?+I%rp?9J3?#dSpF?@Ly$wmm#szZB(VgU5 zC2GB8jUP=VB_#n-#*>b-`~iPtM1b%KTBdoT;rmQhR#z6=sT;8etN$ z1;0UquTadL_Dz}CaZYgTUwx}#K>1w=ErcT%a^>XxLO-|Pw_N+Bv*T^rE+q+97$4@5 zX3Z!$NrrS7=pfJrMa{U8?WeSqhfvdP$#qyD$IN<*+$2Vsg42*i;5T{w+w;6`N>reO zdtaU4f|R?syxn^kkT@Kwe0~rZM8ItBC^+Gk#6*tLH%3b#Rq%V!do$!L)Qu5<5%VLT z9J8{-R%jC8A`@wxFtg*7O&cwJF*=tNvW(~tsh}}snG1<$`7BbDN~4IE2#G13pP#v* z4mkrH$F*C3JC1F#qdN=Z$9$x5kk{5FCoD8X+D+Uukjw*Frp<{%@43RA{XR3kTH~Qv zHYCGjk(PtNmFurZf+=XOuDuyofUqyLok5f}DUU79Vjrt3kGvv0GW3B=>LQ?^`LyB| z4urXvH6Ab?4O%C?bKmfawX`(yZGS&b-~bxtbjaY3SW3}HuMi4@xT;{~-ZO<4o6+2( zkVF&+KH0sb!OAjW)7sAiU{v$I1MQ}M`03}_RPfVfB#;BA@FsyLVx$3`!JD<*u->0DxVS%Z^f zQihQaWA&vwZZ;SuiYuRtlVch^7oPY?*CF=-*{UY65Y|n|zhJ+$8oRG}S3$E2wUfv2 zQMKNYH?u|-P`Xh<0!~Qt;P07E`+90CCQy}?)5%PyOQ%_l9?h|t_3r)W1yd|G!WZe} zyn0uZ$HPG$|IU4V*D(GR9G!kjSs(p){FS3aIn#d(DoR9V(Jp zck4)MTONwent%Ut^)_{taZ4r8o#(?>wtGax--h=*EG%IE?;&X>93BparZ+l;KJGpq zRil4=Qt!@+Gcp~N$orQa`_%E3w5NznEha{0J-un}>%6(qQ**?74F1nGYW$v}1Fa1X z;g(V6WQHx}3Z?Qvmlq%PF@OcY+JDHw1cRviU(dOF_>{N(;i1}Q%{+pnu#MkIOy&FF zG9}#Fbd2wPdKpm`x4U;}BAaE9qFD1E7IJf``W&uq8&O)b?CM*!Gn+$&uS;<`esx7~ z_kUTgu^r?*RU?J~&QtN0Jym9yw1}CRzuTPS$<{=eO6Mu+R&x zy#*wmyVrhf>|~2RFI8F^_G-Mmj-E0Hs<9Ct@uGiuGN1L=m+M?3kf8=oxGtzSj8qyu z9Wlh;H4VSs`=%2TY#-HeDyl1^yV}0Z4!-0vGE<|G{*MQW^Fbcyryc66QGVNIjR}<; z*@P;I_PqDP;o&*KH*w&3_fIuFmT!#tU@|D`Nq8H+SNT@uVq`M#ug}w!R!-Xe?&R@- zL7Ac(cwPz_NAYj{n3?Meefd%=JW58pgq z?(SXya0e%YH)DrUp4+PE%92#~lLu+`LX=c%Rfhlc7u!50$71p`#IM>?p1;3?DZH77 zhF`!O`{K85-u+9J5oR;LGuT*g%^jILXyjQDE*0}2p_WbF_%4)R! zS88`(>C4^gTxqsb0!4GWnEx<9I=x$@okVRQUfj>r^>FVUx3GdDw=vzF^xt+xXjRxpYF#Yu)+gMOt-53!>`Y8-4 zTFbtCGwtsy@imh|cJo)8SnFgnC$jSNn&h?1afU^eV^%l)ecD_&jsNa5c1FLwpo}C2 z6bG@aiw_)UrT=I0?YI`%i4jd5A)czcc zANV2M$?OLy`?Y^Q&kMVJ*6n=!Yd0iChx71Q$AiwKj+&p63a9^+)&TvDj+AYMCgBrr z7X}JK2fwnzWkF&@4c?-o#Td{ioMS#goUB~z^XwT4Qtgt;N(><2!({U#Lffl@ZQIQS zY;0_Z#@XC#zlea{IuTTWYW(T3kU}RkY!$VdDizyP0p+n0S{VKQYDQsHE)vkw581K$ zx^G`E3OPi<)Wd5xZiv@aolI3LwMfm*M$GnKZgMbPpCvDA$_)T#PJ{adIDrk%3I!9Y z{u%c1W>sSeFa3B9%{6!5^^yWV^%)g7$$|gr0CqQkELo%14<=cvK*I~dEw2q=07J3y zX@d_)Ixwyd`uSL3_Lt|5Sr3S6r~^9{8DIx+gz|cBLD|Dyn$O^B0PCH z5J$P^jP-g8D7wEp_zn$eaO!rTikV(laoaZ_$K;ntmp4zB;_UJiricmqdoDfKQyliH ztmU}Bjo#>94px&F90E-RUWC9yNH1Q}*2dkSf;qbxz;#LgyEDv)djryE&?bG?{1FTT z1=Y@@prUCwYMQ8!a#IL)+I-!ia`GssM*yyb?BBQ zgIPFLFq^qz^i9UT&zQ&x=QPO5FnowzH@ejK%e7ayv`WhTRz6F|%j#zQg17SSO8gS5 z;?K0J;cB*Me>pCq*{J@XFL~ST!gp>&r}|ybYZhgpp@uLL=+PAi5*qNXT!1Yd(xnBR zN;y^wXP0Ms3AnkrJ)3f&4Z&=@1=}N<)G$3eTLK0L&-J^}48SHLql2NL<51rkgEAuX z3`}JpyS=U)g|9sgaD&+au=|hb;+kO5`C?U|AeTYE$}HUCu0k6U zXw)qOic>1EEI^awB)o)ZxhtL&wdwrOb?8$qA21CUcA%!>f69riiD*Vb&qF!fmo#W{ z2ay3%m;qV=RzQlH-(lMpW{=ejn3|7$E9teP2=;L-@$Ri~9!HROrtk$hWTVwtRir&OD=S{<9Td$GW9l#LU_Yo2 zgz`%0-MhHXEjzw%C5LtGMXGx|qZp6oS$yzN)-t!(2@g2;SIP&IQm@V=JH25%%J)>? z`{Bab`B2S&I?W<)T~>V+7DkWfhMi`jJGJTmj}_rB;wwG``rM_^ek7*c&t^5R0WJJh zLs@XR!|b|XVL(1#SD+DEDv6mDgGVY7VIbXG#oUi`t8i_CxpV_NOyf3Nyur*A++FYl zjxZt=8{Bp2@h74|xkSI8L5jvVLbK_7Qjh4q{Kjvr4%QM7XCr&M7uu#j>A+l^5+Z^& zPt%o4!9t@F0EPc!%RABy&cqW;ql?MPq$p`EZs_o(;QTWv_$6m+o+ynJKP>@#92_TKU*C>ue`IrzL{iRB*=)o994bmWJ9PE&6D;^6z+NaqEaG15 zHTS51xOasTA2rsRh#tQ!@VbrjpT_TL(p*)&YA-s9{OP1?k?^fy^V0t?^44~M8(h|v zjuff{nyL{n@SR;;K#=gtiv%1f&dGeg%K79snJqGznHa;A|$la=#8`!Q@(b zvt(e-(1DJK<@^x#vr%PU6VTiNUvU20hGAz`!%z+oKxXIXKaMGaJ*c2@*i;qh$uqz3 zJwr#mSsnUOxZw4*w!Z*Qpn-wv+n6aT^}u+WqM-A-Pudr?_B#YgSa0anQ_c_ZW5ob8zYJ6P1)8gEBJV%~^D4tsAxJu7LDP&&FNawi<2EE zfbl~7EhezA@P~s0a5+L(Kw^ps8t}ZU!8IUC0Ze>BNl6!x)-=8L)Kbe%XpmGUYi!Rp zjI2$I+u?rKy#m8j!BA~#bK&gc0;VKXmy7$Ggs*6+%?&1qoQk0Flp=AGnk~#xf$_i*)51~TMXS@yQtDx28_dp3@q5VvE{vhx%5K2Ot zN-0`;DK5%#b@mi86V_K~{m5ujq(e+*cdlvm5RXDuMz0JEH1?=9hFP4VqlpjFqHE{i zzW3JTBHg;UWbCBC7LMs39o3o}?3IJ(0}!vV31@|jTfud4BhZs~_ht9rh{qzt*y(+5 z6g9v;yQG|7JQvXHSTQ-|ZQGzXALkzd{I7R4>`j<-qoo4S1Z&froY0YHfsT35t3Zda z#BzC(3_9SQ{rrv%ZUG+!2-zo*ajk}viML>b^-L6jkH}9 zY|cuUr8HRe#32;A>taQXv6JP$o#L<2%Ci&F>L^r_HNoYC(oGL~)Bni+8?0u^$WVUJ zG!GnfYUm5D)QYrQ{?hgW)g(fRIBFo$KtO?kx@>95BcE;H4Wc*2%c0P(jkwo2t^Af2@jY;-)`e%E@Vc#H}%tG1i*!BaunXwUB`rR zb?Ym`cTXO48vxd;DXr{27ZZ<$!t@>URE;Sfq$!q5bEXl~br2IEj)Y#ba8-o}?_JjcS%&CrmMt$kteEbv)~T@Id|HJURmAlP@Kz%M7| zR;a?)cQ!5$gP`KTCWhP@A>)4Q>a$K=UcLCQZuT8=`ULgJv(sc>mOAb?F`{c=!k4+{gnZh2# z78FM_g4VkI)admoHh+KrflCRoy4$7J9J;N)5+x410mV(u zr}iuOaKeD5QxPX3_zN1i&~@ikTN`(f1O>MQi0lXqmiFMJXHeZpYP-;;0TN{-k( z==1ij2G)(Oev=&tg(HegLs=kjvKzqBgt2q>4-K8>vv_jK^HtLtUt_@c+}r$38(zBs zFEAfV3GQU6Ova8Xzu`2NH1#5XwcN%)2n7kGu?Y1~vk&kB*t z#X0O5K;S)4?P2M_a(uw1%U+;$3k_vP#)k+9<3R#zbV18&jIW|}InbuWL7m-2u0N~2)vtF&+>7?`_) zkr~vPONmyL!fI3B_Kp(JFug)D$b=i(521qRMQ-9wE6DB9F!y{L6R;M~dWX4?1vnf+ zZL%7ZuH%m-8{b<;XlvYl!#qOYRH3W`FLurYZHDYbn&$LE@9Na00W8j$=G`ya4ODlT zh`W`O+*b#Y=H||&9#;-vuK7?4dO$!q2;m~g1(K_&sWI8>i9%6FK;UlV4zM%d-b*Bg z5r#t)Gq>gM80+K-30^3xBO~JM{8i9KDLpZDI z*`rLZzX+-nH9)vjDGUCwYV{orRpNU`JX)@p`G0NGC4IEk2Age)VnOj8_jn~69bj(m zA|lW};K;X&qSFVN9H=eOS#R{z8C3y0zB%8*P_fAN;gTt|O()`OA$%t=6k;kWQFB?F zYYXLVZAeO&wQAKP%2WRB+frsELuNHAkl9?3ZjW;w&)*ETs;_r-qn#XKeeve76+fcZUeWlVR(bcX1;IYLSwk(3^kd z^M4+;Snn0)aD*NV3S>tCs@eHr1zziY_1eZp45ZiKdEd-s0m*G}w!ggQwbF7~5If~O z$RjonD&xALp5^EB1M5tA!|FDFP(nh&DsRu3T_<}AP8Fv2L(hi`D`=<@NgdxFCEWvD zSU6a)ci@WH9F}jRN?tt_j+(HT`nh+QAE9l9Z?EzLH*I#pxw*YSLkJbsw0cha^P9w~ zdp~G)ao1f^^=`Sv0^L->=B-WsNuc&$Y2>Kvs&_i$)b!6G;Q0&8H6mf&Qm)h)uE>2;cx}(#5=9NN*&sV)WW?T!i}* zG%jZqOGHTcCQk@u3G~E$nmDOpT`1QDd>SDyIEabXjUXK#N?E+x6$mm%k3ySPSU?Lx zqva+~p)T)S=a9`d$3h6u zyGY*n5+ImL&E{@+0p#n4o9!SP?>U{VUX4H4+uhZLXyti6Cqqd- z$G#1cUWn2h-@C$wFOHv)^pTF>J7c{Gbl2A-3x&z1-8shSAaYZs z4*V5BUD^h!i!bcGZ=(upRGW%ny@JUxXw7^ekPqdH16%qR5!u<>8&8uZxSreo;x2*t zpuL1o)su0N^516yZ8|*6TM-Q^t{A(KR4vl}iWv*_ZwoXyvR7pz{G&ZyR#I& zb6z2fiCv>cMZg}iA%Kt`H>=};{iP8$FzMum+4P?_+TP!*8(Ian3vpY>d=aP=IwvI= zXDxxL4SC_0Rmxn(jPs|bcr4ENtMrwxo^9plb~`&{rd~N^|^t62-w4HyhVt(tZ#Rr zO?JmdEUt~ErR9UoO+dxOK&q;8#bGWS!u@JM@Vv6EtgQO`N}Zxp4qKhHv9Yn;n94qz zeNnlnE4rGR$Ef7=hnE0aKw3Ed$BJQl>YLg*$f7Rde}>WV32TZoe7k9|-#$l~IE4o& zP!yI;Gd_TL?U6|LQ~WyE*u@!MvYw7h*$N){jTcl>h4Yh5hvTtx$1#Yn?FDdDTI>c;m3-h z3!;x6gJOh^kn|0!`JpSINa9tUmX;PQXp_~kfXF2An2vbOwZphlVCr&g@bB{3m2rl1ElXq#OyY;;AA4ouS8*D2A^~Vb5VQV40P8$2w zH*>4tjoORsY(jmlz|4U@kOzPP=}j|}lU^{?gWQ)zG3OaN7JlqnFa%J@@E@?~nvuf8 zgYf$5$8Q*!nwEnWr%aaK!)DWvJCJ+beFu$hQ8=JK?Mp2PB9D*Of_Y6x`T$C+8k7uS zf8(&Tt=Yt7JWvtj(-gtK@@d1(**OYip3i3EQSsefe0QdGKWcm#fH&4+Ls-y8oY*cwrD$rkg*Z^m3E~br;e=h&ldQD-u&jRTU8lXfNc7eul;+6 zYDKu%+wadJW7YBhesSWFcO?~Y|hZR17(nrI9i}hA(J_P*8a?HWBCvYrr+r`^MZo0OtHNk zh)Hx=hCvZwq$$!@uj6AWbTTh8oI6)L$(yi8oE@A42)jAQSbcS4Bdm)B6cnMJQU%Hx zh9(=*!duQjP!1DpB2l(=JS(+b-24(ME${Cos)3dhOJ+YrM>1(@VgT&6St7!O8dq@OOVA;Q~D7xRHkS=L#B-SNa5nm8L`Llz37l1YaYVTNw4(+AnZcG5Xm>%&` zc!#$BWa2_3NEtbvy4!NIOSp~OvEa!tN7tL0WPLKNW~=1;-wSKsp{rEl2|6SwC^R4u z9_knsl{YZ9RcHH38Slj7Dk>>~aE>#C26UX9L?sK-u@63KV3Ttj=q;~d!PF^`?LwrK z_S6P+zG8xu#MKb?$9?_%j$p|Z3oHo%+hY?-hWr@{OVu7HFVw#tgkOM7@vGH`o16Y4boR>D?=7N&>djRU;+cg)0D0#{I=Zrksk-$=NPBT=VoT$- z1tdiK&;s+hbJ)G+l+TjFo|g9XP~Vhms?5fF^6xg_`t{(hVl1CkJ(q|=LGdO9j~l(V zi3sQB_eQ`l@2FNzZmufmogpeDFqY6OTNvcL;L=+E*oKuX1FdC0j)cb)41gRWv_+5* zb}1>jJU9i~X7F#h_r5xSmItyZ`#>Iebcf4O4nT0EUJCXAra~MoS-|dv8o`~yVsO~N zll3Iy&~$Gc93S_9L_P|p20=W*)iXUav-5?)eDeY*XdvoYM@g6gT;oEjq(H>2s5qVR zLx?8q>87LEkI7VHE9I6HCGq_Wz6&NY1^q>1z81tx6_~Pxky)Jd<|ifmU<`5Q%vm+H zVgs1JiZ%H)t~?l{ZqCa4JK|{n9x{dOHb5YKPDlg9dO}zV*P7hB+i^x9c7>oa%ggVy zB)~f$?E6iS2ZAyoi)ho%Ts6%g(7LtTlBvGL-LymR+$S+3)irRe#q;*oU(?hj5Cte0 zP3xMNsQZ2;Ei+RL7=OqxT14&~iVy<}8V#W*L5!{fLK9g~p_J=k0XUYF&x{cPx;{Sc z-68^J5>yjw&HxwsagbGe#vlO?PrLep$4qW;IuaA_K~&6u`zn23uJpNR)7RJc?Bv8K zl-PDLx?lXr3vBG*&-olaQ$3Dm`1qvxM&Paf5jZ!RA=w%NZ$l>--ADYq^&u~Z;TnH* zxC_XrQ-F>!LE{99#glXOn}II zb>WsNjJ=LDGD@v(g|_efb%~!r*KVe`O9Ow>jb&p&?F_KveSJmu@ywz<&KFXSJzxco zpEy3No^p%FN`>GHFON!B^gTx9xV(Q0%ntgO=1z;Pb#tWP2Dgsg$}j`UK{E|DF|j_( zbu(7r-Y94KS2?9({T0gAaZ@dQsdO=!YGAaC~?_P|}EF?tOz}HaP zXh+gF7L4C{e@PyDV0|U)P zYZmR#fjQwwEV9+rVH=4Zc@^>N|eSKrj*f-mttMJe++_fupFMZE3rUxS*-?8@N@#YXN-O2r32ctkb5fXk|MpG- zzM8svCL;x;n`2)SeCK%V))zxtt{{DC$M+paaIj^IE*CnM?<2>=MEYetO0Sd)Cw~nd zK@d3lUJNtOxD8rzGIM*5I`8>=$x;92H!3TuRPH9+UtL(0<;l%a`|`u!1K^t;oL*HYC`ciyV#S%(;N6skcQrDh zs^|t{YudN5LV$gD4PpAjVTz6!l*t}8TcyrF$}Pt*+099$e4pw4Uj$n~LOgwZt{~Ln z$(rjqdAVuQ{9t&HJx8v81z??cD&(5c;Y17+>#G7v48T!{}u*yc`l4E8r*8nelhs<_+Cb| zxcraGE{$BS$>UcSQBEehi)Db} zs;POt?F)Hh3VaD-p>+bT?(Ou8{aWd9bYixh>itG zWY4OcCnc}RVK_*as9EGQ5Uo|i#qiVR(g??UFBSrhH+1a2<1t^p1e7}?qfX#m?h4@n zF%-ENM|>}THRKoQjhZ*B&Js*qt4hH@7VRKBnv1Z!bvqBYHAcE1EbpX_s((S5Y<7F% zk^2ad(3e)wsFL)#aQCgsoVw?)RnnEb8#nJsO5)}{ojNKei$X>4nU}!?i!Je{>m~@> z0Qe8cTf4YO#E8e#^4Lw|vs>ow(x-nVr(=Fpw0v-w{F{!OsTOm|1t$iC0pTk=yh0Yb zOH4i(^ ztFf!@@^@pq*ZcbO1uRAzsR9s$p|rA6Y>^UDH+EjgM3x`yoE z2)xvH_qPe~Lan3CQeXo{0X#-%B9B9DQqO^GSYGc-~$SG)6&H zDUe&Hxs!yLSkvUgNbeu>Y@ide6BjFY1Cijl6#=&bg)$#+xdM{~pg=!XjC8nSz;S{n zCLyAHG46e~|H~i6{%6wp@%1Sjk5>vEP>KNT1Ov+@N_HwM$7GFN|4AwTd*$sHTeJuY$;4-F)Ce1!f?nn->Jv<=vy3Twjqyb3d2%eS77s7)i%9y%JoLk+Mfe~GUIrBeuj&(L zB`tqU+`Xo(2mGCD`xWSE9TlGUEs^fU$!qiq_oXT4$SNztcg@cp-r^`7`NqDXHRtK@ z*^ioaXp!g>|7xCq*F@S9O%3FESFeX3)kZLB5UtODdN_ofvIk)`*IBPj<55tAac@_D zkzNOUnQ(`qML?VmT3`_UE;The6fzb8;;1GHL53|02n4NX#N6BfD?*5|_SDUxz{ZY> zy2d%dgVkmzD88)sh^IjP^wE<*f?GIEO9V_zEmjfQzU#D@%hd2zC7xSe6g7d^M~{kC z-!C+tsG`!S+dlMkQrQy?V*!CMfZ|$rtVYH4vWnXAs;=cx%#WzO`V>7A4=}&q%=W2x z?IhE07n$Oa3D>xiQ9SJ{x5Ys4slUVhdxkjoS&b!U_XY#-(F@}JsbM+?QWhPke%u(o zhIcmx4_Z6}20?^r&NS9m^3&*M3TV^0LVMl`l!~FDFEW&+H88R+$^sWy@H4X>yzck} znl+bIzDM-{g|Pn8iLv$#xz#nSQ(DWoJd+&9G3FnC%R8O;`CXHmfGuU2%ZtmA1|qYo zvZzkUzFzw(JdL`gX+F|z0%|%NAH7-LRAPh|48M?iM4IsVX-tm8uDv#1&niwEjAI+^ z6&nMkH?UPpn@H=A1z*bHM%&yGAJOYzQ+FaT8B$fLJN|cUVv8Sw=2t(8CfKRPs?t+{ zdM|FR0T?9!54uXO@r%@rNOS9M3lz10B`cI2m%hJ_?4fk!7%SZO^Y+_UR#_#)DVND! zY+4C_-Xp#5U#10z*<-vWnr=CZ75ro4cNslMx(?(bJM*@U|1oGBJ2W+){jxE_)8T+rnQ zqT;}gKt~ZV%Ai1g+3I4Glp!_f2=|zpnTs?~mmfPx?)>&MZBkuxTnUzilutyrV7sPf ztR9Cn0ZIMq7W_AFWO+!) zKbiJEQUuZ#lA*kqZ&dq_Y%P2QvcB@Cq7h7m3^7gTQ`oB4zPvqeXXAW0V9={^Irx4d z<7@HCL7lq=@VU;VsA~c0aPJBh@+_8p3%%r`$prED(J^KK)z^KS8p-t~vA$~F%sK6ea zb0?%f2AZ_X#7KOE7h2-8{s{NRlq3*>25PSv12MwjCPbHIA|AE{7)1O33Md2myEjUisL>c5zJB zlO}AxE_bmvkf8+e`x(npD}*&0vQ|lTpCiU*W%MGsj`vH`ls12(nrkMkip(SxBq94a z`UM#8YGLnAkj>-rYx-OjHA01V3m&5&G}t_#6Ci!%ivPZiDE-<%sLEpNHu_!)#~;|} zQyyR6MMcRh?iqx#@~>3=syj51lUnaMerQoMAHnejV*b|TH&6$Omd-w9xygtkW4Ata znx|zoeY?i>gVASr0FTS;_F|~#euqigX)$+pS}T(Vi^Uu%Uw=QDaOVmk+Zg9C?Is%C zBWo(szcdX_{-tSPwDReh`{!eWP3>*BSJaem&V=Lpr@fF5a*qCHLwl#kHvspx3|5A-tf|DVMm#&k^ zTqm<*6OGru^Tk4W%i*(URaXnEk6e>{o)B@d>?K}RWLiobb5c=N<~2|h@HFb)c*2rw za|c0I?*z534iy=XKkCwz9-Z{4cwk?NS!f4}hZQhG!>by`e@Z-fKq??8*dWx}*|`xn zYOMq+tspd3-^=ELPsIm%@+YeXVTjm(8;!lCzPbNn(#EKj*AjtxH81m{K>6UtUF~C+ zZcX~mCiwRGpP_hSPUDCtyv(*OAhN8emrewX>HnoofHSm#s`MOcNxiydMNaCYNi|vm z3Mjck*L)uqUWl0+i>Qhmt&7AG^EmkhK7tK$FjNXlt~*X{HEXI*CBxY;r$Uzyd4b8t zXSbNN#5V=fsK_%nc2=HzjEtN9OO@am5a(4vHAMScj_2Iy!~SRCM#!JUmN>3Ci2dBp z*(+6oU!#gifNi_rGQKoXQy8L0FsgHnGt{>iulSk!U1cnjKUEZj1piV|r~xv(wReQO zY=i=tL*t$Zfx}nn=AEDz`5+7}U#4r3ydcdqcEk2ABeUKXc3Qu5|IllU%J&x4YZryd z9i~H1@pm$vOzU@CHJwqc5W_65bc>Wt)&(H~=+u^+{qm(^B1@6Fqkk%#DSO zTPW9X^W!Uzz>nu0?R;HHY7420ig|0kf ztPWF+avwWge?f^$$oB=<2MVIwT6l>zPCi0Xc7h#y=XEZH@|o>kM2Er_|AR>2hY-MA z+sgc%Us$?VU%QOiu7292%(G*T6T`JCYCS3`XPeeJ1szSAMz8`%;j=@A!WwmdU~zD- zmt4RZeWG(cDdq8!39yB4h}oKvHh!S;c}wfS6W8QCuYb0*Hp29YsaiR@vF4IYg)>G8 zY|N+Zh@%qRjTZY9&Is5CZJTm}?YS4oKHMj1aFYB13LiKdp_FZO;{`d>!v){jfd~F> zDl^~uF-S64`@%S^xwt_nz!uJLL!USd(oL_`hpuQBH#dpU(Yb{MCvR`>S{i`%znKTv z1F=|Cyt)6)1Kwzk*Md1ti)86hTd#&UuW@l8`XR*ivo-bH^Zi@8FHVQm|5-0Z~Z93I(IzT*Ym0~7li zUtj5)r(d#R6U-gMo_$UwPRAhBSIv}gjuAwi<`|5yAjikY-(>|V=Bmx}z+BGBoTii+b~~Zr%V)oDpt}xVYF254(5h_ z!Oivr$3>7U05}N;&r`gi>W)$WJi5l(sMJfk+*K-19c&ODK*a0T2JUGzz9j5iNfLWg zwuV>=*k(-&L`99xg_IT1C(a#nxFoW5jKHE@^{@#YKexzNB66}_>AbQCEnBK%NRb3O z-A-4D5BO%%6V5F%Qz{C|7A>95dAu&OkEiT)^&-{&B2W``=RK{8DxDD6{mppU!t)(r82E}&4$(p}5R!xy}`vLb+ z!b92peFHx8NoP;lY1+4>|8{3J^g(FxGZbjeW_qZgx4#^KI)tML+S{~Hm#PV3p%L1c zW&4;>+69F7E?rqEs=730U}-zS!jo?ky)`tvN=wjN zA&CQwc{p}-bZWCwfI8arQ?uvs_kXTG%jnOIh#AO>ik=Q_op8q)Wu_MqXPIoz#;o*M zpdb9A_N9qEr_`eQ$ScOG-U2RU6-ZC3AEoZ69_YtZ@Vb1g+!Z~HE z!*H??IhYr7%{=+`L}ig(SNXIoUg-*w{YW4BDGRy`#?AbG2xxI3s zQEs|(f z*9Wf|Mo-UUA8W6@_FDgF6%c9qkc6?x@;_3S#}r=pMvLZlnjx7;&j+MKHopbUdLZm_ zHwIB&a`SqpfWu#>Z0zi127_)#^3eSKcGA)&n~Fh#$QrJQv2j= zX(?I!tIU3kZmJ3@#$3_y1#e{94&Yk)%NIzD`9j5KYt8eV?IcQL&W<0TRr(H6LX7+- ze86`WAZqEI0{OLZEfPa#c_zb!}~rPcHfqHyueXmp+dV`E%m@s>F4ZQHi$Z zA*3GIq8d7qLOp6WA%WlDH&sHS0MUz>F#RYxONeNWjNAwesryFbI$r(E-p0XJ;!|=A zty}d3@>P%XV+bf2m*3|?v~^293F;oMOK%l&j9$D6?hh>LJoCX(j0e|To%t+=%W-Ku z6cr&N;Yl)_>wU6oSLS4fxQ|k#3n>SBgbjMW&B4rv?h>;2;@A?&Zkkf`1~##s8OIb( z9qgmNKQv=+$c5G>Q+QqbGOD7xJp0Hl+w+kOuP64u9|@9@ZAbbkH$o$7KdzRKId9`p zCTevU)#y(#HS=+H)f}H|{SXBpuOK6*DdBn^d-26L9K(gPoDZcjATQ@YOXTSlwe8J4N3jtvp0lyN6EsSh&=RXbz}&^ zwjeEnggt(ehSIRfGkar1i&TjU#osYpjM)DtDDI7$*xlj@Sl>j9Cbl0G@fbuVPl zFiCWFBHzj#fL1Q#r(4-fxbz4`S(^TS6lCOX!gaQyAqP=mM%0Ykg7aHCiO!#o;Jx#4 zygxF^#?iixfyO$T^)UZ9k=r{W!pN&?Y6sKVNE+V%MCN*q)8-MrKOHu14I&1{$~L~8 z@V#PY|IdfGRh67jf1!VwZt>uxk;`|czlN9p`~(jWtsiGq{p-TKd;Qn|ua^{LG%ln* zWVSmu`FCxlsd)mfS!74^UM0cBq|P(vZX=310Ps!MDxm&r$XjS2(4?}Zd>mG#g8HCI zovB*!1Re=74>-x8OF}(~t3QTAfhR6UjAX0{qNnX9&2p4GYt~R}9v!JhB2S};Y&^Rn za6WpWIn5=?7uUJCe4lc*IqFFN^MkA-Vud3LH-k?0Zz+ilZLeW;3Wz3eO|ez~{UQ)` zj5!X9U3d1{x<=|x4187RFWCbaHV4d4(j`Kc$K1yFK(O4PcEfOMaqT^_uNd~5>j*#b z;nkl{UZwA7fs&4m5UaWVtmc6+mjj1g3I|d`BtLn~VBieB2ZF6XBI;JvcFWn7#Swx2 z4137|lF7*NTB6^%`yVSsNy#@yQX0|GvEE^4cPBRn4Dy^EuD@G7_2l&fvcJm%1ev7v zTA<&4g(#6~Dc@cZ-CZsI)L0>$z9~xYM)*sXnLt!N&vWpqa+b38@e2_U((T(3-+E6c zMl_6Mtfwiur<2C$(#BB~c-K!8`-LFK>UI6A@nM6$<;Ht|KMKc1#)+EVAQz`i7K}VT;iyDHCqWubfI68xl)MMSntd2{Q4k&o3Vo?%A?C?Rh?jYwl4o zqDCNLN5p245c>LqTy-xSm8hj4R@~2@LM2G7hX3=!T#Z0BzNDQUQL!x5qXkBSnpRON z_0YJG6UjgCG#BQzAlDZ_N|GT?DZN_8<4=x_9>U(2BDwbI)2Hp_5?+hnDrn#WqgrH2 z_*VJw_kkkNZ!q}wAXCAw@}W8M9F6iEG5X%5o0fCc)uLul|11&l_)M=Ep3h{nm21~ftw|yw zz_Pu-!h(nR10pu#z=5QM`{RvLzLqD6p-Wu;lk@rIZH%Jy#_93`6+v+^UW1EG)^7bq zNceu9S|1${^&&HOB`J7%q+%$j zojy03E{E@V@gs6Nk)`NfIAMbuiQFC=8f*}q8N>;!c51)hW{TwR)d;>o_T1yuWMh%R zAqJN;-EHyzt`)9=q1^?zG=S4Y`h0^I*njYY1LYR9*%e|r^iBYz*$*MvyC#D`NGb^9 z%zi6khKq@EA3!nmWaGI{%7jyj+u8lRE#_=*F99%S4$nOoAke7%dN|vI*Vx!EYJTxR z@FPXB(}oGJJ_)YZMiD$c`SI=JH%1X%_ahxb?sz)L*B z9;2hz&D^|os|Mih5zw_C2TN3-0(;MhM*?Uz-C?d1jA5T`2}kpSt#0x_y6tuzfrZ6V z`-3e!;y3chUj2P*0fLCuE^Ga<=b5jgfm!;&dFZgP!q4|z@oMXf2Ehw0AmiOk+QtVS z_~L!CyK$_D&D2iOcH}Hl-oi@MxdfjLe9Y=C`o;e$6k}1lQ~5nWiWCF5e%PbGZh9vZ*Bv;@$JS?ZK1B`_?| z)}?u)rCgDZ_Z1i4j0?!qKji5j_@>)pQgK=p0dql5DwPRqugL8*wU#Ij1%-xtw+-og zlK*k$`{xt-h6uQ4JvHrmm3lLZfn;$fLZ+U$WHKmID8#LBkY2p)oU6W zuu%ZM0K2yc_{C-`;;o)tV{Q{o8Tb#ZhGH7~=6AU*<&lC1k~Ff4 zGPL1zCwNh=bQDeYT-rh`5)>@?tMx)P}Q)G>zxnG1{A zzwi3m6+h(M1}5v1`;kv6i%zsx>AbRs|ce?hlW@M&1)YbP%dB4hkm$-*7vU@Sxz0QJR zEx=9_hKe6FK7Ia(j2$5Zf^vqQ-5}7JT9V8Gr-O6yBqz1k{;xjBSOjU2&-ihhBJl(!^D17z z!2sS;c-9u>52W6MaXR?ps?OTS{auPlDcD#Zp4jv7(JA2^}xqPKn*4F6Ur|I+2J zZCdKjM~8WE{bZZyE}o8u;Q^m5IH4NHARznZ~>yTKYCTJWj&1TSyqsMm>GD%r2$q}s4f*w2rG@y3yh@vN8~&orw5WC`8I zC*&p?>9N0;0o;vQ)bfa62gmL%D05p6AMLcl#mtSwu5{&yKCPmI||a1e~`^#eoKSJ82oUZ(Nwe@8`bn;T6l4goiKM>ZfkNYLSLierDs!P<_L`RX} zzYRZK6l@MMLJ;YkwG0z|I%R06g7dP{pb}QdidY2{)D?V6kzoE^_FR_F!xHZg;6PnF z@9v#DmTIXWh%m|7g~J6j*iVctcxOPFhcp&}N$kH&wDn>b4aKDPn#L|BC+D|l!N1&R zUFWVQDgaIvsHl50Vy30Q<^Xk&PJiSm)6il-wdb%k^grs09pmJCBI^M4!4W!5p!te zPmk;G%_YEw;+M>H~bYlq@OP^hk%y})Cpc}2#e@!BhA$q%RR z{dOo;I1?Xd2&S`6*+Ip%OyNZ-wHZ@Y1*h_;?B6zI2p5v{!zs zd@?*d9Ob>xmzxf@dT_Y!-JKbJZ+HqAsJ?Ir)q+MP%y1CK5yG|vz>EH9g$FnQttUd) zpG$yEKE-=>6^1!{0vnr~Uf(O%9Fv*9uSH&lT=<|zGqJxA110}yhABFG@cRFXpXQe> zu~*#~QMy*6xvN_Hoye|)b?hurixu&>fYP0Bn{0ohK>^SA;rY%$_r_L^63_34C1&am z)m_{(<|-64V z=b}<{z##o?!;VlwLTSXbw{kh44$XOP1wC0v-(qsfqG3n`g0>JTpA}_1pS4wo{m79r z6ri!w;(?JF2~4fWj~~Yh&ot0@?R{%|_VWNlwMaw`JnS7>gef%+_wF>Si}ccunN=e5SCmpa}X-p0Vh3^G%k1ahuPM;az_ z#Dks%saW~sJ#c824KH}mNGEC>!=ffzLk4hN_TdIyjSAoI>rS3Vp%`ST{C<=R znrTKJz-9lWW%~y=Rz$!DQP%a>wb+M9!@rtNQpTNgKp($*0M|}VKoE{W;%zLwZ!opgZlZCEaX_|t+OHEU!so~{D z5yR!S^l8nQ!~kyt3eUR89(y!Yj}U>RpFgOZ+&B!kR{hL-78pM-ZWg$6ICZY|MP43e z9^ZYHYy#`l5-&&-Ro`~vNQTFCV=2p4YR9;<&C?|4K%}zccmZBe;Zz%8prh33&Q^>Y zWvWH>9r)yby^Hqe(rtT)SkY);qL>sjxPHc1XTrxIuj?3=30ijnN9`U!k#~d8Hq!3t zNHFdh$D^U4aaj7+<6;3I(=ZA?fD}}?%vB`tSxYM_K3zp}{Ff-k2&owaT_3}yh3i|b zYikL*Yc&vC3cwNXAACL14C^BU$YPt_4a-8SVRBF@l0Aow^X~6YYgsZyAt6?HO_TysD7guWYyl- zJLye5o*BI$n!hjuz~WHQjT%iPtJDXTACxzz*lF=mv|(hSWp@T*r%8vNkKVI}PS^aR zBo6Py_UjrR8HR2tEC|C&(}gxYpn{{ejxNaEh{logUCWy6bY>LFukuHr`?+}Jm4IB_ zZ^AvS7Gfo&E(!2`#8BdBR^ildrsb^;lobgGrm$pwgn18B>J+QtZk-2ke*sy>EH$o> zHW=ixMYHQ@E9*e{1tj@N6)Y}oWw^WmubvV}w55$p(2nBUCMU7)+=&kB+a~3SP;yJ! ztZU5vYHPBMhr0I6$wT3dnhIA0ZE+kD^v$s!uxsl@4(GFIgj7Da-~W#6qv`!Jj7e!Z zG*sxG9&H89-V0cs0He6((=~dm9ZJI*x}zk0=#-Xi#?Zjqp0m%HvV;bF?&IVq%gV zg0#!&yLaz)JhcDbbm?OyIWRmB&1G3Dzz>cxFSptF{}P%kAm2SNcDVGDUBfe=DI0XC z;DJ}w!uCTL&~+2k;O^x(%_iuLMz4a7k}hJ6;9O!x)`T^_@&rBhT@(tBd<0uU=2sV_ z3+R9QlIt(F*Z)3b3TRSQ!;Deh{$IT7hwyk7o@bxu)IH!txl`od zbQjRIswHB(AQ{DS^HnIuPya2L&DHAHz)} z!NWpTHFk9w?xW|E2Zx8X_Mnq-gm8rvtuoSATU$GYD6dYeVIqj?*CgsdEcdu9dqL#+ zrJ0ALd->*68*J_egWsRUJJUwFLD~s<-X%^ zui{5ld0!CWbeHu)PMRoB-W1V#{Fzri&``~^KclC*-DYIsVdby>pgR2a?B-X6Kb|}S zd&X(}n~xvLd+%IA`q*>J|9MiU1M(A=gEaZp=j zrstJSM+xBeczm@&vh{Y);Q8`^EO^JJYFJ`UIbSM_{5L0xNIR`rP$?i=Nh)A0-J{kV79&7C z$-S;b{B|hmPzAI;|1I~uqvu4)%$nEa# zJ_e#Rt7}uOF9x?Z!JK{QqvLJm24K>`wpRz4DuxC?mAiiTSo{7J8xY}>wg51n9sLFU ztpv*FObQdvHt%c~fDpHuJ2i?QV5cKuJD_oL!8HlGFwIk2(1#)s;{`Gg%oxtUxSI{HKbF$C?O>jd1{87Rv(^P{Y>*w4untl#kvQM(4M|4Cc0sh58xOBS-=uA(ViNF`(x zfklk@eZEM##Y*DurD5)+l6dZU8=9rAe0w-tuAX%jM)e=>X|(4ya6qfgN@Q^I^_z5{ zWW=r)Sn+VIz2>#xLY{Gkv?ibQZYnAh?QWsCk(GaZK6Hs4UU;4fJ^=v(0)$AwG}L~@ zUhT_CMq~P-^@=+9{zCnKFA%m?0GI!1s$iHckVioKI~Xc=TBs){W^R$`gJeul-S`P8 z#iDGg6QF_;jG6>yH7FkY<5R>_vLk9%Yam7d4d8~UV&Ges{XBq+4d>6PMgw8b^;yEx zl>E9KrJzW0Cw!sjo$+d4M;S94)kgJ(8ac<{Bc&XSUJgdXl1S?{ zEK^wCsf@V6?m*EBN=yu|+3TK_op9^YXmcrjjm=k0`dexPzEIi*{N6;v}eY zaM+ApyEhZ9p0AGyObuf@4M@jb?y@kT8vF6XN?6omr?63_s;mhHfM+Nu%!Q${&y7TS zeKcLgmKsr$$n8qxu8j=`+=le+G~i%v7X6|1v#(>gU(_BWg=wqMiOYR$_{QIf=p!p* z2aNjK#P-JrpT>4=n#1Z9TdauZviU;12|AeiT`s84Z)xV3Gp2g0`y+becW7Ot%R7T6 zk5cx*{ziX_PohP2#toh7wXjz~VlFyjiG-+!uENwx^P2#=_ag$aoUjR=%E`lLWdC+k ztfUTp`q=G*r<9#tS#L%ep#tHD)qbylZ6Tr#uU8;c31P%((oQ>>9~E^L5|sQ$2h*N5 zQ|SL9-1wnh(6Gz`WvFKnz=OMv_yimtcyyE^f{BokgQyG4``7@MI6PGhs!bE%F>rUy z7G!o9EHP0qvk*5inPsN2-&j5A^2{vGwq~2o8C$DORkU9ugjexIoyA=0hm; zt`2n&s_A}({;H+0>!~G%rI{ln+)=19_Jxm6gqwQnR|U}YTo1B#t*-aa9Ykz9{_(!K zU-?G)4H4)>>IsL3mV|%e^AcB74E4WP?m4cfVly)>Op0e)KM1;P8G2HZ%e=!{ez{to z{2?wA+`oUBYK(%wW82!^4mx<*U21RKou$_0k_fs$F{~PCpu4MD?#csFP2O<%ngE?X zQUSY*Ab}AIcZ6CH=z)bR*cu(1ob&~`opcy9^@A=3h)Y9{4fJ6VMp~vyF7?FZVRDJR zvvUJ*3t0t6fu|x{VFw42P^M}=9WsZr6N9$_x4KlQ%5L1a!7(xd?E+=!&n-R#RaRuE z8#M?mzW`vZfI3ps)JEmu0R=*Nf%{(A2LVCBcz5!qPe0~^_WQ3o;=rq;xn~{t(Zh4> zb3>>(Z} zA$7|<^Pb>4uHzTCjUE{rz$lM$$8>h6$3_n9ula)j1YvL`b{=CL2DyFk@=M!0WWEDJU*1w7B&Mq8IzF`k0ZS1V4^I;A!Ht%qUWXzThX-z} z_T{82TYX4V6be8{D)lad2zPC)%11@C4c=NLpS3n}&v8j8jqtby(_(nJc_4U=x;9ew zP$A_S8fY0-fzAdfgp|1cx^i%^yZoGb0c7kf2a8hyl3;Wgh8hF$DL75jp(jyK`vq?0 z@;RCUAkj$#_wMUp5rW7oflf!fa4}5G*gckRsJs4}_k#!^t0a;JA;>m@!kGM9<;x*$hAlbC!jD;%up~GyraScPWcFDLGm08%iao?h zc#Wr@a1zP*#ahN4?-ztG!koQWT7iZ9GvCeTfg%Gj#P0@bokEjg9O&y$Y;jN4dG_W= zU1CWZ_B$@T5~VJBR-YwBDL)eWXv5W=W!bQt(i7npX(11n?+dU+k;^w4pJzkel2Qs2 z))bT4xl0S{h;a)1x8LFsP=p;pIuQMK#ENh8n@%#wpp60V1{uh+=~UpM0>d5#4)z(R zT^kp*v1!(wCR=A_T5;am%|>52+*I`1Ta^i=Y|O2=;2Py{zSX4*+Gq%{9FLR(1?zQ( zr)rhm*x0!Fh0uAR8P|i9psZ&MSYvcNTpq2o?&wQLG!>D33CqUddH^p(@9<#Xr04Ck zMwPcmidx{Sqv#Iev zCH|~?brW6phTQcxS4Pu34^QS56(CrTPqsT8gR)4K;l6PYH2ng}E?3o6*dJv`m{qlg zpNcA69CpNcp|J@+CyFn!@GcE`RTDnRbqBnAd0RvqI!lMa!5pCXFT@D_N?jg3Kqb?Q ze6;22W=S4hPOIAzxp9jZrUKWG8fiu%Nh893rv@3sIWN+?1zlmf_d(3ULZI&^ST+6g zUdq)$p!vX7L{xmh9|a5bxdC&cR}2lGafX09NB+lLl&{NTC7c+brkDcu4&R6+*Qp<} z>(q6=zKgJ!Cn0;0a+{n%7?g;FX-5YkxbW0J*YOMj;kB`Y{aqNx0jOf!_xwW_m$IMR za7&JWIeMym>UENguas0&3gxbj&|EqPU8Zx`TnG0m+#e!JWxxfJDz?b=j6$ZMTOf93 z3j*Nqk5cZ`Fv3A7S|Cb)9sC7!;z}GG9i>1)&juVLKqgJwo!Z=Cty#e$v75$!>SA=J zCN@5n>=Y!l1mkf1$y+}!K^WXY5B|>Y%!_^)tTFGJ0!z6oeaYACDZ4E@inr7 z>PL9$AL-ZnD~m^t2<`6rj#~b}82?dLGhD9IHW~G4(#hEAs+$nk?f{v^jt37qS1_$k zn?JtA$>lrA?~~5?_>j$)hCRrV!LUOWO5Su2#-+q#t?4lL*)mz>d@wWhQ!wQsmQiMa zV~dTX=%?^O?B^2AEk0|zRP7~g4p>IS$e-7vP;g}7>fz*&5XmxsYoH_Ph}SGn&~~@S zHcQJ5(Pr9NHwD|Fn>#GbT<pv!%>##Zue9B3A(2;f>eVb^YVw$UvW|z6y_y>ar7; zvBPY8YWn~S_Z#T(m<*NVD~|r!+*EPV0ZrZyo}l0x4iF@I6igM#fWiWE>1r@%W_MTz%Ao?>v?!Pqzl$5>9gDmFDL^4T-L17Fwi z*R1BGu?iU5t{V!6B>fv*hpKJCRaJPuwk}{U93EaSBFSqw)=Y?eo#>CB&#->vD^7Q1hrwJl2Zszqoe{YTl ztO3BanBY3#UmHMxj(T4ZMY!T3qG@(WuyM(Jp3hh6Or2&K&Ng~SF8j@7H8nN8wj`02 z_4RIp)U33aZEcy4-rgJy&-F+pEzYs8)_PX0UGF&1Dpc}}=aggOaN#Q_ zMa}%stIAxCjEf6HTQbvY82XW?Yn&rLBd1f786hfVWW=!J;?%%CvY@M1y;i{S^OuD# zjq~D`ArT8pKN=ezI#;(n!R0M#!Y}?f&iw{NiF&Qo?blzFk%ju43a8^@4(LO>bR3}m z^z)_cR?9lULx~p3nS4Dm>6xSRZ~#Mt|>WEn)ivEI&}(!keOM~pC%*(RlOFwAi5DwZGS&+Xov*^ zbMMue*zwufK)9=OaQK2mNTR}H@VJwGA$V`RKp|av?lPb95XR1a$Z!ZFy$~0=qv&Fw zc!5g6xDv*394Fa7r%qcA?HdSwe8kF2wxgc|F4JM9q+A>`qI)W z`LK_m<8zb`nG~{U|MtW*5an>;pUX8P*+bl`NG$f;{K1%BpI%?I|3PoZIMM z>hsh@ViM9+omLWn^$Bs-Nm)J|WtnljCV(XGh4}HnlcfO~ZfhsFntIH_8PF^I;{>pC zPSyTLqVak}I|l8SYOq$C5R=VRKPVc&gDr8Y4xV3F`bvca|J4g=oVUj8`D;7ZC*Iw&+t%iH_3|J!B^xL^%@l;> z5-l#u{8R!P^W_s$M#d?cEk-sJ{O6}h7DI^r|2@@|yTpll2vSl~a_H4yOifRpK-EYP z&@u|)pS$wwG3~5qCMz=+8rof6IuX|EJ6TZLfG!|6=(MG+?VaiHl69J5;^R9H8sN31 z_Dxc`aR~_qa50CQML>+bbe-00?O=bYF5Ua9o$>{y6nJr!KKRLWdCkfV@MN0h2gD*` zJEz~giJP6*#>cLhtA#CBa?r+#H#g(g-OcUdU>dOG=4GTSO_-zY6`UkdAz@~Q_3{t*Cgqqt7bC4RJYSv%14;U^Re~?gfia@yoi9(yB?$-FeecgUjVFSYd;OLfmLN}d zuwEQ*SX9V6*HPK&#kyu89)5WvAZv(v=o$>Na`w3a^=o z2+6g)Y4fT^Zw(+bsrD&1R0JL!@kz;VCso~6d1083xIp0+EXznG$(TGLkmQtMe0bxw zTafXA`MDEZK=<{9WW~kdn;FYdO864m1Q$TG7%K02IL&Rrx&Z?ln;w2*YHFHST4haiD-0!qsQKe0MubceT&uJ}uwM$VU-S!loRvuwYFYiw4eRwI2B8!P) zKEF4a|FZo$X&G+g<+&9TMa+Qb8-CeEjan1R6K-p4SesP7(Nn~Q3+=d~yI#K8tZi*I zOx*=e%aeI+4*lQkekUjUm*X0b>8a>O0$5+!a}?g-qb|SRHV_b*7*n_4{q(rLsU@U2 zAmwvzlaezcZ&J1nHS#@QHS`pYtp7;he0WHg8BCYj->I-)WMDXf(oa8>g4F%Cj}o1W zYe}n8DN5qZio;pE_e#V+?coLx&7^SMe2ys6OxBM04O|CEIdoBQ%}-g*dg6oQwSy9x zzI)9`9rrDtS-4&ZZ*^0B@htuw)vYeFKWM+pC1&OZeDOdz2{GK$pudH)TE*}0ra9>3TYVDD@Q5gq@>_| zI!e8Of>)~JeN+8D{<=ssY)mDoe~Td({pjye?ZcCC|Ex6BFG4U7XUe3NO&EV# z8^iwgY0dNhk#>gjz8Jj)lT#sRilR^_Z9Q?^ceu<-N=r)-CErZV?aEPj6%8&2Q`irTUz>4fKiazOr zD#eXJqQkHX_jBFrl<&K({2$+o-HI85Iz_nb5C`4WGjyv2qr_|0zg(s`<-tui594>L zN}0w#a`=lkZ%4B%idi+qcAJDMmzEJstc9;QGWbr z#7{(zZfnp-`5=R_HeXSMn_>r_C2Lz!PY+2J#f0}7jtlDAkFNwbq>fIyl|5yuhHqgK z(04&D@!V$JH+EP=SSj1~`Yv$MPrulp+TX!_lx&UpS#z!cuo=4GZ*c}={Kdu{$#1nW zBnoJBzR7yKiMBUx-@29|h|g))$ex!!3Fn3+mRRjZZuyfK(zdHg$k$sNQy>*|%ef|N zaW%d#+uMLyM4>5sL6Ky-qn2@Hag|fwU}9`~gw%_2Rr-zMC8X}SlbTf8^Lf;(`T85h zvT7w#h$BWkGS%?&H>BCjYOo6APhP5Gq`+OHWg^!yqL!4^7yM)IcP{7-_5b2|-}!V* zKL7Mk0Gm$+&wtl#R+>3?8+^%(=x9Q zz7kSJJvMS2sa;(b9@lM4xajwAZ4aVIpGN{T4CD~GXhG*6EFLk>$O4Qbdg*vB$;0dIL*BpS>9@7y1qx(N(4^^}Y*%$ivPCJZn*LEhG9x)!RW^q z$mPCjDbUkHZKt`l^)v$Cf>v%(oFQE&2do8J;7Nk|-T{Lrl;^`X&AO{I&uPe~y-Y{W zgWu(>u-gGH^?uA`&jJRw-Q^Iaci3qTmq#vMZeH`IU0>j(jLc12@vQyGzrHEQ|IDcy z7Ln&ok`zNq$L>Abo%TMarO%AkNA+ac&S7=C%T&shtQDRswMkG1xNgv$eqlse^{!v} zkJB)^-$rwE``lH<3HkT#_}nxVC+est(!_D=nemZPq|Ge{@E3+T)*y$%bQl!Ze)w#|XQd%HsZ%IlOW5DvUkLH#K# z1FqYPK)Ei0F+*8g2!-IF&ARjepg<_ee?PJOlk@NDpSZ z$L-7V)+VE)-+feKPwkdls4zzybYW5SboB05W@BC~r4g%$wpJB{3gE$Els?yvQkKc;$7d#OpPLLPzHPaQ?Bf0S}3Fkn`|Xeq1Z~%+V}Mw;5dpOey2yF`bt zdplHCPFmW(#*JA+hzIV7avo8NEi|^B@^+>0;L4ftRorDNN-7T9CQ{8(2Q4wOt0?g{-*Y2E0{wdP zpRG!4>U5BwwfWMW8+mTW+rlD`AdlCq)X~v38tVc6StQeK%hp86w4F<9lq?M}~U24Gy&6dad z5T)qz@`HsWj>n{W)yMmVd<*2ebW&~w>@`kBc0Y^WL4$IxWi85JN$S+vrjkqUyNWNYfO+(l|>dC{8S;Fd9eOnL#fTo&xCgoz~xC zoz;TwTO2S%ef`U!--{mLbaZqNKhB5D)?0qaN=k6{H`<|&hS1|(@X&uELf1k7oR(g7LnBjE-#b>a#qeEd~?){-OqV)OT z+j`IW3v``WaBdulkN*9ODdBUT3x9pV-~Zo9_y78RzFbE#rGNZV7+6h)eEgsPQ~x}M z_kVpH9jm_0@iPAX*G7_X^85brQOG~u2!R^s&(DH?ekt{TFAg2u|Nr&=KfiivvqI?b z@&q7BgXUWU5cHnFcng~Gpw)nmT3lMvv$iGxfOW~ok7PN(a4S)uUw*^?KwJY3%va(p_JOD=kg!E0h zU0HxUT3BSQk3cPX?uxR@YY!qM1S3sv-@aW7%q8395fPBAy1>ec51n%<1ofUOf!&?0 zp)#s=kxl@{QZDPWk?R#OJaBEsCyd<2eUG@GPE1asp{#cW&SQwbsT2|v)ZbVfK==l_ z1_tTdvI}uQ6+&o!LqnRVYu}rmA*2rwk*nzPOixQ|2md>8?L|kiu(GbMj7tC#XYIur zEZBp$;7^Xw8a#}BXS4eQ=q(>^|Gb2l1;ORv0vFd>_UJnxi~!61F*G9QIGiQddCP9T zp#mH1&gDc}9DcWrtR(vfF8w(SmhjypjSNXJ&;$eoTw7t8^mJ{oBB=q#rw_aVe&-*Q zdwY9N;^W`;ddg-OJdcU-2g9*3Kund`&z#n-^b7+8>9em@?O&GwFOAR?k!b^_=9d#J z-XE~_%HE_{Vu)XZah(ldB)i|LtK&80e?CZDsxj6WNU~X=%(+viz z$~l_bFE+@4WelCj+sn73nS8{3vC6wd@3a6m3Bf)C*hpqn?E?JSED6TI?VX2{U+Z-UbM zRW7bkfp+RIZY$$(M0DH1LHE4i;QWH{w%_B1&0m{PRe zrK9`B0Twg?5J*5Ka|arxv{_c!*UKii!jg16w=rSU9LdOcvGRd_qQgS(9d+r+7POC` zD2Pz=5bI_Du7dB)DGdz`oJZbp1ij427za&Fm{E(%%bx-F>X(+^H}3(2E>X}C*UHN3 zqsTSZlQ=lnpg*7C9Swj7UEm#&-!T9eCRM;AqNe0*(QLOS8UmORUd)zKfXR zh}|9ccu@L8lvusibJqE zKJ$qBESk8}AF!LN#e{P`vYHG~72uI7p|78Ior)PIG=ffxfnaD?3l6p&9UUc}`-0#> z$-bm-sDjFk}Guto~JiM_X7dD6qR0 z>K>{AfH=aWC*GPP7Z7d;z7euUcks7g8ymeQq9F-dpMV+mB@#t!?3DNZ{dEY#9yY%1 zH%s_+a9PQ-`+n;UB{9Xr=f!n(btCymfW6X@t-&V@$~jfyTYQaP#&E{P z@|ef?+vVt3wq^++gON6^3Q9@;H^ z!J_q%SmT4$Gpt{D+tabatybx3FfP3g+Tvg!2JA9)APaDtv|oF3 zt!QP%2Kn71BO_m$niRLnx&1#fELZuUtLEvVftO&`N-0R9Fx*WFMXq$;zNwE}gF4tA~^ZupKaB{RJ-{@t6dR*Rvda%*aZA$f@{3PK(!|UWmTs z%NL_;ENeJDF_D(Xv03qcZ9yFIOo4)_8r~~0UU@?t?AAu zV==v9^sGPd-jGz-C9nRc$rah}dl%_CxuJrd^JWT0P^@q?vhtAb0!9^zj8vfTyS8er z-Ul=$?AKdV!C;%AzdBKe=ohiDu&lojc|j;ec<-%pWYR_sSh695Yj`$u?iV23jR0N% zC=)fGZDCU_f9cYtYAEyPpBwR#@HXQ?lwEB$l0Yd;S&KCUf4wz3_QiZpEvDynx!-3QZrP=&Nt%0s6 ze=9%{Y{AU$0?rAuzML=sa+EK?!Bq&G@hhr6h z96=7u`#G9H zu)vd`P_%uZ4a5oASPnD@d;juskBUl0M#f2G+z;szDwx$?*J7LT>tUEF^Iecq^K2)j zQJ4gjryh9&aB&js3_2BcnD1v*L5Y%4d0w(&h(Rtf5O@d*6a##wi?weeJW7O{ieDqz z2e4b=?q^yY3gD3xwE`o*A7XF|Pj>%#60qf0GPn$zh@pVkp{^@+^o93xq>yP@ghOf-)l}ATDI~AFwrKck|M6ZKwnejxAf^yCN z?Y-i-GN^^X*A>D1g|!Ts;)ZS`VRI!iuwTpBG-T3lS53BZtCA3mHDx{Wms=@9oj zf`8`}64DqQ=nR2zY0x9u8{Yx7hM%4B3k>70ymW+pnw^mWP!P>0;je*9tbT`?k57dK z7>&wK&)Ic)&0N1-^!fp1Eb-1TmBr#%we_FdK0;t2lq)Q~46^v;c+gN@oiyb3(euHC(=V4CFqkXTtd z{(y1tLq0eF=gclHX30BY?nmt(`1$)^vG9VrAs?V|*4f6;3?X9I`uOs7dRG}RR!LMF zM#_xhh4&Way4f30M4kswGMKcjq&bU(IKy?w!WHgT_DP% zxb*|7jA%J}s9-EQUj|)I_@+t;2k+?eFr3++p>hlXV^8GSXJ=*@Vf?7#;N;Zmd=l)d3P4aqT5e;4x>2VOIH(ZD}Tc<}+lJU`Y6Vo8#no1|o2{ z6o$b?TQKk8n$eq7MR+fqqN3WPYFBZP9Ng5>)O0H}Fc4aI>-7x{GG?=|5#<-=c;MGt zZSaGIInypErZWso0XWy&We&eakH8O#l9GyVFN{&{GI%X9BO{|9u3Cr&jsE@nF%<=BEamRoim7`LH?N8ckzb$m!VWKr ziD3(+;LB;R9^(_!DKH8vv;?V*e&~5z$Nq&U(7t%*XM?e&B`YtS4vdGP;w~;FK)p51 zIHCSv6tZ{Fdcp_OE{2YpnzSm{UZ5PLzco%{LKfq^rKRQCvzyRBsRP>TWsLO312+hK z+x|SgPB(LNbJ&D)6D!@_zt^(fB$4uojS zyPI=eGCSIn^2vcfMN71T-Kf!b>yzRpNTcimMBM9i{Zt9){@6n0sm)%oVn#KtD)ss-q!?AMIR((dkM zyolEl<2rFk0yfcW}yOV z;NvnOQci90zF8o*$n7qbpHtsaif9{ujlvRjC{5Y3IJm^@+kVHP1FAwDG+S%4V;5+HwQ z#O2j7I6^xI(xnG~{Akosz$mMrv;D@CsbhLyaNwz7k-lzyhHyV=*I@_7g)%SSVjdCe zC0KQbiq1*8y16BPIP2(GfeK6y>Tcdi3kudi>qk8fbyRqvQf{M8Po6zf=g}M<9X*aJ zJUm#pA@`D+ABOfCGxhs{feo|Hfl)=N zGsB>Kq9rF;dA`nCb1zZGkDeNC64)d0NE4e~3O6PXeQ!T}Yu+|h^sqo&Gw!*c#n|!_ zUC-ycwki`tp+m+D(hDY;Xtw;(jglq7A!^}lrDY-KE+?|X37O6b{U9lB4;mS*bIr!* zgd=tp;mgmT?;1ub)!MbQFNv7ov-@;t=OTiQvIqg(h&?m|Ud!3gC^B7`V)|%oZ|*zd zQD0yG60lFY@%p(E8_jXo==C?5xO43pJRf1L4NXnlDAHfri{yZ|VilqBeu;V+BY%2^ zq)~38W&ZE)7#cagZL_;>&&`(;q&&)wZ-a3V!vBi@%nK%HsiDC#v_6IlN;n3vnX_v$ zu7{Wu8J)!_U*uE_RW*5j=q{`fx9FQUiKRTzS*nEX;Zt3$-TQty0mUKtX+lQY8tX#K zYQnBxPka``w|K;{uDZI>Z1)f_>K)8}tOe^$l7}=(cVYa^o)#%SlN&_GpK*ic=D!PpYQ*n2ggUkp!G^IC+HeDTxyMfXQ_4@U^d(uL5|l zM0g6C%W|aa428Zw?)l(d+ILj8^S}FeJ&Q*TPz_08fQ5m9n;Tgm0nih4VMu)FHntg^ ztfbvVbYEZFiw+3%+}V3;2281*IQ9k4npEL`q^%m30!TXKQo9R{;kjrW)rt5puzIG* zdh@X0BWa#GwcfM169-nxdw>|Y>TdxK`q|JZ)9oIVk%(7}^T)>e-Uu#OhgZrcH=7(K z=uWkiJja1O;aV|m3(S4B1^ILO-eg{ZiESH?X0KKfAdJyJ-wvBA&}f6%Fv-Og_i!eq zq}Y5{-L-Vgq0dx!_t_l&4U7cbZEo{T-Cc2HbNDTn$?+|o#kRZYNaF~$dG}D7E*-<$ z(qXv?bdN?1{KB0~i6!E*>JznQwc&mataeT~im*|Dow|;FE5Mau%NycVvIo9~EIeLh zw8`Dwot>RM*!@LGNy(x7^UISZQ$YcX^X#e)JgM==FEi_f@wRl_E(;!^1PAUz}M&@ zlA96sW^iF_qeZRgE9sF!D+vtI#16kJbrZxssF<4ks{+_%#<2Y;b7;tk03`GU>S)CY z+k?Kae?yZ?a74tKlFf2*%`Ps3)_i^S&ryh3tM6`q^X3iJc~r^+UCuBfUd36Eh~m!} zymn}_C{~I-eH)lY3u_(<)mRpw%d5f^Os3S{bnTWv&X%v6;c(N!M*fMp2~dmLi!IF? zM#>+dkR8WF@8j-)UId6&Xhq<1HLf4{O=hTwHo7V&wR^mez{M4a1BG{Vf`yfh;VFA1 za9y1MAy}hI+?osVu^n>8R$?6G;%O4O_kV;UAQR^^;?Y7(H8(UPCGhb@K*gW519fPz?GaY&wS6@ z={|bC#pQSFDwU4~uC2Zr=eP~IX5w-4pcY6BjS;>D)QJXD>ncv*LE|Vgq6e~K<~ts7 zt+p)a140?*+u zM{JJ>%D4w%N1&oLilN zngbf2&Z)xuRk%g!!l3!Qk4HVs6<@jtG0aDQhY%MeXBR-KhrZB}){Vn=(owtGz$*cQ>e`shx< zEsjZAzYJn(I#=!gG=9DQ#O>P6gTVL1W?4m!N^FR#n%?Dgdlru$4Tg$i#~!`eGiRpp zB&_~5_;F|NS-?^TFHEFYc4nWy(Xt>G#ZqfHAmCV5=WdbQ zOx*^WFqZ1MJWG{aat)a%`qG76oHy|_Kj1KmRJyp?9S&0zdwdNu;L_-^~?kS zvgW!1uoaq}EidolyX>@BK9-GW75U>&*nbvux^oL}Jui}yoXq3e$k8=&|}ZmxgZo%<|XL~&4REky%)!q!%NLM3?02>!nQ+}F4JfFq^qH6K;C!~nbOFV2(j)-@|vs^2O`|D6|8*KGzcgn2q>=@z|o zyRee)i_J0SGxO23A6Q#w!2WN07n$8@_*SLG8EYrgQ31?CBJJ29><;VqC!G4xQ!)pR zhp*e;alr$1{WwgYJd^PU4PKxFd)(Y)hh$K$PAde#%--Yns@*qEkL!^z==t-9$~N;)VEjq&f77A%4J!F0khVV3UW)>dHMj4itG) zoYrY@p)$RB{c`IBjjMHRSVVX`LdF<m`C&?Zk0W>(fvzT}PR(Z^<)>=NF;_1@2zyO0 zaOj(#I+Ydbfl)#0^;%1Gb2z#ggQMAqz+w)rZ>3jD%!Z3SXI%^L>&eC^Dl1m##fl6r z3DaJSc!i{93QGlr{^}7@qczdp;9({)J@1IH8`*^|o{fQx*$;eRg@S_7Iys>JfP{}_ zI<^&Gcu@80j(B={I?cIgtP>j5+}d6|`l?tv>DHK_Mu*RiFwt8zC;RuGI3Y@$2k>O9 zDs_y)jEeze?t4J@bY<2$<`EI{JS3PB`f-@ zcIQr0G+ygce?wbLo)fHKEEbGNe-%HrHv&dHj@E&`^R{i<7^(sqrHF;6=!?%wZT5ir zpjyFD#mPyRnPG^tkqtygOhw6PTMCf-hmIcuw3-@uTk`gB&BSpbHW7XM`?;_J*VpLh zPvg=7fDGc}^^q_3!4A80>v=A8XO~3S!BT}rh>Jci3fqrS(_yej2WK|61K-mer>C=~ zJNflN8A_S8_o7AEal0@J29(sH&4}Eoz(27t@SO=d-``++Iy5@^3aEgUn0E2mfFzVB z>4bJ%C<9nP!v|Up@rkmKfdz9fC2;+)Jb^r~OX$^gp2=dzo_u>x?L#lzpa~ep874n0 zQZg{_Ro2zb#TyRoyOx~1+VXj%7gl;jYinyyt>z7uTZ5nP-nt=V+T3;zm>gLlRz&cN}=xb=K>BRU#M;?s)6blmfIY1?gW_uw@U zNT4@7kp6O&ioeNoAnT)V94}&KnZe4szcoJN9jBeio}-r+ttQ`V7iOtQMd$TI8&723 zQ`pzo*>gJv@97GBM7MD&M7_w3qehrOdiRl!OYm6HY9Jeu7r#jzpQ_Wk6Wr;ONNKa+ z$VhgP?3j|`;;SjQB?4B^f_&xtB33cg(V+?iDi<&B>XIAtEN@0Zud!6#fdRTo9-O?F z_pqg~KFL(`gZ@cs;RN9te|a&mHh&?kVRco?v67g(=>lYKE_uUVM! z)Ij4-Sgx5Y(8^%>R$?%aI{w7pIm+F;xvD#9kw}e6EUzheyWI5&^tc{ez2AP$C{W55zs*Ba`0uXl=lXlXi9s z`T0lQ9NNq=h}@9lz-t+UDMQfyM5J8ZG)wN(JLF=^p>JC`Yq+|3EgJ0*G;1d4nLy4Z zDnGb;7LXmaaf`UQlO54I3Tj9NEeqRjwI!Pig{WmK}RyB%= zan$La#SegCKo!D6$gz@BE&mg6fk`9*7Z#gKL;Cj##KL_+6b6i~3{mBh;rD*Rh&5ov zX+=Ev(q+rw&1PqioC9A6BhNn=zPrXpenDjtRiTUmf{jn-QSQkWy$#P&b^78i3T{3b zd`U9wWs0s~x&)wgUwrJl;aSTsOpL+<`IZHiTp-8ns*|3aii(QF>?5s#XzbrNjbjmw zC283ZBNp!W=+PFcZFn(hB~NrPTr8G2=!qU@_vplunJj>bD}ZUs-gJxYE|2@pv#BUB zBdg=#=}J-it98vcyShx0Qx_UKE1K;^o)-!zz;qC6vvBIY*`a{lppO5tFd`x%a8T72 ztlt7!kzThpHSUY(0HO$6d0L(@kL#ATB*VpHOC*6B6#~x)wR6pga&>8a> z!0`YvgMh58M6qOCUEi20nj>OiEHgCw%G)f%7Z`>b(JrI%uDcprJ=+^06GH!^1j20w z|3?W#MhT#G3hcA z%IC*!?D$%aUCpS{6Rc?;sh!=DF9tZ+LwPWDMy0)5X zOJV6*G3{%DT#fG{=t@E&?C>%fd3ioY(%QDL_TBmQ0P^VMq$_uOeLK-zzyI*zcDNHk zjN=GaLyZHN`*M`$k4CMD@HJTrj6~=EeC78n?ymsuut~rm3TG}*x#K{C^#rQi)njGF zp-}?t{|xU)cfGv#t4fSaD^{+2nSD$J@QV6-*NaVURew3gvP1(Y%NjZT|3G|rLaG^~ z^vJa8br+5Xc9gc6sL4LNqjPRNkt;ZjI*ac}Suwg+uhZJ`m7=!HsE*dW4HD7idB`Uv z*I};1`6s<_qL5*-r|dq3Fkx>$#u-jB_vfeXqYhq%lXj()lvJm_G*UTHz4dJ(qH)r% zR8mqpCbb{!jRX1|Z(*m9d0*2D4GYKZV-=bj1jkfmtO{!OlxFn?-0FNa3_dqGc^3BU z!ez@cf12Eb?1_R?F-|fDFD*TgX#{x)N_E-^41(eqcURRj=cu}f{G%<@_If~zNAv5u zN8HGo+y)gISzss#Z_r`x_*t^J`e9MJ;_7uh9d0u>YFsL{r4w>K-8L{*ey))-< zqw`wbsDIE3J!UfX6#3ssE1&)SUA`#LaY*{0^~tk=^b!2O*?fcC;tAk8{N$c!L?FRk z`M>%#dHHha*Ut96de3Soun3whfNl(_Ko>2IBry_SZ(N9pnfa=}DyJi9bFuq`0RFFk zV4&u%U7XXb8A=x?CEH-kLnuj7cSeHOVgPBJf}1(18l#aN3Kp;YjQo6@{3a+V%&Tmg zN_MMkyNwPWG^q|=PO}SU0@lpW`)@$@mT<%R00`ggU$UDnKWIz)j>qMqbK`+3SBTh$ zrZf$WD!*?w8`scJrni7J>APkMKG$ens}kykh#i-|-vfcGmRD72-R+&vC#8CKY{Pme zjD4W%pa)NM28Hc}g(dxn1;~?#ge;2Np(~2eXn|2Uv}a37x9@{C@EW&Frsx?4$mH+$ z|5xC(g8MO>Ovs;_*;Zd{v@moewXE8;52X&>5KB+JqnWActiwWZ%V7M zs%7(Ke{J5Acexg(``vEVa0v?yWd%J*=ho@dr|CGvgtlW0dMK|5#{wQTv*CRpmDEK0dP$@sT}nGMME;+tJ zz&*CTLyInH%tBZ57}7lm{=L9J$tx<}Mq_ucpy0s8c*r`@f=yV3g+{KYyyJs+KE$s|%~u^?ms~DF8Tj@9K6)U3*s> z>xz9zceVR4vMUK%F#Y0Wv>2Grm=`t+M_=*`tRKziidjkC&|A>0jD^ z(-Q2536Sx(Ma~-_j^>@TgwhKy!3+=tSf-NNv540@54E-P^d+ykXFL%P#)%e{YR8E4 z^zN!YAK87@+h3cTs0TCPUQe5kc?2RjsX%$3q{oAVgn{Z#1F3MW7nqw7^y`CuEiYa} z?x|zwGd7v@ZRc3^Z(M!P_6cCX@9SB(I^`&Hb(6_1{Uc=~Z7El>xUwJBe&|xr2xhRY$sJIch~?9??W z8db?qZ>`K(jim~k1@jiVdNbsaF$VW6JoMJ2BichIFLE8o!>a?ojvqU$sYV*s*P)5h5)~sd8^PYYs+oKZi_OtXyS(_>8 z2W4k(K$q_e^o23vQQ7HQ2GJ}PJ#dEV7#YsS0R!dyW&>uxY+C5}h%|Np0n^nmPSK_V zB9EVU5qvW!e3+(!ANXC%PzX-@Kh*+xhxO_w>-tV>yVrf7VTzYkr zOLHOCJdy2a=cOT?U|jh4>D#x=y{AZ+h#g{Z3hKIH&2SUgL?`hcLBw zWgfy9LnWOA_s1t5f{vN+SWJ&)XX37~j&w?_FeV{0f+<#(Adl-Wu@E7b!N3t*MlD89L!+<+bYxhG8cKt_CmXZ`ffF9fqXb->|K)+$Dv^M7%N% z;2RvxI%v88-^^aTcriQD7ZG>pIc2$x1NRx>Ryn|KSFoS@esNJP9=cQ)#noF4RN_8@ zN*NcL)|~1nK2)f-_Z5%H<;%E4+F?1wI}8S-9#tSU_kvGZCgkM~tu;S!(63$|hH4Vc zD?^PdK%xK&qvuhZ=HlV{im<00D^n(&0;s%L;q9kQ`t<8{W8@c>P5cq`EdHS4*u##u zWc+3Go%P?A3wjDc=;V%P;ar^ej^6& z@KWbb&-v-jG(yYRyP>guf}achT=2Fg>8CE~b0i9;?n&hOS->78M01O(q2Xd=*(8az z;*se?-=Q;Flr!%p5J$7|;h&7bh{HdA{qhEv8F40C1q^|MqvJN{U;v49J%0PZDrm^r zqRtM)ILW55U_;mFI$$K(NqOhN+94!*tjygF>TMDb4-|i0Cblu9|6b0) z7q6PZup!xf2?IaFoER04(ngDw3&3U1;a(%6tSqE@bgcT_yJ=Mq(94d(*y+;rXu{9p zmXgHb|6B;OCnx?MBbr9_=(7zXA`BF&82j3xg~zq6T{+0U2T zg_lo@A-XW?JVW{2bcW#qHT|)!!ut$r1g^#eTPuk2elJu$;$8z~GN5 z)H5$HPpe6L&z|j=EpWK8sBuZ5UJy9}b+Vv-seAA#etl!{FFNmN@WuIVq)QaNQG1yf zNONps>ub^zYwnn#Vt~f3!YBPTq02S%;K73l;5i}r*#=t26eY*+h`)tte}C8RPmTRoL!gX&$>QI zyvm@N!(~x3mL5Rdr_kaxrtT5OQ~o06oW%KWW%y)nZr;`DF%{l{ndJf2a(4*{g z=Xz&nXMfsg^c**S?8wG9nsD{-IK!2F&+#svIk4H;`*?mrNQAa&?-~A0ToB%;8D>Te zy$GvTnI!|cKxXpF=m32lO_~aAd%s+Q&|P_?hx-V+h9F^;Dv+kX>X%*zu8)Mo+t}P` zM;>hiEmR-cZ&HU3Sa`d&(sN^6_-q6CTQ+a~!jKg70t@hydWi-wbJn2eqkJih zlm8&|^*9i4=-*4@&g`(SoTe9>mO>p#p`oKzcR3h^$oVKYv@79b32421T z1fS<6wjRKjMRaC>jUS3RQ)vH^=ir!5c|%Ql43l8FM{_+oUb5_=8DK%__C-keWv}B;Ij52OYq?Lc)uDl| zK)(Yx^BPd3^~V^&655kZdsKWLJdn`}5()(9&@cAPHx;@be0>bT!P^>h#jsqjxiW4m z?94x8WYis`GFgs3fOm#0FsjSoL=8g^t-fAtm$KgM|4=q-_Gf8mL{(r;UKT9;Q2a|x z0=mL=#Wyq)%0L>HAxGmr(iXTSDq<66l0F$28+k+`jg8_jsTTnH0g)1V*AYzXxpx z@UlLcwx@AoFY*s@y^;Bdx|5a%N{w=0ec`XsSgvY@Dn|Dj_@2=*H+nFmW=6%?pZHJ) z&*<@s9xE*#GMb#@&JT5$k6;yKIzHd`Os>&0Z7LTigd=B1Ha9o-uZebOvEM5$5ZWs0 z!YY5JcDZ@&}-``cSvwHS6K63>F4MM?=k^ z>2Xk7I*@Gx9U;RTS~6!mq@8SbsHCCptcK^8t~+cy%l4e`T}$reTm zauCtuCH>UJ9T&~|IqOv1et%k|&?*wo7H=pgi6)a-2Ej;pzS5aboG1skv{ z=gSw-5^UkT*0F^%SqNJKt(?S_K(emq@hZnyx2&%1AB$J<5Q3SHKJZZNve<<))z(e{ zt2s+$#|!L-b~Zw{^}3t|WKW2cC|?-tIsBJbQ1A-iX6?Xuh{{fg-gp``&N&muy!a2r z!KR%c!~bNC?aZ%D6ku%#yhli;lfthn3`&E};?{@Fm^9;VI zB}ikYGu-|ZzXi*s%E`_GvLFpv_93EhY0Lh19n-2Yz2F^ZJMvHt;w zBRVP474A*m!P`5KD7pg63g7Ih|H74YkD=112u|MqLqaVQ|0Ifb1D70l_X&{XO695X zGd_$!?9>l;vrCz(r)pjuI(+yxv_DK0{l_gpV*)lzH{lqIO)V@|&&yrxRQU!hfa=l8 zUK}|UNW+`^t{HGM@2Nj-VEJ2X^E+O{Mufu?&k*AvN567iu3g(=)uWFl@T20>i#Z&_ z^%n*YNB(?@x-}$hc6hzo&6a)6C>pVKC0mF&|J-s z1rbo*Goc-P(3dvars~@devRYvdklKvt>4jD@u|3Ph02&WZV_Hh#FQ&=Em&xMHZSnw zi~$aX>EIHuK3pNehf3;u<;s-`XJGqSO;VQ6`EY#!3l!nycx^*rW=$3U9@Hg3F`;ZH zgE>m>E9Y|oK7is(fG40OoYo*Bd=XNRUvrMF22~E`rVH<$kN`Mo4dSey8$>!Pg-$8$ z-nC1@@sFrQhH=qiY`x@do>eHx(n#2M?vM*f$IThc5tWAF?HWmxsnm;xdK2iyQA7@k-)!c+P9OjyCkk!mi*g&Q1JGN4TzODejB0fHqFn(2R^w4C=-SPT^c9#PK z!;c<5bjVt%r27sq)q|z3TforYgV_So?we0UUQdkuB^j-cU@rVVlzH#)rExi#iO9Ta zRB$SjD`|OAdJ_o6(AcMHlVZn5#GdDAy0^QSxnH`;6N_@8+Hs*7B$ChC4|9J#U2;rB zNy*ssoHt@M$^o8(Wh#dWk4jpYe> zwn;DefEt2)hSc394UA`6Rjd5m5T7qrh@Sv=`4HS5dZE<@dmpBa5Vjyn|Hj-c{+xq_ zK`vbUv-Ih14Vag8Soe^A^*{Uo=k^X^zXZ8K{SV53MWv;sjpj!APv4$eHdG3|f|1Vl zwr%R_XRlwAp0xz#g&o;oGs0Uxp@jfVU+fgeamf?Ejnr%-mivt$^260d>m=;~WBP(a zGI@MyR+J*U5@tK5jzWq+%%2pxJsF%RZe)Lgg-TjdxL(%1QZr*u*fet*@snp3kX{*N zA|xYsb(CrZ(F(yEr85AV^O2r`dO8JR#@OmM%ERC&F74^*DQ|9GfXMmX$$w$0sZtKcI-NED?9l zZyVt;I0p$Gp8FE~_TLVxd{@fLAH4OBvmHbRPeTddzRcus2&G%!zSXvH29%STy#Gka z`Jda4)2&}b)?m6eq~xN{oQMk25ir+2V!a-BJIW^Vf$XftCr+eK~T zhm`L!L?GN|$BzSLxF4Ge=N2q(GJcXpsI5PT&lOYZ1sm<>cUS>Hn**ubi|{GxTk?yZRH4~)>yQaH1uk`MRLh!oKwH~?i06?AJ5%4*cO5kL zD1U6h&RdCJh5k49`Wt|RjS#YnW#Z`fNmmZsk7JAk{UFA1xU&J?8i{+jXeJqsu=pz= zmgI?umR^jGkty2&4H2fc6L8(>wVeWOJ2loh1(HcJ+!%urG4Wxz{RgA-x2@sUt+`?( zvtp1r0LS0mpRyqA2^&L#3Q&Lr%EPLrn`fyXPpMp+{W0E1=GEvq603fH$FJJ)T&s3> z9kBAVm^*czsX!G>t&7ho@>3Cts(Q3VbNlwnnwsAB`Km$BCp}u&jHQPP|70MB zlrpxW!NZi=+E3UGbeoCL8Eb`qmC}V~8Q->9jKI-^`2qmm0?Jtt>6B34Wb>*V+Mnt3 z7ILMaN3XL03Zr(;_#g4qXghXu0`{b~u5P_^2bLy+lNQ1w;2#vU-Nt4eq; z0&W;-aI?f|uW$1#R+3XJ=F`Tal!5( zcoNHr><8q?7Abh@w^2?GrN9rQ?gyme%!+73kBK4SmiIfo)B5+1Pc!|Ht1vwAm24&D z3Fu<>!;~lBW>Q5hCW;-}FuEU@F{r!62rtIk^9~T5(b_w^>5y0LNJvJ*f=OSs<5dcc zUoga?l(I3PyAh(G1vFIMC4W1bcEnwYMzV^|V}6`axy}XxTJmwO3PlQrl;&? zqj>|+L%zk8ZIUD(BFLW z4kD1`b506mTt#z*m!Un-d1Ut&U98;eGh{UOVMWfOiJfS4-2G{Ks0wlqnyKh@BA!6P z?txRU9wcF0Q0%>GJ>ARQf9O3hIv+G>=?~Ypy}*iyRxWI698gHP zsXzz7!uKNdK)l-@emIB89^g~tJkSkyLh}iM1;WZ2SQN>-|AdL@^Q`zjO~zuw&B4C! z6?-N(I}I!fOy_e@<3a`Yc4DfgwTX=iE}|P>d)>eXGz#P%153lw6K+QdbJ(z833qnN zyji>I_U+sIkYR+eMU_YM;GskBNi2JE_COyQoe=E<{g?=(S0_($2?zvW{OIAIie5+5xG;rwBW0n+CLj)OM5Bc_4QH&RJd=%jdzC8zGv3NbS*MHcLjZ z`x)&mVq#(>oqmZ9YDe}gHC0r~u-&i(+b~}>6GHH{(ATi^l56{dSoSV2OXE-r-?y99mxe_wU- z`DXJQePklcg(DG%W(>aQS{JYaqmCaQI$6}nEZXqP_6war7$O-lnW6sLg42m)CQi~{ zEKklGz>6U526@Rqj$kyPR3u!!B@8{pbH#6pYu2yAb z^=e2Y`jE+L0LMTmTN)8SFTVE0j@Nepea!odGz)ziIOxto6gI30V81*G`_TRCnisqU zrXqx9{SrQu8rMuhyKu24nIuW>rnE3E10B5}G5P3{c-ZX7ScpPo+b@;Imy@3#akOr> z4Ug91nB{jBHI??Q+cPP)W28%RqcPdGNM_^en$-U!+Pj7-KM+I`+E85Abn@iO{pJ7M z|J~_XTyBI~gCmVqbutI3INJfH5XTOw#bfkZ!YvY;Atgy_*8F`M$PZ^%nIDHi?0!eI zhY%B}Alu0s6e=O%C_H*7_Kbf1eLru6gm|G6>+Fv!3Y00 zMwN%uHPz*OAIhWGSD$LAjo3+s9{@j|Kyr?kd$O#%aaS}z9v*q+v9B`Ni92G;F!h-L zwJ}oQXt%~7hD@dT30*9AFiMje8nzN&pIOn|sz(z7o)_&lYfsy^5}cY4y8Vd01JE-8 zf%XHyEBHGhAv05wJfncMC{0D7fQ&|%_Q@tnE?Fp$_GKScqJSv$6gX9#{zv$TYKObG zXD9F-G{`@Bh~(m+?OnF&F*hAjVhK~+e#X%e{`tFH!6`c&d|es-7`z7gSKnSFEF6Tf zoRB9>8f?K+g*MjKmjUR~Cc29*QmzL81nwntfIRtlo1z}HT70e$hd z_(>2KGA^ zVN7D94FbGo=&;Tsz&1B$0LVhGWr}nRfiukZ=y z{M$U+CFnl(_qg}mm=heEirW7S!rHx*#URAWtCC~kg^r@E%p*kKp%Xg;H(YSYy)n4C zNDiyRr9Q7(_XYeUG;0Dz5ka<{>g1t~6!}N`LLzmYN<`Ytgs&MqzPX??_Fpm6Je=UJ zkkT?g-@kufU{Bp%kDz5Bx0`bgE$Cc>4#I`{>O3YitWmG5!t8(XZKM|mqq_SHMz2>I zJ_8N}rOv<9dO%w90kRzOWuixhj2(wKS+1?w=40@5bVfN8LDA;va z^9Ot4U4plu$l*227Rg>&fD<9}b7(*i`iGSg5}b*mbQ35vlQ-cugR$_+mARh9YNeO@ z5|@v`3wRI6txt4Gxd&?$h(-kWIXF3mu{}g>pg>4qsLSQ#3d+9$8Ur#!mg`udp!`To zxN;9Hyn%M$Zm&Y%(#~?m`a^ofsYxkb)=aPP$je&l<$xvJxD`Y$gMuXR6)P+h|=l_9LA@t&!Ac#|Mn zReYlMb{&ay7SbTP*~D&fklK<4_C{j zY5fu8;h70dleL(yoYhI`AxH6AY$T#U@EiA!W~8}@d33;IB7jYxg%yw+v3(;a#*cC+ zx06(#VeaFRKdgt6Ok_4O!CRr;ielCo`uO?LEqq8(>%-~27q2x?=*n_v;6qxa7`wVJ zZyM=O!(0!hVJ21r2oT4cf~-%FErfyeD4K`l`~>`KEOF9f1g4s0Ovtqiq+QLj6p?;4 zh{1~VT1k!yIFvBmz=RG}?`5!W>RdAcQS8i2pQPyy)e1uMz*))tto zP#$0@rr1~1C)vTIJ)~a#*{zAo9p?Ze)7ocvhh`C+aI>JKrm+n$!4n(>2q`t{`isP% zm$)mK{Dzo!H#pSl*Ul9=(UvdiS^Vn<8Y0B&{6QqNDR6(ZwD^Mb#i;vPy~ndSS`aPW zb~xWE9{oX6Cm!1XxGAX_GJhyIA}r^Zt75;IJ&%;uyC5(olC!lXXUEj|uLYAMpM|NF zNA4Da0c^+Q_J6GuJ1)HY0vI1X2Q9Srl-1I|K7DuF+49DCV;u9_Aa8b{CNp?OsHDW; zf{8zHbHj`=J6J9>^3G#^lBS_)O!}MiaakZs|II-nup3{D4mIt$kF2ER{NF!+QbG=| zC09t^O>^KfpyVj<6&vL-E?^hQo)u+j^UbD(Gx3s0Ii}p+a!wI@>lzdjvP3g?so-1& z3`Y5UHq%Fe9p%2x$5tUH{|$jgSR04G+zz6Y8kA;Jd}z(@l?Wj5nE1Vf(FGOEH1qLb z*HdkXeHr6&2g|#zI_o2xh`c`}b->g?coLEs#PjT+TLE8nz!i^5YZKS+HOBLn@;5Q| z)3gD)vA5I!56FR~z@k|vs^j!zHt9xh*2G*T4JH8~DrC&0%IH6WiCMs+Hp~n2 zs!l}E9WXMkolC~C#x20p*vFP z=5S~Z`#ycDy18gF{#J*H+Y-=XYu9c!3fK|7EA!-8A5{ArUsIA@r;x@~OLxk7G#khw z7*0&S4|WAKSO#8lxv=S8h9*V?WB~dpIhrXT4AoAT)zjDiSvEDX$7KMp>YE(Vr?dKR zj_!_;za`b=le4q!U6JzwypipI-ETdJsGdCo{pb?B0;wnq^G%NpT35@e)UuH;zz0Wp5087r2`Li46qYanTY>>r6%qK)qjA8rfs-jcvY6^ap5czY_9E#k8C(Yl;y`KQeuMgd#3ntCekfr80Anw5%01DXD*rC zl6rf+iv8w&YKXmput}Eci-XGe(%4$-ou)8G()H!j76aopU`Dy;wvy7b6s zI}YPKA$bM9=;DVOSPXklnwok6V-R8mnpOew z%`kCL>aD^)cnK6nWBYbi27Ibla6$=VUU&?Ewh^Bec6ACNCVY`x640xo1L8#ycm|Ek zPjZqW>gQin_8ICjiB8E-RVWPr2id3o*nj=M9CU)r)f8Zu+gA&<85?ETNU2X_*?@vb zas0jr0MApvU#HQKF;o!-Oz_F9xr_6IPQQ4lN#bsfDbyUT)4agWf}P1 zO2nexwT7aU@ERjs@9Ic$K;bN`fYJMwl&x3047ouefRlnDYF0;AGm zMsQr*H6X?mP=z=ZHB@)%hLG&aAiW4iU0{I!rZ{&Od_O$72B``k@1c+$Jm_$ug=ZEn z`lAu06Y9a_{I*iJBHT?OD7X|Rmt!Vu!OfqQ&;rr?12G3e$siMoe$2ln1;;sK(A@*^ zy;BX3XF#>_4@RNs(eThAByA()b=AC&tv~js?Ny6f=I8GOaAo^2ZqFc40d3oR6s*Ts z55Hq2nNs-)mz~J%$QVp?f+k_gEn)f7`;}{B)gOJ*>PHEGFVziI0Ks}ikA-|@p%V73O z5~rt4vnlk9p&f0ltu|W&p*|^ZYdid5Y<1;p#1>FT3$BI6WFE^|6iSZ+%Hd-zZ|~Tk zoI>a3{!nl~fzR;)KEP?qu`@Z?dJZRN*wH%63;V0^v3V`)frnl{Rl4K}a*rVk3LZU+ z4XX(aIwF%g#HP&UJ3OY`dd~tu1mM~TySQ`fXM8zRfCglMyc2{2(lD(#rJ=PoYQfBf zy=rP+{HP<1&iUGN$yULN#`*eDyzSxGziZ2sPdhpeDh>p3`mK+vUb|wW0V*q>jOA6? z^5a*DBG1Zz#_cd5Y!XoEc~+qhO@e=)wlV7rFz zZmyY>1_b3JF%^&_%mXhf_IAebh9BK_Q$VnYp(C>)X$|2}sqd81d#JUp=+$=|skeW0 zz+8-4y1}ss&3XdFPt1#lOxSN;!#hSN_8+KUsPXHT-z&ypc_uv6pB?A+;>g(wxH4n6unZU%y!bJ0`UV9BZR05!FDxN^?J7oc6L1)O(#QXS zu)GNJ?U&GVrN`hRRL*0^I-Tb?;0>u`8ck||;9|9qva3Hjq0%#Pm$lC>{b$9B+8{Ui z6_tPRK+%e`0(9P9>K?xfXfqBNl?1?p3JtN7NCrrNGnmNI(g%7cNel$uIJor*&Oo=% z<(v$M)2G$IH&RD!-AQ~Ge&&&eu{Uxf4DJ!tz>4ZiGUg9>5V|E6dlUy>a`O>^=R+iU z$Faz~nn!%-K?gR5EA;s|93G>C%ewEj*b1B_-4Bc(njOA|@O5i5q`FB}X<+&&Ei&+< zesjKkXI4S(cJyZjxjIM|Zt(`{3`;_4c`paUTJ&9n!5p|A(6G9*NYKKEY#DSD#hw2SlLp5NBTb!Mtt+S>=AwD&6tD%=PU_knXk zXHPfuEy&xq?$LP%qmz!^LU^e$!bE zKyx}mB(|ldm%(U4+%1Sq6VL?%F;*M5!XgN3Mc}K&=)Gjf&K3C012WiYP0cS;a1)Um zEdm~In77j7%w}2H-0FeP(;}AxroHU@FUj<0SGoBPA zw;yK_c?mlnL2rTI9>^os94ZjH%N{^^nuB*~Lcckl4)QL!fJR}P>ck(02?OgFX9_3S z!iLi`Ss$;y7?9d65P2GwKk5nbTG#&=z&Sk(0n<-(^%Y1hO?JcV@B!2zV>lsFWbpYa z=4Bi*E8$~7DhH6&kvG+#pF`MeW~l|A@~=fUr6(vzbUA?A8UYc&>881Mq?rv(ddUR) z5`$9ncQ9sMhEYdnYM?V_WZPs3?omLlDEY?0$<3m)piUe#>&YGl4O?S%sR~FA zay+&EB|8<+7Hjj)AtYZ90#$bUh#gvm4uH`Udkohx`}+B94{0ON5K9bODl|V_Otk#y zn>b|%Bemk6tf{2;)~i(jl{Dr-ACh?7wtk0fL_itLG4LTffXCc{$dOd43ne5uYxjb&+?x_`$W813^pvO9DPb%ccBu#W}M( z>V1FxugUtqr2inrap76p3E z&tDh1Eawn=?!=ym2(c@Hfq-_!3G@IfUWwc}x^NGjpwyX9yTBdxDPA%9g@Jwn&J89f zA4n3AJa{xG-emCOSW8x$v5)8esdfeI`oo1{2ase>!fN#R)AzOmP9o}H=i_iuEGQ0Rhl*$7$Fg9PL#<2+0|r8gJYV!ns9ntXCF8e(UWmZOh$WT z}v#oi6UxV9R>g z5Qz*2Z%F@(e-~`~b2H=HQa(O^bW4{63`>qE{O4TX*9)(GW=;onLp%rGg`!=rM>Zh{|E1UA8 zSbB`PQO{BoQpb!a(CXj4el6(7ExmOEb#5bCmfy z7c&^6`tA~d_Xal65XeA~rFkppb5|F`-hROK!SVmbnCmGufaB3pC=P3s&Z()d-yd0{ z9bNVUJbWeIJ7z_y`*}`I$*8wjLdsNtu}MrG&!Dyfm%@&e3JMEvh`o_>?mTex@ZI9@ z9l;vMyok0Ta4BoZ{ju*w12iD8>@B_p;&TqlFPRqcYBb;qA~P_qw!n#i%ySW3>IDYw zJfdQtCI*Y_k5Q)^PYMXr^{fYYv$zLkvBtizxcs$=1>#v={B$SQuBADh95@CD@?0MNup7-$U8s&K>o&UU{n+i z3NI^eN1rq2JmOlxRbPYMcSdzl0!0iHgoKbtSq~9~>7r}fyxT>E>V%GeB1Ix%<(#{v z-&`}9f41KciDmCmaW)U1-D?5Xn!(Up+XFQ_WNG&+oWXzs`JjLs2$ELp<}&KUj4yazdH1_ANOY;HU2zR z7HA=siA^N_?&`PZ?0CJlZ*T9bM@jBG$9*0f1L3;*r31w6;2_fiES%i3$j#N2FUE8m zn(5>gJ@PCf{5&%mqie886{%>L~UWLSa}W`p6n>g4-$S%?l;_^Yvn%(@AD(=4*gk(*S6uM2NkL@B$=ezV)PY9LIcAb!kH^+%p2N za_`;S>$0Nz+jf=EH88$%QsK~@)8JY;)G--2DaM}NJHhVhR=fj(;J)Rgrt%& zng-ZnNwNWc)(?@j$hWBzbuYS%w3!^wISk2f+125WA8$Q=1TY6 zR2dQz5m9vk_mesF=nT}OKIJrT@$f}1{{V8A;_30t7=uFoun^0=_2cQWKi+?RTz!@A zlg>_I)uWx6c%?R-83tOIdG>+Yx87`E#@;a86S{}mTT=Hj2%$mhBsb0i4K&FMVJ^ZZ z5Lvxi^L_DG+;m0`HYbG~3zL~@)MsKb<~eTFs4~YbKLF~(S`%qP#6>rA#KrGAw`OAw zroNI^BxaH6?46&K6TWl7vwbgVRta^&RDbsR{QrlpF9FB0ZM(e5jg69sE?PLO&HuWQ3}h*rpJQB2W7UR&Ftcm4OH3vc^=^fe4bThR zc83ANVU?|m!q{KmbohH3jP>vU9c#pc2Coa3F>8>1Bo%|W(3Y#?3Uny$tbupm)s~On zU~ZIg6mFe%cOT+JMflI1@Jgj!h>&ZT<4ux-K6yjq zzuwz5#rjUjdCU1AHko2CQAG8?9aJYJ7cbEAFJuS`^T0AtuetblFLn^Y@k8}wxOH`z z+w=o!qGp%6;sH4T6wFvPg^fQ*#)o4=SZJmee3b|vF#QrWw_jy*0Y`G~t?KMm_yR=NIkpgHi3!(d)|`iBGSW~=PR;>LBW5te zc%$TGenjS=&_W25j(MCP$XcWFO#Q=t+O!5dEjiQh2eQ&c;hV`Wpq!952vt?MgN1sR z0Cy8S4orj3a9zPALQN(kyp!P&iXB1J4`J_CU8qc44+xI%Tfi2kWhvv$cR~6Pla&>Q zgdcOXLjEuP;g|}-MI*iK2gnF>t7H_w48id#NIZfL&*2IFs|ccwbBIkISdS=r;~Q4x z(hXX+c|24xR`i7FHsBrU8FHr*K{eSDsn#qIX>;r;F>nLSe?)G z_w|bqc{n^FC65r717H~#{BKHm0%V*E(V+pn2d4kl5AWi78SQGWld|9UZUy)U|KG5> z^LgL)_(BXvv#1#uJGz9ZT3P&7Q0Ufh06yTj`ww23+0LDpE}Xzs+@$%bxd~6C0(^a9 zO`%{!7>kD38AL=84Df~FvvhhS62o_^$xoz%uTHC5{lxw8`eLrxG!lnR4a0!k23^VO#^`BHO|Cam)YzivV3C!DiZr?143;#NP&@_o8 zWLN%Nz{kL8Yq)VE?&Y%>8dV>MhcG4c!;7}5P?1H!7}O-_i10bhaOOtf2%85m3=(FY z)D!2Su@QAx=th>-YfK9fm`4J>LAT4^JV+e{^-%S$p0Me<`;a2|b$dCvVu}d+K7wpEb_T znW>l6_d->VZ7NmS@+V^9MPCZ83z4x@Y{v}0-d?Q9dHL(*i9*2zcIUk74?Xjoa8+9B zYOa`L+l;T%HVqx|%U9WXu+=4oM{K&P{3AQFCL3OL&-z#g$wF1D4g2;z{dj%x;0*WL z(AqUMPh*~CZ^-I?7P{WxlfKRfT>CL|kw(Y`hRGwNb-TSi3I~w}0-zpIa|^{GBoIy) zzDr6e!`*IarUnhpZN?!voeW%#xV5dHGEbaH0;HR~U4V(VksC8G8Xjn$3J{ndRW^-R z`?t@R^o?2mH9l^*AYY$v4d_K?O)~Q~pWd8j;NEOE^UU_qPoL~vT?ItqNcmPcJ>6*m zZ_OO0La}HZ;m7BwmozSy$I$S53|kfYguT3KOC72WElZ?(=kmBIJUuhPzp{%9f??Y4 zgW1!Rgd+;J;BcSAc}U3jS?}XR=3>lPKzB=v0Km}H*jAf*1PgKp0qdAYG<*%lvI!uS z^rD3#4IWeltJS+n4jasFXmFw zGK`^(RNdCZ3^CQMSRHiz`gO9D_rC*nmdW0|1~kvF{C%`MZ&!?oJvXrXrL+2NdXeMC zKEIW~kH!a77e3`E2MLfdG=XpsSU$R@G1%#_wp?sD=o{$s4Y3%=XP>cT;olctt!O70=k1b}QB}%(>9vXb z5tuJv8B9isw9XH;Ebt9JSrL9f;YB=|Qkc{!J_te>VQOA%gP^|Ro#>R|vM5(~(bI)s zTzb!$*I^d@Yj%B8w1(IeyV9d)H=wMA!bwR`v~a4bszxZ}(LB4AY`=^Hh*oN0 z_tviBl*b3|Q*gL`J=ScT%Vur&?Me+yD6}?qsou<4Bmvr**y`;?TkQUrhDE=aj}b^_ zXn2FznJxw2v})14ui%7XRl3silHkyyq9UK4=>MHGR~)b2^G&4gTh7(KVQ_xM z7gb>EszYxmD4=F|lq2t-EM&$zp0+&>&28c9HU11~M@zLquU@Hk#MztBq>*V6X^joS z#u#+hAt50>&{FhEXZi+Vr3S%qSasC|SPvypB;XmM z-;R~4oQNNRFKDqe;IE#60ds7b`ICeFy9%nR0xBvh;B-bK>jC#HFxr$cvDZ&#rr}=6 zcR85CrQAQ28m}WXaboKwJzGXwIBI-63^eQp483GUEC*IzY~il|8RP4Qy<_f0J1R8P z`+f@6KIsyK?(fl)Crb-8P-zhzv{1u$9S8&{ZP#a;gmN|1JVmi9CM6XEen$H|@yJSt zuo0t?6V?(=e?$c4w4p^!4Gjmp?Xz0q$$DRZIQ-!h)KzLw^$YP)b{B@S(^LE6Su-82 z4|By^aG$l|y^NC9XzkjW?wV80&CNaCrVuG);TBm0O7d~_KU}uJE{G2odsXz}WP6eqlW`CRS;Gbo&ZaedY6MzAaUqnnahfDE<+Cw4=~jdZ0iIkD>=;Lsb>V*D@l-m zw$?ZEc|Tf2{4bTCs135Lm$Z>5G66+BG{44c>49f6KtLSk)PzwOk)!l$0Otxl2X9>C zekA_KB7OF3{`N|k?iet4?GBSz0I#*h=o1op>qk_cYo(`8SA;2$aAPh$9fn|PJ2I|g zG(gZ-<@k9OZ@k&^dy>2+Bj`0%A4?WmxQ2X11*9;4zSB^JkG1u(Wl{-kMmNfvPT?~v z6;jy@OvCDrH}vy3xRh06<*MvW=yy z@(TTUR3(K9HF%>~eIhN6O$dE0m&#R+_{MlfOc34IaV_NbGxVkL9w!1fteVkq#X-qb zYlPiW5ej$K?E6v}h~TDFz5slYeZTMk#@Dp^1_zp4Sb)g1dS3JVaud7~f|$wj1I4G% zCD$dY8X9Jxw0=KTN}wF17fQExVvr~bV_5#laIhBY-}N?=}Pr9cPBS?u1bd zh_l(=i3Bjf(~2dw2*Ac^&h-xOYr(gvRwg#D6#z!MT!D5HA3RrkD%SE zoGt5o*uiF1qL#P)7l8Ts1q9zPh<1!$PG&c{_IzCT2I+8g$^A6)vbC1a#^8%WWJ|+Tz3hRzI z1QmWXk8$rAxnRp<3PPQ#oWz#ucGmg+yvaYU`A#W+OvME;TDd@UYEtXl9#VnfOzb$HG# zK@%o|1Ip6GP_!P@rb51q&vfwI#DS@{mE%LKcGrW#HG;qX+MO_ipuDt^xF_1V7SBVVk&J zlEt^(GHKB*S~d5`XLDlXz~VrDF#mV(2~{`3fM9VM-+lE)ML+2%7emRBk?NB^n5wJNd%u191Z z4?S8RFbG#PeF@h4tE=_l^+@h?bb3&isHJW(P4kqF{k(00*ef4}r-zb~ZXnVu<-G%F z*5Pba{EDPCM@3R}TM-N(ig8D%yO@COYOAqwdIr z2G@A)+7s1vuVG;V&zFRDYjBC+@?yQg5(y2!Nam%>+UB~iKOnTT^$q6CnT(4iDd?eM zecQu}EB;D9kHphNFyebKPaF6xA=1jv0C$%f1Lm>M2gIXMwb=}I2b_IBhh z2}^K#mJjYIX3^Qf-x~o;n2&4ESzxIU_Veo`xCTJ!@Yg5{N?1Tq^bZk{P<+oKuMHqU zFHsR?l$wSFyE%yoC`-qJ#=Gva|9(6`K6D5v?`D8o z&{R-ZMw=IbVw&QEV}Nkq{sXj&7=DGEKM&4$sI$&65!fgi?}0TDyR4Q7Hx?gkIgJZO z&Ctk)XN!4xdC8^;$AEA6TDUS6Qv(?@fm48$SuHx-39B{N!hDJu{__<@H7ca?QI5SA z%+JZ;C-t_K-7NDkEFdxYU80TKX1QjKod3iDL7nLDSUCx^KQorqT;V8G3bf=rIH9x1 zkGMIA2`9l;X6+3#Hs*?qj0D=wk4lO9;-Y*%CsI7J((>^h&B$2;-5z^(1Ie&3eIXJO z0n^N%qxofJiRkjJ+QSb1jlVB^N!Wed_T9<1%Y5S46Lpy1)oJI9h;rtjwhP)Z5{^PY zZt>h4@LXZQ$R96PC%Kd&yL5rxq>^gU0$IBapqr8V`)KC}I1fRb)hrjX7vvy>dU)hV z3TmKqb<5U}z$9iE`akZTxP+hBvVsSL7R!e38}HyKYJ~>4-j)C06PAtu{1D+KO*H`s zqOp!=Ozr(JCwn?ENc6Y9&0~Mju9dE?s&_-%DA$3*6oYthPJ}9%bQ&ff3l!8Vy`D7SBMl*zad(YAawA z@dkPcryHo~h`AbEQ)*C#vK~wp?nW)tQVRK{mC(m1vQ%TeuwO*g`Ai>}7vzD<7)~u@ zpE}#o*|+K51fC;O*S4{rICm8u%&_$GqRonvckSARQSTYE6OO}A(SuH>UJTn`6Q1V_ zVtgg;wGO%`qDz&;wf!0Yu7iE<)+(LYWNU#hy*@l&df9jI%izOG@Jl7@O4ZDj z*e8#Qc^&Fn_OZy5v~>O;a$Kr)K&%(>sBmnE=#HA-+{lOl1g;po2J$|5`zT(Tybylt zAo`??f=2Hfw8*9&xsK%Hcx$Yx+0Sc)g@p-S3)Gc`6y7f7Rog*~?r1Te&pwo1&>>en zQuA$RVQvIfi9(7v^V=tF?YR7{&JO6!Q{`|eFFRq zSbE6c*GrcP71RM${2b0oWi9NAG+fUaZ5)7JiXrMhQ*##hs)l&$HMVYsbYuA z8lPG_<@E^*ncE2P{TxV8sjdSDpB5tY)d{Kuy&oEa5o!_;rmV3ZiYW+bB=bfs*0RLg zZ3J=DrM(8*N{71H#zIS1=meBQ*K0dxGYHT3MM; zjnQA5&e(3YC8(q_N(U1YdF(20$xNG;Y8(<|?p(?Nw>(6DP0XG0;g2%0#(-r4LEWnO zkaR`Z5lH7mK-Qkpd=%+0O>0~78_1@ar?PaQm}SRkU-LxmN2l-u?%$6dwHVujsOFOA z3fqh{vnBXTNHgB|SB~pD1+42cnYG6z7{gfGBl8m6|3nK)6ihRY@^vG$pGap+{Or#B zc$0A5ZliGBw_p`zR3XxJu}<5$(b0i%%dD~yd|%Lr>2#w1 zb}YS#>N}V_M8k56FV@^P%V|GHM^#cyVJN+YXW8!k z;CU1JugK%jd~&Aa*)=mzvhp=*(Y7K0ZY>|9`8R(4@zTF zQhMhHwe6H;C-mV5l5g*~F?68u!toouB?YEgTYv3nl`{ZxK-zDh4J{X;)04o8mt>n0 zK4^W$@LXH21ptab!(m|>Z6okru}x&K(K-nN5df6yz*fXe@?mj2)-PaHch%LaS?s5? zHH+6`H9$+<4s-kiI#&fbII}OI@J&q}C^ZI_ji)d{E{RqG@kChtIpQ^@riwB|2uJ_$ zkt6I*_v{fufAGGP&pZS118kEfIL34!qX)NRt+p~q)fA->P%;nmdVra|V9Qh0d+(BG z&0`(wCr1NUY|=q-hW-VaUES$6eiJQJ0abygr}mGntt|~AS&v*!{d5*XABq;W5jCC_ z`(od3OTY%beSO?hr)CNLTE9aPfDuaIS!%P?&r6@au#H|mV6t^U)*LQm=5O>X)ir%y zV(T;e_J0sQM01gd+CW-|<&b!sYqLAmB{^AsCogaBOUMk!6e(j(NhrQ5kkzBQ()h)3fHUw&O%H`)3t5qE zc4Tp}YH&_w<`j?;n{FR__v)2J=V3bFfOza%N=^K|*RL;1ZsnjbYrnSKADF`onio>@ zXU6~FCk85Tz#~2!odIm|@bvNFjWCSQfsO*T4KZ0UGQ|~{qK2qNKNl`&Dryp4$9{hM z7(^^jF88BhlYej;-YYi)jQoU>!4)D8gZtlwh@16&3d`#Pu}Gzq(%SAD3fK*+tzW#5`7Q85~|{40ZAr4dz>8DHG&*|U+=-FkUfxX~ONfwc+( z&4`Zy8lWixpvWxxK8*|Y#l$bYrOQ!S;s1!CfzQj`QNDc_H!)>VIHrFeqK}S-C{49s zH)ZQQ@JbM%s42lX`PXqc2tM*>>f`x$M(ej*Y`e}1+D;~|Z zZJ!)|JqL$_Gq$VYb%jPE+O;wd$0>t}R=060D*}5>w1jyOL%=F^xl&GF81wD^8P_W{ z$kYN+`1R5 z`9K9~`3-GN!KxazakGphCtY++LdogV2T@>0whwqCn4?8=x)8dNU@a7s{HUN(S6?j` zDd@TH3=Jae9X)X)*e5n%5#>}Mc4F3h>a>K9EC{d({vhPo(NOyj$n9_mXgafjCiytU zL9V!;g?nd9-e)N(@se6?+c|4@PMqO9ZSB9-#i*xk9(pRj1jL5XB_HA)UL}(*=>zzF zQ&`n4JesaE9@7yHnWhz7n3Si~0c}R#Z$+Mu4mz0!@k5|TU#Fo*oYN~&QQg(ffBY=uMt_f|l;dM`?zoCh3|<+FLBhyzk(0s;r%vIYTrf2g(d5g)bXNH!@!H zf+|nt<}%1jv=wWqnFtrbuz)Zso(eePDJrCyHSn})v|}v8hs~oMckibBX)fXFic6b$ zWT-J=shJsXh`t<#XT~fu$&ZUK;mKm`h&yS4=Og8>X}%1t2r@Yw*|#QC_Y96-oAL_f z1o`b!Rgt^h*dlb|l<4qlI3lG1=~jpz%b}!$^jtV%-|UYsYAu0nmpZ-zcMi*155j(6 z^corQ6AunRgF_bk#XqIsy!@wJOxwPRd*0h;7WQ(yDau@Xw{rvUo3Lu-{CUU{dl;th%ei`)m?MAQ~Q5Bw@Z} z*M9psu6g~-5lmrdpADnTe}9>+trpm>=!P(Duugf!mVz&PPoOSJXE>?V;iV|+fip+# zm4d<;Z*K?aEFEV&7N@Mc?;DXM67S$BNOxXCqY2tJV7ni*S1yh}FYXeU)3bqQOQ1}a zXq^Z{5)1=8UO*J2)3*szY%}>6prVc2O4AbU&5P%-79mz?X=}5%xzH)= zE?(?%`6=pS$RMYov}4^ledv$~CQ9@5Zg~fRTtJ2@7;pF=u*=EM5jf?n@$Od%TjJFb z)@{22EuQ4lx~9h-pJW?b2BCnR3#%Gh6Nq$P4-yV8!!Ypwlo#hU5^?9e(CE8Dxr zI>mdKTfL5}L)5G6?s7GHxdM2m zK%RK<%-yR)DsHyuf>F{Fe3_sLXuI>Q($qFu0jC0U3I~u%K1nI5j32XMDaDu74!7HY z82_rSN%(-!tSTBB&cuKh^Ews$TKF+=@CJDpWbKJq0WxpPQ)y5a%`p!5U!`F~PNNNQ z-$#v&L#r>hg54*KKSg*P3<|K$zyLig5U=-T0%9{~BBjz8lstUzzOqwg{i^Adp66#5 zBk@Y|d#;~XtbXo$``;beTten;l=r}bxekF4nHWPk+z8_&jIH{WvtZeYgh3`?!2S_x z$fMaz;;rPZB4D?a8PEojW<4e6DhNhK)HiCTc}Zg@=l-F48T((fwzP=apTa9yv;Tpr z97-!-vMIJg;p1Z!PmN1SvDoPzQ?DhA*EbQ22M9wCUYp!Eig|D4>9c3omT2Q&BvPx0 zJP%Nkzfo&!Y(x=%FB-EE(mr%*yZDRhT4Ka3^{VyP=eKDszp?V5sr2MnsH9D3N9*xQ zvbjcO7YMgVIEPFG-$nV5nw*>I*z=35}F1 zH$M_Z)q}TtA;8|p1Uz=cUkoRfH{EUK?}t+int#;nQ}xC~en;;n2Jk;y*P0(SvF8Eb z3}7#nII+@6I?CtZf^>; zP(Pw|szq*@=z8YD5;pgd2hk=eJQF5NfF(JD>IKk)O?0Y6b=Qxt?ki1p489E71Dp#E$LQV>$3y57@^B3bW|Fp)sC}o$QDl(8 z9R#~iN1#sLCL z*dj4?s^#;bhWNCNCNg%fFQ~LB-AwjfC9z{;QYr62%ne$&kfXtzA|3b3jES(d3)x7(%3vGy<7-nQ5|Lg~GO z$731uD~MYs&z@Z|YfoD)UN=-FY#_&_ek)ZA3CKC=Z}1dFN)J{Df`H&qdl^BE56}DL zNyP&P4@!CF1)m4{R(We6SY2v{gDhB9#P?vve;+b3J2{9b1(%A6#DzhBpM2uPUSusew+q7b9K9Vv%swenH~4Ou6=QODGiR5x@BL7xQ;Z+7E`MEZ5!xaJ zSg?VkUAp}*QyX9VKeZ4*SRfFz<&Qk^FWht%Qu61|2+$eqdQfTtN*qe?WnYiwz-`UT@ z>=M00miO5g?i{&xy7JgocE*oFl3joX-(xi)2SEdJE?%& z(DAxe*=%kGW`6nTPAYXWP4fJ<3z!X7J0Ak8ssSGsIxuETJwE5yu`K8BJ(Tg^qxs?ub|GqWifSXsjv^K3prjVx6ZV6LF|i#;SVLp+lY zO}EbkgGMbtJcW_7^Xw^+u62K@EIC8hGkhP9N2+7^|=XVGUI7*^)q?VnVAMjX`ARSLndEj&CTO5Nu zSFGxP{rV-^XVSc2owTQ)p8&>OG|#cHu;466%2)B*D9}_|^s>VTfGrkH#NcKEDUeHY z@eSf*i1q@U{1P@xj23Nr^A6H*Zmu#OYud+PNtP@{sgIU# z-n{Vyrx`C8MfQGpM6qlO?btr#91GuDbWdnxfB*iubsSDAg>vncRh%sYpnNc`PS2fj~SPT9%G-JP=!cwEfqF*{xW4AC*r{70d`P&%_he z4INd2u7aBh`T6y<>j0g0<`2sEUQ>L}Qup6bCi?*7&KiPHhfo6VV~CTh$zB|vz<00F zL?I#4fFZ1tPZ%C*R&2MVPz-AU?csozVe8_3fZR~`rQ394%0j=FGzAE_9j$mdCU1O_ z!iaC)^!hUnI9H!A5P6z?Uy<|q2uPq`7ud7OKQPE7kY80d8%O0 zW%r6f)qo%|^YfwwmJ-)NPKA0IPltHV+~xUprPq>rJ}X%{i(m`9!>p2k(BWz}IeUaVLg z_+7*pMNz}gppzv}myL@A5HAx@rINQO2%;%A*^-Sf>xz4+c<}RPNgLl}%L`NL_aq4&S3~syEui=b#RP%c zd`H(KW}sj}OFB7X-&zd7OFR?%O@s2Pz(}C^0=!UR1eV?9ziE?tTCkcV7s#3WF-O8i zhgkHny(mbD0FKKS3hO@}D3miKRLg`M50A`vEH^Nb&Rwt)8Wq%GF-R;JG1@yiI(i4o zGAP5TrA0VGWH+*Z#m+&qjj=_<(8nkLoOOuW%?-lQ$jD1ehFowQH+oL)S>eDPHVeo_ z5h#N>uCLJ<3Ih31Q)nZm0{(O!nlVfq3knNU)*&NmL8nKl;ql|gowd?<&T3JBqS`Km zp-v`}JxH$Fr{hfQEphOCa&&{ZF^8V@5LO#@XS7VEtv>|$_$U#Zh@U^srrTN#q0E{O zSG4ivh-3h_IZw(%-UY{0PYEO!+$j?Iwb*>6om~Wm6->v?pK5dRbQ`sRuP6J@kQAz0 z@e6yVwXj&%w*Pel_azK~tOD+e4=i>(Y%h+d#Q*mx_25}j0fn9!y_^uE%y;Z4E+d*6 z3~@m}GqEpXO93b5tNSRr5D^`PXt6^mx^m?w*w%E$!Hq+!YM8NaWMHT-PpOEzLZyy=2J@5E)oM5m%q~fk})K65$2OBvtT3-yBU)vhiB!BO~1kRXTr1uPgr~Rb+iu z?6BvWP@ZRPEkL1KFC98`=+XDG702|Sk?|=a>nAj_w8;b>zg@gakoy zi9#^DE1N3G{fp1$5Y`nlH8V-RfDc-6M`}z24sjX_-Ti$m0cyYDwVhqCMo8 z-2wu%Jm&Q+%$wqMs7(UhBExB`Bu9k=Fc2K2`~v^M6C)_^6N`bxxL@6Q`lb^Kb+t#e z`wRhy?g!S6fN06yxpH12q)p&cPA5^#^V)rD2j}YP=}GH{XwlAKVCjoiTjdoS04QO` z!WTj@4gQMtHKH4Wm%t!Q@iF9c0mjy191jCJ9;!zKJn+Cgs>NHd)=Xb zveXT$ne3az`WKei%%o1Gpb6F3(*9Ujuw4fjMsw#aPjwF_8OW;v6jpo!A1EibI@(a! zgv$RU4!lF?=qua?pC`c5%eP@n6%`rDt!)L*Y+AP96tV_4yrVjFTXp!)DGY&Y^lO^> z0YN?jl|S@D(+o4>ULw-NUzSNXth!ODLHc6&=OHKO;)*#f%*ju6uiblW^EW)8^41q1 zxOyTR!PZQ5;k?aIV<4aheIND3-i$tgbup+S9EY5J0pDA@Y_VteCmtFP3csWZ5p($dl`){E6=}GU^g!l=x`5){hS3Lx!S2tBb5w_0HQY)fZB}@h#D0C z>2kr}W}?*k7Lmtt>XRwwXv-rr(`rL_5c`9Emz$ewRJ21i74l(Xv&Vyh!jRApV7GWy z+R>wlgvT;}j(>q{baP5%rMz?x|JeJbvJfpAGA~>L|7weVL^q1<`#x$u^hs5c>&P_j z5p*`p*vBIvc*b9RyZ~%yuz(cOo<~}b0!H#CsxiDPm>x`Oh$RyTI%--12ZIt?<$wTI zuAyo90V5`}4?Cc3KofHLq5AA5kHRdd#-KDsY09}UKh~=QJ2RnTs7IFw&WQ<9H@X_k z{oL&3Kw$={ki2#(Ym|h{2Z;N?{Qg-8K`w#AZ^Eq2?Qkq0`{83SwmO8MCWxR03YeML z*TM@IFDhT5O%a~xHP9l?GL-l%@e%4Bgbfmaz~rF;Y)Zdqbs+>5YoDiej~-RE-C2R0 z0YB2#8URJd9x;C|g9Z^AbHmryw-=|>=S6F)b>@0|iki2`)`rB9D4a2X${Y*#PwNHj z*Mwlm3{Tw~@S?`!)lNz&3e)}el3Z7c_TGtiu0iUvVNo)0_2(BSL+~Y zub*PyZ;g+B+;HFWud~3xdJsywD>P%FYmk7*R|WTlBf9}YDiR%m?11uC!4TWe;~&TC zAY-dMY!Mz5-RtW9bKm!YsY=SqDKGICVF3x-=r-{~{wNhe&`Sd9qM)aDQF)<38cE=AXmNDwZ=8$~=-Co?oWU{$ zUD-qN$(Hg*B+*4syY#dD!*b-^kJ>D6zB&; z*sQE0x@Y`>1Lc*vFi*@DNM>R>C zx3s(jsz@rIB07I%R6gfNaTg6D(OU_?7#kf2gV{He4G`id{|)iNGdj{V5tSRzCy?qf z)3+C6EY!wiEr}v__vP2uo0>d!eT_5it z1Om%T6MoflM7LaZ%&}jD2F0Uz@{QL>39RrV2nLYl;T%(pMd(YUoW-!wg4| z=C5jQXZ6o%Eae?e&1vinY%*9AVjop{9{-}U4iL^-4X;F9ay~;(OIwLR1V$YghfXeY z*jIJnq)1<&zwrchr>INH9Bm_W=Sqyu=8(?rILsIZb=KPdDWui{7PZ_wJeVqw&~R>A zFRoqq|5S1b7}R$xuMnn&1ODlA@jQTi=Aj=0(X$cZ@gU|0`Jf82Zk*5$rGCy5TUZGK z%k*uv1c7`e(w9qXZQ5|$AYKTKr9q0ae>DO?M4}r|WTx57tLTF&!4dKh)F@=StRFZc zaPUODkUm3z#f z{xJLkVdbokrB4vDL_*zxH|f&bf4WjJ=g{)NTYV1MZ@dA@M!S-I|r7y5c75Lujcj zd^2EVyY$QkBnj5t)ulFQ0BBz>o}d5A#l@5PQE;aQ1l@aPZf2HTtPr{9E|M;2)*|SpST~U1 zWMD8vKt5Ar)$#1vwJzB^r3aK)iEToA?wgAay?;L`V&4)J78$yI@w${d;Ym4%(sZQ; zSa0{Wy(h-xmqU=bsX*RFOHOF`<9?-vAODqktH1rfnYWA0E~(CCslR>s?j4zgB>$v+ zEjVc6Vju8`o7o#cxMjxnodh+3g#d}Pm45K94TDOE@3CCr-%3a_gR(mbmPt7UN^bo2 zG0xdTV#oV^+@?T&NzfQhipH+1g%cXJXj}`DjzC1M&~Ds141Vns))7Y-)V35U3t&&) z55FK7B$Qu)N;p93^r>Q2!}2nHwVG6i40t1M&SO&nfDM|%j@H+O5(q6;}!wBW&{gZs7 z�c31vi2fE`*k(;^bnK?lZ{XeXgs^8o%EVwUk3f1!6#Y>X%VV`8D2d&YJ4eW8UTR ztdR6INeizcu4RD?RORWeuV=vg*#}W|CZuZ0fS-X7$CHZ|ZUHeGT{D|OFure*W32eZ z3c8-j%hJ*DRwFZjR*@w=>zeGgxK5^=Pc}1VSW$@{V_BCc~z;9hyAKWymu#nA_}Lc zM+%>anu{J^Y$OIB592ds@gpa0yi`z8iM4Hmt@BLyNQ*C?5K*ZhyF+8{Tru*!hOEoS z0lL|+#jV-pHFxGAJGtpX}bf`%PzIB2>F}rI9K>0wX(W z(9wYr=Y34#Ulcdx-7aXPhV=o`!cF9j$E85S8V&C3NzAIW%aSmYTacHR$Bez`0$P+XyJt;x$Xwi8v!nED4f1*q!%bd%O&7luCEDb@Oc5S5>oHd;cDQLCxMRom>;cLbV6+~Ca`~z+t1oa zJ>FmKsO46YY(9ky_Iy^m=Y@dM0D3511qiV9n917|2tBC|)jGNrfRgKk65u@eeeAQ5=_01j^;&+ zMoT(<<@Ny?O$kSwN$5NhHa5I}bD8Oz!A@|~8pgkWn!E3J7}cFf-L#!O<;W46=Q}V3 zrhL5oxPAhV58CApnsu;*ZwWUlK!vE+4fgYtjK}o7@?zP>F6qvPsL~BU^atAFSpJFE z#G(GY9$YGdXQ^KN*Y-oZ=Lgh=fSk*Q>e}G*K*TE)&5+W(gk?h$y8D7vH^9Y=!PVgE z`DTa>${P1a(f~@o z=m@G9rnwh3%eZU0Xnye22NTCAKP*Ms$c=qW)Q^xJ!nsspCRlGxqiUbfyP~ zg#GhYHs}s;5{!XP7uX08Hd^Zd1F{U8??6FlS57>0{m~<7e5nkZZpjfZww<^?-X+xnQ9{UnlRUW$pCHlQH7LBK2k)F$tTF5oj)E}LaDFYWBw+SGvF zukz~A??lfWh0uKR;JH9-I&vX_yxmZYDF9=ZNswmNL>-;;+DQwww8&H!Dpg)4@I-fI=7F=jnY+;!mTQPq#Ckq>fd%txiM!tIea_c9}BJJ62a zz|-SmfLux&goTuNErf+@5S)fn>IrD8cn645+&Ff%JCUMq&;YeJ(9_?)#Higjsz(OLe{1%glfy!fai2Legq)f7O3)MpyXOR4 zOko@SeaBxF&%D%Vv1aF@Ch<>#6fhmLyfv>13LrhfxHAc_hjuds zUrNsRgN&r*l98;d94Ih=f+VwhlFf1yG#L9+pDn1sz(^@!tQ*iGUZm3$m0Bn}P>ZV! zJU~Z_z~|_1oPj&nooV@dORrv6$t~b|118C(iC)qbQ}detI`Xe6sD1zDjcadcWLHOI zMJ67AHwu>2>8H(F^vvg)F92|nw0Yv3o2A0gGAmt~R+m&>I%ww=8LG+-$2kcF&NzM#BHxTx(a zMMe9+-ZIMCi<~j5#3CUSd-Y9Cya3GMRG*^jka$HKE6!+okB58ioG3NO?j9aq&`ToH z$^+T~wH(W0PsML@2O|*@LF=L25J|V}1c}@p(WPWfTq#yyNRXc)GI%4vm%y=MFHk=e z7B4<3Y!deChZN2i$(Mlpb$wmD%6X$ebrq2JgR^TVL#9rXG3vl9zqW$xffHQ; z;+p>6aaS{=Zsb{ba<0hC%D+VG75=)Jv)U2arLg{I;5d0%5)B*>%K&FDMs#%gvA^1@ zAzBc$1v50_1se^`L&b#WRca*S7UGShtLJk5E-SfY^~WyNAaU62vVKj_XdmL^4M5sl z!?m;Ug&h*4AHoD(V{MHrl*k1`C~|u7C(Z_Ub4)^ViAwikigvG6=Ar)8w;7ys4u-Hw&}=i;+&nQZlfM|F2u;aB3gblrlf~`rH)!cf!wUt0ox{qV_UD5AWsUV^)&~c$x&5h@-@5?}dgKS?6fW;6hwD zyk5XrF#2&$Mejkw3ubK5P4&?Vl<)*pWxB?=j{jthe*;v6PLi%8%tQRm%qHdGv%^yj z^K5?n3)tVOehNXxOrcDgi$2Hu;WvzODrrrw;AyAjwB-zCN%Cu2K?*c3v8(NMpl%g5 zwn9-!OfpEwurR~i_v8kF9~(5~%MkLTaY zRbABZJEAJQxX)w+-zm{!EqFQ!x~963xutO6NkoIrgw+=t9Z(HG-Auau?DW&E^w#mc zEP&E6o~XuIB`M9ycl+;Iv9(d5jEa|0T7P=D{s`M@*XBf5v64 z{Hw#56|j}MdigSuNgyY6Ni=qKvrTs4ZPY4=vbie^OHk`38AQbpxE&yM7rCBv>Rs3?N$>-GqHa<%RdG9nH`~v+qyEaI7)kJh4gd60!|S z{Q|ilXIVA%?dJXiq9V8EV@DWLln2zn#FDO`J*Q+H!y?D z!zsWPgK2pBBM@<=wwbu5>%d_O{b$S9hJF~rv5Sx|kOxLSOhq@E3 zqnPnq2p?hNqM*PTODWhFpxhryTAP(+lAe&uhH+A^_8CUR@3Qce81;$>Rv&vLeZ zcq)UU3WSsZ^itX-uaW&7a}*Sz@5YVbL8XBHi;&QKKT{}r2^_#@Z@uHV=rK!by((J5 zf1x=CGu`UI(1L;j;xB-)%i`ICQEP!&4zU6dnHtbqfCtFF+F7FwM2p0M zvLbP|b&`J;2s{tRX}tkASY=M^XUsrcb`P&P1`xL%@02z@18h4uRs&~hi&r}kvTS`! z_M*<#KD*KxQ<4l*&&4QxVBQCR5{D0^eaL(u ziZ1D@ZA3;G!Cs{OH(n-;1)+$c!f&&t&2qnnh0fE!@Fidc3|gcXnqdQsH4j!lRaofN zmaYan3#rgKm}YT2k^&`Hz;TT>Pv}j}^(1ao#GjkXKw?4Nk-B>Z{<8rja-!0qeFA@= zq+ItT&ZXbOGcz#p;*0g7T#%ihv-9KgWtR|D)l(DQ?*nTJF0ss*6{^;0K6Y&OQkXTM zu`d^!!kHL-b^vac@rb!L6Psfu$d}6gf)m7X4DcZqM1E@PUu=^yY#%=^1t2I_ZFxdo?@s4&A%pFRbRj|FwE zP@M$)H{}wxL!Kj=W_J@fRpZkMu2ir?= zvZd7YxYF`;{f^go7>EP4fc^(K`;n!F(Qc3aGYwlLQ9QOyIvcQMiwLR)l4YQD11W{K zy%L+g65A4W7!Zi*$zd8PqHT;ia}sSM0MCRhrjQB=2gDRML163Y)2Eef_;+9tAtPTF zWaD1QA5bway?6-2 zed4`T^S}cQQsn9p>dt=Jdfd}cK|TF>&%uKS%Vdl6F;0N!!e=eIp8%DxNA&^4NFubZ#L=I*6u;PuGD!RPjz?3a8zsWzJ+WLm*+V&mW{*$DBNG5`2Sc zfvY(Wot7O?H>wiB4!;f;v!!MMlokL|5lsN&oYg*I&i{Lys5*r6EKB&ewDbMi4oOa` zyz7^nTDF({mq7(gl?k9GsNvDIk!c?W3Tpq1!w|v?f=Q|yN;Y81tu6RoZ3Sk$Es4Cs@b*){R4to<6r5ogJU*9bNEgGHhoww9k+Qso&aL;?kuoI zC_qtomUXL8RSfB3mbwjq$0Vem)`H^zDpQw^BMHiDdA{H{VMvekFYT4#9?=YpP zL$c=@{`ncuM6%U|g3Vy?gkg|O6!K0(HMxM5y1rsCD|z_v1`1(vL9b|AVQsx1@cqc* zOrZI+>bcEDVILTVR-YRI_L16Ld(#G_Bml80X`z7j*y^PRl9DVj(!ufI&GaRnBKi~S zWx3=3WyS>cuQ6zUG3zAS4h;Xal@oZ*HK^?@3Vj|38Z;+5f$jb~f_O8uacWdsLRhqvhx-dI@-O%}9YR1v|LLUi$1%umMTF9A zpb;-f@PxY#XtVu-rW2{c3yxu7lOmGw;0|aGaxu&IdtQCnX0n}ACPF{?e}#ZQ1VMl> zKoRYDcGWjhx}tj-2H#@B>ZSJcfwZ;sSRs^G0Biv{&9UD@d3=WT^5x|0DYf3t3^8W{ zlOk2vX%S+s)~Lm<&6i}*W?R3Qzi5#Rq{$>vF#GG50Y#L$xX@uQY0Qb14ebV`+jfervS{!FISJY4AUON@_z;;31qjew8i)W!rt?lxpw`v|A)SH)=cJ2Rc`@?47GZ$7 zaO-h1K#hi8ngZY2DmAt|ty0Ll*ap@U8*T)1b;HKawRnqoMc>OBeDj8AL7QQhN88pQ zA@r%O{;XdbZ4?Ic81Qx!S8^qT=?SIf>ijKE1$bN}P5Hha6fnZj6%)fs8^9Q%YCY`x zXiWN@f#CZMAEm0>`$twhbG^2EeuA(QoL?alwec0RK(_@cJyeHF(9)rrVFf!*<^8zN z?e|K4+z6x`J*LO73~kIDon>|~eg}rFevEH{g2H}*NT&FL74d=rQ_BBDY)JDDAMAXd zoeJbvu0cF#S{ZA~7^6Z3#t}EZra&1Y2E2Uq5H_QCJ74 zPE^lFH%{mh?h-TXQZh?fIS*r~&IK#a2(0NxaX=z5@H@zBk(jHEy~RVNQf_wHx3`Q; zE-b2q>7BMxh|oZ((!Kxr^h z3H$L`(CB~h^&W6N_y7L?CrL?BDLb^NtPnDyQc6~(BwIvWMrBhZEg@3rLTFDjtL!3D zW;T(Kz4!LJKlA;abN=W5zx~hcoa=nA>wBrs=ly<-=ku|iXTOXBRuP!6ene{bMc2*p zgbu1n3XgI_jFi30bl&{=jW+sj?l~vJy>x^zFgQiND(nQpH;QR-l+sefg+6Y))u~~h zRJ0I=4aezKy?QmI^eKhO#&cJ7k+%(6<75}3Sv0L)lw$|83kX!sekQ!ec;_JQ7cN?? z2TCUzDD8SKCmK&ely=~6BPaK<`Il2cCS;SOuHzH*Xlhg=*#E6xUMPsdj*Wq_>51N_e{OjWik;x#l=;~;tE0Ja7&DT?jO?G^S`NP zkFy{9a5-1_j#?6@C)H-jM$Qa?$yh=!ha(ODvKSAMi`}S=x-EfJb zU6`9Txvy69Up86TrzQ6aDIAYe>F4PTgNaf>$I`6F%0byJLiEx_vSmLa4ia(1qW$mK zu;Q8$$~#o?cpYJaLSK7&+{z$>J~Hh6-!mS>ge-5-B?e*rD zJ($aFKR2yZt)J0*r!do>eO*31*-HWwiGYRNAOmw6ZSaxkJc=#~Lsgu|0yjv&8*D}) z<{!oQx3M$`{}l%mH~3}UdR=SBj3xXd^+db)`l!zq)lM131&YiHP;SigdH*jtim21G zZx{`uju`O$A_^C=a1DMAsiKJo(kwQw`uu;&QZNH$1xM&|`UG4&>(z>9^Ec?zwUhG+ z91}u#&x@YC1)69n{%N5$j#+yI>z1ze(#*6pfx6vxhzyZpe7}2HTHH{OMwwJ4Z5bM} zGhn4sG6(+t*%WNjqH|}8Ax#XJD)bTYTlysxXd;7bNA_)_mUlKBU-RhsME6@nQUC8m{{bYO^!T&l{ z?Nm6>*@;z%L3DP^sMs?pqU1wgx|U~6hb6~_6cqGSEZrQ08w85HG|`ut&IWWcC|eeJ zH4VMkaBygj`PGQrKD-9T+9o)A!BW!y$2xnJhVOoQAn#(Lv0CPNSJlGV68VpIhQ{2e zLx&;8M?P|wI;XX1*Pw7wNEdK^KWve3V5O(#M?FVEhOn-?>_3XPK}YKoIdg7Pw|b=S zswe>SoU4nQ0@E1ht{#7MahT9}AM0Ixwq!I_S50mvgC!QV1ub1=pQ^LeQ|afs52!yQ z$2w9|u~Y3Yct(88`cF23&!`Aa-%$>GUG=ys+qs512X{*OEkhrExHnDa{n?Mble*nC zXzdC+*Uq6^exs@I%+aW_Y{RMKabRFm8B;dp;?^~yi{`t1CBpQqdQ)4~-WUQz??}@? z!ci33rjo%Ehr%@)5eg*3%eaH9o<5bqY*P1~_|oDSZSZNjifDGI^S;Hl@e8TezO_yK zGx|0ay(HK_+IGmo`;?l9^C&(I7j0ypTfhrc9A3}>PV@{7b+*L{w`sN8Ijb0UuF^|~ zJ9ZFI4PqwIdPyS@oA3xgh+>FZ0b(}btfp#_yH5PUXfqPDres}-a+}Dd5Ka>;a~)9+ zn8rAcr5Y41k>11?P|fMa7x* zF&nDAjpkH*ynl%c&e#qgMph+0Tt9fa`UI~LkKVX@^>ev}P03+|T11}>r{(BN=dwNf_W-3sZg+XG&hb`7cLq zyw(kOhT`Y^=KA_;*y7`oZ z{I@0~wADKtw^9r(T)R>V5EZN-nGQTGdH3#h8d^XDG9*ZpSN|REkjyQoeqgKJ) zF4M)rs87$jZ!708>#P=`NpM_x({_$|dVNRprn=mW9%0Wk*o?d3((;Zo|6|W{ET8oR zxv67(%6eEl()n;sH(cf9E+oA-aPTR_c$%?pF?9Q`b&J#MB9vGpnQ$D_E&43fV&ych zd%hCy5@FcGA9O(!j^wxG*rCW>=wmyq-}k zLpXNhQQd0>=H^lR*8cxUWpd|!KTKt$`1-rWJ@yQu4Ue>e<|`x?eeSamgww0+&ue9{ zMukeVpJ~h*;9IIF9$X|(l%i?4sk#WP{%`O#szq-F`JFR@gDD~);i@5WAMkVwq@+k}9 zRx*F|W>@_O8ux!hmH*Ad%Cnzd-{16@`ZaEQUcvO{Q^x&E0U=`w-OO^iwxkquFLi4p z>EJp_Ptn4}H=0^+IP=R@d<{8EoArQ3!1((L zIpHPFQKr5i;11zm5;2YUHCoaL`owh$!hpsG4;hla#K5^W{~lYUN#R8rE>&HVXwVdq05gBZOaQWk5=Str8#}k0^2y+sr9YW` zoL=JN7DrdFyu7^Kkdzp!lsWEo*0`G2;$_cv-(g+aP+s&O&{UsW~%GTq&pDm`c%C- zpARmKTg@R{boOjr1SZBm4`#S-JR2}i&&!=6OR!hcQa0X5um5Fi%L{AndwE9uFwq$6 z>Kvnoyh-Y~X@f8++j8Kfk66lIWlow@QVKszVQk+&kCi|182FELU z5?SBGT9o*Ub_zkYgk1fFhy`oHhh^H;c?5P5Btzkt;O~QnJktNpnOFAgxo(3`RgWUx z){5rhZg)xNnKb-Yca7inRiA95DGxNDJo04rBVeI(n@^_A{)Pu>bG6hy(aO`AB6Rw6 z^Nk;Ojy~8C-HXhI->*BwEiJ+?%4gBwl)5(nn3>Npc_pkXO>TetRhHIct1Y^93j;uQ zFJ;>J6*Yfrhk~=HbwO{5ZSxS^i&7nc$C2)$Ga!<{nZ0`X410pP$eZI;Gzjmk6_M@R z#ANg{*W{>Qbotb0)7c`iiZKjH7{nb~7TB|HgH0lNg1uUoRUd~NgiqeC3%NT>C1gr6 z>!cGvts!BS;;lM1Y&|{5L{7~n0IC33y3;y_rR0GK+@~9q1eMjRYr6#KjN}pm`uk6c z&e=cNXR{vGK5#_4qMui#<2=_H^?LWnvm5)`V;Tg7t`|tNX7Hbv#G88m{`t*q*ChTb zwm|^Jfm=Yc4_|%Zx;#jqs!S&b^%p`;A(?DIEvd`? z{Cu)TInx}~{M z9{B_vyb3N@Xsn!(`CR1i+^vdWXk>2&k-4p%E=f5=sFf!)!#?I-<~aq$@c8M?+By%! z#VbdlJYU)vf;lAI72FCQN$>@fBB`K_(#{S@YuYfnyF`9(ywP?{gx}p9XZEakY7{2` zCEq=}(ss`=?e!BTPY$Icb=p6epo8WjNrQS-kr`B!^sP@b?ks4R<$|c=uktJYA}N}sNi_xBde(-Lpf=k-9K~fts}SQ zbGJqF7SkX|fGs*seHsMGSfwVgFX}zqG_JHOgZ_BR42C>-^-1>s&WXrs_wiSUo;8ZR z1}uk|QJd-xK{%f7)DX^D@pI`PMn%Xhp6naRyLPR`G(Le-R-u3iFu#xmuU|*h%j8`| z+JGp-Z`C>e>vMAn6_$jtbLTWD(PaXO_9+l{64{tBHqoS1uVp0B+K8}B8lK;@y3IE~ zMCGTxG_Id8u8{|E&3BF}2Lsc{;q|wExG?`}N<>CxCKjlcfGscx6(oO}`wVyA zCavP#!;MkVw0RS#RJ`rrgOK+e9}{Fk9indP{K+$O?6jhF+dgeurc>362os-i+qV1T zuje2yIo7^*M-Cn7$FmeQpa0jV8(off2-$X=FT_GROSPp6ld+j+*7ZbPUiw)I~u6VpqZN@IzEgQt!8#TO{szLQR0ULTSJ zM(&b)?0IKpYcKB^GgfmA%IzsTT13!p2dk^At4L)suHDmR=~YA9G=`DEBa1wPBmAs= zT567ORD-tD?7Pj=^>W`pL0b5qc#|e|5aEk!N>cc!Gd;3jK6*46>PqZ=KZ9^htcKti zbaHM~)I*YPFw8#Kh6X*vM&}>&{1koBf6`IJ_hf3eI=NGz`U%fD z>J4_6DbH&%p;1Lwi79Al-Hc97j(OsHA3S{MF zEKl{Cb)ON*vsBm+9*3pDaPRl4O$pWh`RT@Z8N9lC{;N@1wU zs(uVWr`YJSZ_IwFdUZq)LOF_3y|F?Dsgq$@7i*RzrkcF2WSioZY)+J&Eol(%7La%# ziYc!#AhNxMU| z+Urj8YQ$;&_4}vXbLE=>-1Ydo$?U7gU9uzWl*|*x4Q^Tm<-3cdzh)*w?TA2AvvL0@ zllf@jWV*>$`V7!2CO$Yti-wEvilp@PFqZ79$Rb*SH6qx;Y2@6q;}8zbH5vOI9s1WO zOcs84yyn~s0Iz>b%#X0qOOFioF(Q%+R#^H&V7+#G?SLNG|pfVBz3+Hw+WN+DCpa(plo7(Bbu}oWTUYe zB~xocu31hAjf3Ltoe{7VVP`xYCMVagbg}(9Cw$_gF3TrOnKIy1Q=jUv^vGdARU*Nb zlap%;pJy>D@t7@J##R+@BuOZRgyK+5r$Cnm>>l|oxo?Z~yt(9~Q(6{NFR$#Buwg?T zd^60Y2XdZ(?N{zuh^Rpw9`bo5P>M`Ib-F?pPmEp=z5=eUeQRqkwT}@NZTY+*AI^{1 zK5{wNcypVxPn{6I$y_i}9b8fMu`E>C8Jrf5EC_6}Sc%)MZ{z6qn|;iXhhX&c{gpgT znj=wGzrUp48*WYsWEuQiof{m=_12r~J2uPouues*7K%@*VZ&JUi`lD7Qp}$f$7zx+ zSuiwQ{a7w)kiI}iC;7|+m*s=P2+RraI_T#sY=5hu5~?UE6kq4qQy3|qIQ~Tp2K1QW zl=rYkJu?H@-vyWVea_Oj)VqIjmdpUEp|f{+?%G;wM@%bnF|3w+ZVyR7eA;T3l$&Q~ zU}-fBHxz=Q-1;f!;>|){9HqSt3u#^0`l!NoWOkJ4(yfP*>&NzepLS&3T@2dqgmGT> z^PMVo>eV#bg?I1j^Jh!S%8V|)0xp-Ktupt849j2yb7<^67Mo}}e7pSU%>bU@tqo^_ zWBv3&V!P}bHXPLIB&RBL!X099WpVOf$g`%`iG(vDg9Fv*JQ}g z@bO0$YQuPS(muTemx*A0;*#RmZ5S=%^8C@fqgl&5_53owTQb0#^qQNsy^}*nvD5<4 zp5z{cR-oiF>FO`Rrf26vxpkY;KJL9fwv6{P)DK4FWkr?P-MzQ02!GAac8!53lu!I= zPIi!yg~jf3Vfl1m4%m<`?VUOdMw}15Ld_d^P?!KJ*|IBvD}ger9b<2&=G9a|>Jw+o zkeE`3%?3S07EV7=yu|bvhmGc*w6H5ytT^>VxQCYytGCp$ zAE)LvS1LrBn}@@dJwkgOAhdMsa48Wq7HwGsw^QT`G|RjQXz@6S#i zG_||TFHYPUlCqs+dcDg%#ql6Zaz$Bm38vmN(c4%+FAx6LLJ9epkQcx^PNzK^wI3Ob zUSpYPUTTS_n8H-jSQ^l_LQm~gLQ zwVhS3YWjwDKkw}$;S?q1DS6j>MTrLNEB(~<$0M3i`)FFGTbnnm@f^fOQINHOAx4-9 zxnJ8Q??YB*W&}@^iG>m{+br{EyuFoc;I#0s;sWS6V!$FU zBMz<655A=D`1~8Wv_e4(HWH%}glSS{X4_UP%$(Qk9%;xp`NyAVtXIxJ%s=GuCs)&A zPFshTjKgXK-9C~*$}*Xf#Hv}n&0j5`Xhto{?Jww>exD450xb*E#2rtMqDfpqlWRVt zr6$?O$jYP_8#wGI{Q}hH0~sZ;;hDP6fB3tBla1l;03BoE+Imz|s`2sLENhH&s9ANm zU5r+h!*treq+=mE22P(;BQsu`H2~Mqu@0Zu1nExt>oSDl&y(uDa=n$roSU!|KTsYT z3&LQXOy?I~3{!OnyOabOG=e%g?!&XsO~`1|Aeb%Ggg1m@UH97}^pKzz;ano#k4j=t zUPuiG8>lxq`#p6K5m#Y`UBPf2+T6dk$Yd@GItibIplJ@iz{=$2Qh6Kn;3YbZuM{LT z12B*Z@Poie^fpcyF;%)Dm@%1D+P?Ef)5Nvonp8c6mK4*F*deJw)>|FYbNHndJvSjP z&X}Tac=YBlj;J*qYkhn33n;ZKI74z z%LHcrxnk+~MB;~7Qsov~>@GJ6ZJa%p)LnEB^0jvFcG)+r*)Xy&ZZU!@b>tH+Wm{~E zZ$^31yyb))I_T&VD#=Do8J!gg-WjVm6YsDHmA3r+wM=S3Vi?ygfAJvJkfWoRPfWht zwZ-oTx8NAgxLj*h4&jjeBOLSfgaIro8#Fo3eqF;K9L&(yeTD9amiv(EY8o)W!iLir zDBUyp;fViPMH=_?^@yjdo(L-~DX&7KaBh-)YluEt@#jZbUVC%|NaAEhx`gAKBzBl* zi1?U!g~5jpFUgMS9GsUrG=6>s$Be}03bZN;T52AjNmdTeI(O}=d}4NE%KooEzwJF| z-%u1!fRj0YT^b ztRv|0(WQul%KjxuMv`YGk&REyAC)1L({pcy4K?WN@|O?XMdgU1Ag# zTg4@;CU`f+Amv`MqcwoPbB!&wm*d-4T1Q$zL7UaPlB1zFC9CE0ik9(u;pt~HOm#Q< zVH4mLsd5M#cIu(XVt|3PJ(MXYZr>hd+sIs%3d^K?E{gd{H+N4I;|+5k=t=c_4IK2% z2y6if7ljIz43t`{evY%SxB{KcvBKIc#EEz1VlH+sJKk1_Lp$BW3J|zZn=qN%(Qm(LoIC+rOE!%?t(4xJ z5x$aoA$DoE6K^`DXJ#68@7{U~akvHrKI3LHJl=4hEsU)6bqE7wMAMf|hD(fR-te#8l6lYo!_l4&uQXD>49$n)3*!kD{3s)v*UC4hAltzlcL>8TZ z5i)6n-Q=*@q_GU*0WMYqJGFelZrUzFeR~f^p$Ju%s=co&l%}Wmj{pb!< zOE`w<+OlNRrqZcE6d|Z@pa9m~Jor>oVkMc*SnI^>GthzQMXPz``KA4Xthk;^#lEG` zzTrIjUnK=>LCI!Y*hDc zDJh~-RgfxXm#q-MCe4Y8TUspuzxbvtM+e{kg8PDSSz9Nk$kVyQQNa$3aM_S`$UD(+ zEcWuM^_i-yHg!I_Ysy6Iy43*y?QIFsd8Y zX`nUYh(%vs>l3yiBY(JsO>eg-@W*omsupE`SfE|$me_`S$~q%rvO5mjD10=z)vYf+ z=Cqd3qe;ZUOp5iO~Hk^8C<6AauRIAl37(xj`=#Kdb4 zjUf+m_Qq?p60Hm`Xzd(6may;v1Z4?HkAgntO;b|)Mu zcZz6PnujyA;u4U;eZu?#ZPZ|Cn&@FC(g+B+-Jan`u(yg9c7Vou_obWXeLXPx^oc>SD5`s(p1EeTM0z!ITIkfhsIx34oKUEY8tOU%0igY7?30o;%vIoJ+bFxO z%whad{HC|VC#f49oW)VdYcN!H2Q!1=>}o+ysgR^>W_~g9w$p%X{3+Wy?j+by&VIP4 z7k+7k=hNVp4jaM1A~@#h*xwNz^S5BB2c)O0=(4dX)3rr&Kq-Y3#BKMroPoZ+reaH> zi88ybn{5}?P}k=n2!t{Mc+if>hHVoYG*dve)ul9M->$p#@%d?dX#@XYqlCNMUCId{ zF=@Dw1p8Ht*9d@4LLxJG`aBnRWi(gf3YBmj z<2|LRU_Yl#JT$;IC_jhDB6WwOqRTz8T$${kG2e*uy)_G(^4xRnan?V1*c-2<`f^(f zDb2Eyq34BE+uGD7cn~%$87F+t{~kCJwPIou?s6k0z)Sh)XJu&EQr+P?BM=n=3@#-b zO(<2m8Wy|llqqeRwu7__p*i8y7s#Cm)O)n%ger&d(DcwH@jZv#Hrxyi68ZAU zlP6@0kIk9q;_UYG*dXP3z6=~V={9?lU5q-kau_oy7Onh6c!ipVhCKoJ&enZ_3#>Vdtx#Dc;c7ydlk_atzd!o_npsbgRa%2!F^E+uMP#-Jb z|FO4?pHhzJu4vvkpy>3_;a9PFb7ZM0E}3#$% zc>AR?p&9-&lrUrhle`{$(KspYYq){spxU+CySJ+S>(ql)Q_Sra_VO(Mo!`D(+(_DB zKO3fRjEmZ{cdyZ}Wz&*6zH*IEilNsq6T>y(i=($#FFA0|);f6CrhZ^vpprI;dAyT< zDo)&EbL-9o&Lwo8vsrwGiheEbvbR=#aPi{E4P%#TpJAOeqi}KHg3Xz+_YmC$|3llK zq@8Ouxfsq$(xs#qusIw5BWEZB;Aqr+t@zZ8$7_MYkXXNAE%d9VuJQ3gVHHAl0NHhu z0!o%5W~F^J{2~JqyVQ-}4m1E~Hi0Tz5zJ0CGBHu1w}@RlR=4^zCTqT{IIiCT&2+gb zyNv!ocu4$HqFTLuaJJ}6n>1~zbU(su2+-M#8STkykzRpTLy=t~|*+IY<<@xjpip+k}p_0wn_OF8eNPDK&? zw@i$8dhwk}%5wP=FLf%*HC0+`v}y<2xR1BTt$3a`3*cQ#t+Nm za*U?68oE#;{_`)>^6&$W-~Kbv?op6;6CkO^Y^t4a6KTN&CIO?J&?LkDwW)f#6?Lx! z?46C64T0#cs33sD(+bqlzNjny93!|@()hdN{==r@>}JB2gnv7kTBu|tdxh$QXZUlCrGeWoqd2(+3?LYV`RcDj?ZUm`?0L1BSezDnnd zfeeo)K8C|*&$2&D#WgOBd&xKSz#@!crkc%#P901A=FXLH645>^8s9Q3r5t6a3L7ed z!-dvHH7N~_B6S7z{c`58oIX7u)#%TWrN5dfDkoCsbd^Bdy@%vrgFI|aFlgE=j{%qd zAh6uEd$voU$u(5qr*is_^_(#8ua^^)>NHh{l8t?uLH**jt;8yLz+sZOzrW}tB$fnh<9iUIb1)ZD{vyBxh%BKPB$aH`tsRux~FB|c{G9}*C3 z@oWHp1)9>4vr>hoiP1Y;j60V~S1>=xC&P!P_<6PjX0>(q?g!jm!?|oBg^Bj)x@cUZ zu$1ls(Bf83ANo_Kn!q)1)mm;j{m1A?dFGLQD&=ILp-vGnIcv|PAx#rOKExG9p{5RN6$pr@uP#SKlCRB5#p z3Zz!BS!amY_GfM_C9&w8p+eg=9sP}b8g`3J6@+~b=6004%Qvmc@~ncs?>qGbfG#TN z$RYLOGDFCI4mD|C0>1@h$E-jq`Cn6&Gh?(jkdWhNr3RJgXp5Y#M$EdHVBa_xeVJwPKR_i*uh(3G-dr(;UkF5#!8QYUSuT)(a*|N@ zOBmTk_wU3-zGbSKM2QS;fpOu%!OEqnD;DTRX^=bbAi5Zy&CkzwW_QVrnAk*I>DvRn z3=>b=Tg5Nmt~GRC>oQ&Q_ja*$%&KkqmoGC|UhrR{39PM;Uaeb?+uVFE$ow3RC`>NxSZH}DbK7IvUQmq{u^xaK6S__&1 zY05oi(4m8-&DoMd!74y$*h94TJ57hngYO_BdwM%1?#hx!2N|xw_0VzeA^ut)97$?^ zoc`EX_ysG@J&$b*PRd6Re1MWdku*7OHIj!(5C&k%%rHJ`c;3U5*A3d%HrflwTIB0M ze(_e_soBM}s;vd-oPxTy;s)$E?^6OJ0*==4FviSJL9cqE)!_e3Aj(pQ^Q7s`8T`Vu z@5t2Y5@MwI(+k@uYPK5tu18RC+?j{W>DdpNz|&DBVbttX??K5~&3PZ(?tfw3t{im} z(1`%ko?vVs737G0D|Mqv?TX{#xd-VE95d$m7DfF;sq=uN{xsLlps_)nejZo+Z;apV zhttjn-t0H<7^~jB9|FtCgXcsbZbT6{Z21DTf++q*R0pjsclk2av8DJN?%lh`Qxk1^ z@cKz9=s^C*Fm{=plz;gJfw8L zGFQ9{*qwa_91+7Qwq?PJ0p8Zd4~Jhz8A|D_SNG2lv(5CU>z@pZZ(C^gKJ0+ryQeqz z7~T3!6g=hr%40!4O4AS>&RS^?0yA3Azh+hft4fpOR#T>QP;!d)H@&4`fl!K*sEp^j z1>0rUF8WneZm8?jaNef&@ezJ@TYkLc6qNa_T(4y_7Wmw)ERX&{?<$%d2~CAGpWDYa zlBHB7_EfZO{@*`NV=i}dNaE?A`ZS8jsGHk|P9OA{M=h-)=pHY{K!j*Uj2OlCXMB{R0AWEc6CaHp)C|1f(GqFL;T=C)RQB zuB}{~8%ghRm-B=O-)hxma^%Q)y8VX5yFRhCnl=gJIK}?^u^j5#NJ8Y|YjcNNu$lhK zl^l+(fKNrPrZ`sejraq`6O1T$f`UasJ>dXcT~N6b9XNUrO;l(|sfk;V<3Ynk`nW1NK)#Xe9uUaM-%O1MkycA1Y@#uk_oSdMWoDjxrT z#rMSPP4o}%V_%!qWOV2GhUOlBF=(cWI&k8bTx9lF`Z!Mt;`DJDN3Up#;VQ|(J*C@`x>7+4nse#QsJ&A(50Ek8 z-EE$i!5fw1AhDtmfq~x{6C3c++9q30IXbM`e+LwT^=A3T#_2TE-~N2?#x7xLR(fAY z%G?g>Q_9PUGiPclE3XJBli^2x+|If=Nm3^W{rmhD=_VYge_N{7&xWlU zcA$B)W+F(6vW!hJ=3OWBdYsIz1QL_oA)U|Vc8JxqT# ziCVHVspay#F&^3#JroF2=7r^cIC=79XN)d23N!1;Q>XQt+C=n*;m403cMe|I(X6+tBPiRCA0Rmh;n0NM5 z3ekble&G}S5G9K(@Onm$o{%e8OcXX9?!8-Y_;biQ)#vX!q(@%lw7Cobqo};neQds; z5_xxSW`8&-gbPOGJn+Y2A|;3$Z6E*MY}1PSOIj~%TmB_(229Q)pdy@__jxgVG*16B z3{P`^{7$e9G>i&YS5&o1lu6-o8gqJbzn$T-KpZ(NTJQ5i=Dbf{=I>l5sDX>IL+d#6 z-$TnWlpxCBvK_Ft<&d@yP2C=X$jMyBk{?Go&aD={ZH+{!oyQdVeerC;UqlAh6JI!} z>cA_3>23dZgL2_BTZ|9|XT9$AiL0l5g|x>#vMFSHx@p7WANgt2*<(*QL z-Fp6>)*D7@x4)She6xA6o#(TI$NPP~`*^6$>-(QK^zOUs$-A>H5AN-H_|~1f8-JZU z$D=*jg+~_NMHwLZMZKD}nOTqpPD-T?{6WwL@zQJkUNtJD@ zhGP*CBbRXYNG_o1fX+vmgMw2CC5PxuUl~{;RiDXm^_L>iIu6S{P6yLex+-?xkt^?{`&Qj#pkraoSA~E7To(LLKC!IBQjp~ z>e$Ic?iFnnV#<;dE$QWj!DyW&EdKzkZFod3lZd z4Db%xZbv2^)>2kpROWcQV3n4g(^QPLC%gSd+iGP|*6=*YIE z-?J}iZP8g-8zfXlN+TA_FHK|x>oyo#a(i(K`h6uXpG6?FW$i(>F$MKSG}(D%paIFp;{uEjJ~nR5rL8 z_S~kGL|jFmidL^&X*T~ZrSU}i!HPKxNi@^a(MfT6`vsH_uk1=>!faToAZi^u zpC8F}pZ2nGUghH@q!_f9c!VjQs*ff7ZUcf|7qvJ2ND;2&$P^vT&*U3DjA*s={)J3v zFK@1C^YcP>$dw-)@?k#J*}v$>oIjX$LcSyf(Vmg3sZo_o`^;Pke+L<^!B0%ja5vH$ z=AU}<-8b`fTRqea?EQ5UZ6l*D)k`mT{pc_YJuSnlSMf_bmWW@1*$jUgAN+&T9BG!G@+FMUTEPytiJ)Eb_wLC<1RLcB+T7w3DZ$1@AxsDNqQWM* z@q9K-5gdkr&PtKsC=D89IZ%8B=sasFgni*ZH0o*S<^`RTgaS>9W(HycW)yrF+1$xk zN2a~eVOTMcm7y3)y7fza?;h&b*vUhCKz-N!(9aL}=@nBcK$Hfhw!AuUjh0<$5A_SJ zL)DxwH!dh2VDsz{gjrSJghwD!vWDp->R=DT9MNal?=dFTo{cy7Ab~5^G&twDwstm9 zqu&7lKkYonW*?MhSPMw@^#i5Ap!{@Au{+fr3AbXqtbgeorPyQrYtv)xoEU)X`- z5e4TMd424qCUf;$=gfa`y#pYGkck!Ub+@Tb$8;FG**uxWP;0}6xT%;hBe-l9+>;n4 z5PM=oDz@w87(dANeQ?==++$>=aE;AR+U{&rrbMGXp zHT50U45~M8y`<7QRLfz^p}8wZ?__eBj92!%T^ip`*g0V|6-USoMFj(UIecE{MMkeS zHewdM;w#;?1v5|LkAH=lG!bxYXHhvMZTzflS}9F7c_x}pqL!y{deCn^R(@v1&(X$FHPwtXJhtosUvWnIQ2DZ;;YT5n10?R4j7#&bC$B03rRf6K}ZeM0y zhA&Qs8oD^orwm0>h&aa)#i9zRm9RCgEef7mE^jqvOrGriAnSlnXDCVH(*izi;rLGG z0y|QD<{K^&36;JkZAKoDcvew!_wryPH-1WO$%&*C8#>CTQ82*E)2vp`B&O z+$?*<)Q>xixJuSuXUOkaJFig!ydAaZ(F*1d(2_w}aENuLbds&fu5mm2nJd1ky1H23 zD12gma3i(?AC3+`<>RIC0#8ng8#C8@0dJBtq1O)A7HfF z*;q%obr}GKCZaRlAjpC;qwcQbi2mQTkqE^7Fak#kRW*b+upM`;S%pKS6j^>G*C%ps+OIl9kSDd zL`p45zv7wWUYC%k`{NFke11@_3u|pd*(hTOEHqAPo;gP2*Q9y#glTtx5T!@^@g z{(S1bFR}0Kdj+keNFY}?fBpqF(7O#!kZ4@IbD*w4s8&h~IYWIz`Qft)SV`%(!)N#m z?R7jS;}&uTisw_Gu4^Khr_v#GQMGa71h<)8`Oe%QC;9sM>M4Lrg7-csZ#cxb_mk|D z7lX{Yc2KO*b!9r0XEL)Z`Yk=h&Mdgd2M=ttDsDL96iMbTsJyV=WRQEWKLL<=GBKw{ zYUM`@0&1Q@T$6cO>Je@V&4bGTeW-K)<_6VK$ovH09^4~y&l`riI{|x!V}WE>iItyI zg#IeMWF7#;MU|-{5<+3;Zrvp`| zFh5QWm~fU?OiS%*Vy|9Fg5zt4?tcTMB`lo+pk`%Ghu*kf zWT8Q8U$5IdP-!k!M$aF;KOBi~9%8TdXcG)DP9FKFT@D!#u+cK6H3py%o=Q zt=oLszE^3>ZIp>r_^|_DG|Zc6n)AEu-M*|SyVAzZdalq)(T21s95U0ecT{+n$yd(S zneHp&vqax1DLJ|4?jpAoTfd9Nqc`9ln&^3EeXT_3DEKuCrlQ0kPpN77MOkS?bX_|< zyqRG~@^mgFRz)Y1#es!E+Cs}&6(gTM$c_VAzU0ta{nB-9MgNXxtVXg@MsQwxX=(&E+mplEs>wiyjE&;%NkSAKkhBN{uK!pKk#n-d%q>O&StSd_#&b4i~8-|Q>{V;HNVUi}X0Qab#@#}AV8Mst>P zHzpu@r?UN6R;WK}kjP^1L}?fk3i!Rv?nc|W=fY|$)Sss!OMhf7f~r|w&0niLSN}JO zcco^K7MOCM-e59Tg^)+NL;BxZ_N;glkS`cVJ%!}E3Gn;zv(J*bdObSjWyoDEBT)rK zLu?jMsApqq^o&z>)!=rcd&UldU5*fU5_IL0gKwLef+t901CD19;}kU}le9vlOp~%R z6U=6x^4`N!)pJMFC+EkDD?KU$SBMzx-<_66{ktuSilz+V_VC~KVcK&=A&CI3Z*3C}ezP;&n@pPMc`Q_8NCLmugs;sI!zN{5Ru@J|! z0cUqi1udsKXhG+ue?#oTm0qq8lf6&3N1&#P};*ZANx0B-SVZ4x_Ra$_C-PK4FVDa zWF2|z356mx5}*u%Exd}=-TKX3#}UdURwXkukP3u$j85A3ZIgp#^J`Vkoj`C9gwdJz z$ZuSL#D0LEgc4|X~1964eUR%xJ?){Ry%7gssS&qhZ5iu^ssDU_XC*K5qZfsR`zHL1fd zYS-Ow`@R?dbfu`A6?#W>qx{1ok8o`k-c`<_)_&7;q9a*V5Or}kwc;D*LN18e3!e4(BBI-1A(SP`n0deRlO>uk|~ zL&}#4(;j}{Wr=oe*2r-~7!%NZmY06@F4Wu1L9c6L{lWR4E-URr$n=?j&WK431(%A= z8q*eLCA2oXkySsl4>ywZR-mvk^)AUM2MKZ{s=$UI8hDlEM~_-#e+Tkvkks0$n)OfG zrg!qI(!P`SwqM1qaN^>{VLN^dh&(`!fFptlrp5Yme?6+MR$8~sFviDAEI4d$gcVju z)Hi56I)ppuG!L_tiSTi%B(-e_gj`hHKIQ9rpK3bNbh0UyFVo6?|J@D;49 zOuyTbo&nfj+}DUymHH?eWe%mJ^+L> zwUXJkl`pIr5*I)+uYT8 zlecB!?MCLiXWoku?m6xQt$W)?ZeV7Tgj~swmdP-teVoNHCM55R7rEWGjEr0?HdC4& z6&Js8ixw}&Iiwm%IV=-(BlSZmf(^C%L^M&%o`16 zZQbbKi?RT>7*}{o>F4F^qVCgsDb$zdg4Ejq`*>`X0AfhOP?{no1)i*on8K!csm7#< zzTdiSMoYT$2iE-zKqgwlX3`*vi^lvFPSSULQH1*kTz1X2bNc2mFF&uVrKLcCbpQy? zf%!=_a_q;?o25&C|Li@@rj9~9u%x8sULRVJwHj-yxJ%&9=O%x1_*+u0`7V<65?oYBR^(zC^=WUY&GS1sU5bUN+n{rGSgE;EG zP9C2q8Xm#~kmGO-EIG}$zD$edp}WQ10-e0NYFpoW&pz%zorqYD(Gc#YchM#=n(9*iOQczgY93{J)6mq*?8l@jNZ%w6`%>0 z=IpCfGO&R9SHC&-qrS+My9c4zb5?H|_ifyc4$lVo-OY z>;N-#hrS_)RbZn|ZPUMOWHVLKKuFe?C~G9ou_8Zaz$+tZIT_f8L*b>Nvi)YuX|=e< zP~wQqCRj&j=3WxW#=v7RK8sgM67LCioY-?kMUT{0{l?z^g|wy~0Tq!2n+8M!AgNI z>l=jn>W8Pl9uun70Z-ehZoW!3?33)OoLU78740?urCL(b)bRrbgxeiu6v=E6s`_iz zqPBao_*3Jl{5Hw22o4$dm<0*>+Wtddt@j0h0ViOoY;1jc_v<(QzE1pD5&>>Bbmp32 z&?l!yp53%`&81O;|1tELI8i(v7(aOJzGFIVr)d*|D`Y?76jWvRyB)`;(M)Wb0M~%v zzyc@#SsGYYQbKH`DkDQQlfG7cQW8#^4TL4+{;?&LcBFg-1Mzn}I>TrKdp={>wrD!Y z!gJ?VzqxBry^h2Ew&P@;I%VZgyUa_#fu^ZAkuy3n5GyG!&zzImW-9@ zG}mSo!wKuzqLnZi{L*)BgJxYBjWVanD=ojdAT%2&{OY z@JC)e6>D9WocBX+Ew-QbT>tiKcA)4(dmP&Oa?T#WJt3d?cO+=^iP z(hAckv}9UJ%$+)Ac|+3;jQl)a&xbM4Cltg9{eE4`$H4S942(9M?7AhC7G!BNH{9;VziEiS0;b0}4j zA}Amvw)+=0Ex+G_Y#>sf=@8z14roX3CoN9j8#_iPBm`bHA#X%Rzfn`FubFQ!==)u+ zdP2@Y$rK8~vZ2-l%iej}`F5}m^HSM-6Sy^qIP$0*8WXVUyWP5|!Dxac+nEa_tYcJb z>c!%*${#;|v_P;IR*vvj(y{_NSuHQR(ePH%32Itz&M+9JHBrvPEaN|JJcXK!lCbJwr7mQil^3Khh z6CeY0|9#hV)BC8vvp~@FQVQLlThsKfJLiE&6K7#wIj~Ar27U-P-s>RI#j-xA%lcpUhoKK=Cvr?cPvaA>g@br!>gAs zb)AiRhqh5rd?!(^m}HPcR7}4ghWb=VfswUacdJzHy{iusvZUHNIAhYJ)Vbd;j-L>h z##QX_c!$}sZ;zXkixaS>qcEh57hZUFP_JIKT`rA&yLtApDw9_8ynsZAAv@pgXRYxq zdy2FMZv%9fE#C=aa06nTO1iZe?d4U)nRA9tXI~f7Nj2T-i|T3Pm#Dmm<6vJ$Mn^w>EoS^87V%{wzK_jc z>AMc@rM9Xn(|`WG5x6h8e%oI=xCo=0U$fON-P!3MaO`wv&v8WTYqKY$9FlCKxUY$0 z<1XHy`|l0uR{ebD?cj_fhJmrK(jGs3D%lS@S2!o{1B3^pP7(a*)La5=t3tX0z2VZX zM@W8`_-%so*1$fR@2T`3azs5a-vJ8Nan6J~E|=;-oivdMhJ&y87L#DreC{2%gy^xP zL%Z940qo{k8z(lA>wlKjk51UG`Zi9aGR#IF#w|v2S>4fB^c85xyl^Zb->|T-}w3unXBZc z@nx{gpRM2mIO~xoq9FL zW6-)u1Juk~joeUOOKEvX@-aO@)nWbzy{>kmoAsw2FZe;QQ;-Cz%DigtJxypmyHKKu zU+VdSNMBu_@lmlMllHS*?_K|sctB;ULS?wxHUc|kDuYxaq&d7kyngFSe#?a$dvritO$V9(sOP|H--`Lbs$Q08jJRw z#{w2joH`Z5Q*DI}=JA?MRm;?Bg_c`KGEg7V$)N|bRwU62BA*Jk8}8>w&&rf374a5wPD>4odsHS;cx*bu>WLysPzlUA zx=Cld^gEshxnWUB#2)@Fb7=q?!$Q7D{)KpS%gR>uKF&#|eE)b=(UOl(m7uL9qjJZO z*2eS3<#s#TdED(uYt}`DKKJ{x2qdwwG0|fjHK{I#h>kC~$cGhYU5rvNRi?;$<=PnUPbp@*;%?xGX?Dw_Zr+xwlJVwjG*dNx`pwyN@a(|3$(cN{LD!oN*z z1(Z_0&#h;XUkx`$WPCQ(S!q8q-_HOb2h9-3Kq!d!1lUmXTfZrpYVg~|zEh^up}dQq zG1{~|W&ormqzxf~h2D-HT`Vo_;qfUG*Miy=EZXg0sEsNpNAJoqZ969~SN)w9(iND`j_F_Q8bo z=v`-DDW_kiJK|cXV^esHEN1Ca0K*mP0W(K{DRcWtni;!P-Z$*2H$BK^c=()g_S(|m7* zuKRr{Oi+|$^;s*ko#+q6K$DP}p*j`TQO~BZ=R6Kmof~V{tRWF({g{U~pd)%0hJ6m( z#iQ0S=`cP`vrbN%7$;DjJpc;h4s^1FJlp@sXv_E!n@f%@T(`wJq*M7;{cTTn3^e!L zDMs`OBu|WaxY^0LhiV_*O`}lXHaREt-tc@C>aTm$lsD%ogWn24{^FI4EE#-* z{CVi2IY-N7^ip9{y|}qi%~Pz@eO(*C#!XlqwL0U7dGmSK94)QY><@HsYo_6`{$lU! zpbzJJs6WZf{8HKOGzUo772dbQ0G@DGq*nj^)8A|TlSdUVAp)%OlB2cFqdxw)e}4k@ z5LGt)4vS8XkjIMSb9OVJLFDv6hnN=|%>I_OYDSnL6_^;icu_)VSO zae!6iWgKJ1oP@uTcPs(4>srmaLJrWAUrq;MGBSEdK%_(#QMF70loF{vo^QR|Be+t)<;1;87@^Cok_y_m ziIn1jucJnl)Ty5P>b6I+CHTg}4XD2nc$-&f{R&ihscmSPb27f=wW?$7t*y6=9`VuV z5u5u_Ur!be*kQ-cTNuVb^uamJfgLq93_{~p*BTYs>oN;iq72n_JwrYgP-FnMB;^05 z|9JcIR@Z__+&=$=eTdamT>f~W1;kay8eN9(m3m#~{h%86!(3;fVe}>MfN5T31ZAHH zKDthgzFle_M=eyNmM*-sA9qo8hh2SxzRO4tq%=sY#huJ`bY}F@z+9jn)N7}QOv_{W zd9UcoJLF<&bECOX!s<`r(N1FK4%l}5`}5p9y>Wv$*zccdwd|C9zh-aJ&n!d?F!I2R zjRTIiZtHV{LAwQ~g9R_7M?dmt9(e)q>2jnD@4vB7n@E*PITY6bp0V*nwdS^J~v(Ltq{^thW zh#j{V9R19oTWSW+9f8?XdsQS1bljtmLGEGYP7%7Q>d_k^oQuBQx^rhvnPr-lf&J*} z$B#EGo_pdwMUnXyNc(zYYp*fv?d=I*hBo=w&1Wy6c>}hQu^qSR`Xj_36?y6D={hDR zq3faseHcBmu9;(+=J4lVGdoZ(Ns&@er#DF4xx}6gX}RsViKU-G#4Y~>hPwybr*K}+ICAcR#fcFMeqX$kAUOWqN{lQ0Y_2U=yA`xEdJ=bQ-VA`RE*iOZ8L|! z7(bPyn=VCeHPmai*^(CRQqZ)DQv#RFm0cnI=t}`d^#|CHLpScM@ptv-@n}<*PHyzpdvi0%-T&rr)B4v#YnA zTJ}QGl`WLyU}~OsTi15Ce5;SRm~~%qOoS8s!4+G!Y^hjGB!qOPLV}_9f@*JWW82^+ zWtEudrgu8r%f!UQzwG%!+bL70p3(e9yR+ophlh6-R0c(0T35#Q#Th0%A@HkMw8ar8 z!RP^x)X+YaZdp}+Umji9hnNco&(|b^xlD}su!PZGr0Ydeq&OL`Hyr|&M$@y z$kP5(542aI+oA^x5G~I>m(;E85e!Ip6`JQq&64=9_Z^zpK*8jre4lx>r!Lc*e>S(lKYy1@sHmzK8nyJ7>YcuBFPHhP_C z$-HQ80QbCC?`80stOJ3yn8PH!B6y3!d1+JAAZ^gV2yUZ?9GdP~m)B|;vI#WAXv20CdpMI zIm2n{n@lr5ylsDuh7n~ad`QFg}tJUU(D z|NGzf{rEi|zu)yZuk$+3eEWPq@8f+OuXP;FrYG}2h!Gp=&_(EMvw_Olw{L6GZWwm6 z;omQL{W^cl)r9+n(~6>JnPqm(T9$wHYM1WwY>cO0ZT>D}gId$@9WBjM&CAdJ`Bk5{ zXVl_K$(stQ{{zeS4! zH&--E0CIdIVQcReZRI<~G(F#NMV=^Ba2XB834Y=iK*r2cbq(Gk3-o(XNNWIJL?vGA z-hb!Fu+izBZc>H$B`7QcmH^J1@sllNmKNL_Nb5Hla(;h)qeGCcJpJKEi_Z+X!D#RW z|C8hbBKeQa-P&eGtv!>^a-cW zp+nV9!(%fViz-R58E$da`jhs&1sF^#FVmy&7zs-Nej%eU`)k zDq{_6)4#uSmrIuSHhIEdvuAYG*d0RxCy9KG)56Xkno&aBR#QGo5a0?%6cIE18zY5$ zD3Rmg>GCR@4j^*`DNW3mxMd`q3Hs5EyYt;V4j?AQnzmgd4#s)qHj-$s&Xo1)cu#T; z!u~fG-U9vPK<=v><4%@$gfyf3%tq_*0p%B$S2h$U|CS{&XN4g6hQC|VM932J{S<5f z-k@s39HVZ2^SKiwYXk9%=a^MT3y;?`xHr%?2*(R-I|6;4}#VQ|+Dw|Bb& zDDziOPr(bgbckZ1y<}iOAQ+bSk)1w(gHIQASUh;N^NdwO%Fuae4Vd^C#<3yGGZ=$4 zw`SQ}f6;nKNd$%;R`T|bScgzLo=}^!{Uj=`Z&`j*yqQWzs8C%J zbrEEA>rTKgUNuRsq(D)coGV73KncSFK-vEl%sM>`+M@o&7sfi;8sHRTW-b+in#L7^ z`qjLsrl%I zSOHK6>qvFIZ*!m%bdZw@`ub0N$c3;*I{k*QN>UIYUc14U9<;5-?!JTLS zbBHuE%Um#Gi+7O`cb|3ianrtB`x0D)05*Nq=_gv1V5<$))e~-5+23<#a{NYA+S%T> z7m1yU6|k&IJXN@e^C&BJIdEg1<%fs^5v;W-VpeZ&caO_OyZUbTgwfuNo@+AFo#J#L zyBgpi={hf=QG($rpU^ZHFQKyrIUKwA;X+C2BEZXR;4YZWz7y*NTf@?;F_E1|q^(eo zsb^G0RYYBZ4U+QFrFZs)3sqkmGk%Coo8`OhcRM~H>J-D__+BS%7>Gzl^1z~|2-;a| z#h;i7gV#1yIx?;_;$p1UVJ)0orM5uw7@3ddj0_C?mv&K>23BM*S1j! zqpL!7q*~iwrzwuw@H1mx^|}T4BEnviUO3G4dqqO?gd0w-t&;%->@xlG=1lO(!*#Re z|9C>heha+4$1!~_1POXWn-bLJU~AX)`P6*%5nqWgNI!P(9cRk3aw%b>56e&b{W;_0 zEZkjZ!#E!r5V0%Hzuo=r>#gvJCg@qea5T*|H@UB15>Jt_<=3b$%SJzcR92SE%~}Zx zt8#j&MQh7`q557eBirvtHQmGXdYCA;pmi(X9Qpq98~RPHX3dTO{+eq}>dH-0p}Bt3 zn@3#!cL3ZyW*kjP8P}@SPTfN*_#Ie7GP!!SYIkrAD1Ei9v~6+F)qeHp;bX@(L4Gps zMT{*8%Bk>>`!{VGz2WOxq$2u}BLSGwkI`P1%$OJ49`lWh-;b9aQsEEwBUG7yChk^o z@%CTOx6Qi)*ATJT?>CL2BN_GPWl7$SlP2x_{PCmR^@ks(^EG?k-c8RIiO8lPkiQs{ z-7mJoXrtG&QXcWfdA>w^;b>xVO1Hoe$;m?UvS%+@a?*ck(s=rB&x@sCfIpu6 zc;s7opxDa_x;i3n5muT?-I_;L$Ht;;5LcXu$?)^g3JR7n9IQ+~m+S8#` z&xho=eBqln*L0kdG-YHrE{XjpZx$WwH2zrdi42xx?4x_2O5kMf63?`LXS_1Bypb9+r`NzxZLt&)<;&tMA>n{P*nUT3UH*&RO5HM~{x5PNUE`g%SRMYF_wF zz`D7Upl0q+5Y*G18KNgpCexp$qTdLrK=!$&_~W4@rNl|)j2~Z6T1^4n3C%EU3@^7V zc7~0L-BAE+$%*7#xXM+C8t_5Ldf ziyo`JYHP<_cm$FO6+_yvTP71G)l1-xIzRQF!z+N`t81JN7^bXhS^NJYpe6CDX-w zgK{JpKD-^;;>XxsW3qa4Yv&)ECmfw-BMi4>#-=ZHV+fl9_FG&AxTiW!>vLhl$(t9= zSv(@$?@xcaFoUm`f4~h*r8Yr}MoeHYCH>|8b7)ZOJw6+oOa%?YL#%4IEK?|BzPb&k z=8eT^A!1;v4S=qoBtXRZE$EkiHKF2FjfM9Po%x|EJt~FUE_Tn22GwqsJhN*tq7N9% z1!RHbmaTU2JJl&a>&6XrI!tr!aE*swc)tY(n|q3eNfzW-=h+x-vd~?@P799P7f76(U_yE)@r+;hd$BEq z7QQ;$LKLd&yzrUB@T@{T^UilBPIIn07m!tJ8=a&X?s@xCkYna>q&&FK#q7Z!2ZwI~ zjCYL-xnsQin7cg${gaynYQbEIHjtHH`e_4%HEk}@K6+Kha=Ausr!|L_uU0K*N zx{#G9_uMXXHR&ZIJWxW?>`I<2$zCLc)v)z&qJvqlGiX&( zr}}Yym}L)hf__r&Fu``Y(Y1_~aJ+32BKCOW%vDIH*tn+z%=}aVodS%APV>DG_ ziDnyAU3!=^q2uS4vLk{@hvvpe_9FqdE}#DkRDTz}^oyJOG>OCck#k{7i1iY~$QUn; z5@%V~Q)lCQ%65%hhcUh+IlsHpt?o*RUnqZ;J85_YRC)=G<j})2LP1L6GIFJucEIX?7?J`DsTe!Uv5(n;-XNrd zF_VdQg;Ut2fwKrDctq#%*4dr)T^@8|%jV6&{1KjpgW;tlH7YV{n|lln0l~>Vziy4& z>g)D;A*M-~L1f18)PosKzAlP#V<}2~NCeq3HY=ce0?J*<+Q)$b)-8D_vEF&kwJx&$ zjcy3h1!=NRP=LC1IaP?buNybaJ10+)W;{GRHY~VB2ZuI8o@r3bO2!|4L!1DsQJL(G ze-V&(_3B`{YY{yw{``_%F>7#Cz)(1L(KGvT8Y?38eve0gUaxG9IxQ~we}dIUz7K66 zgB_deHoQ)=i=d25#V`hLvMBaF8%9vf)aCWuduP{L0MjXOGUFRx7BTeCSR`?IE0_c& zjY{NGY%q>}4+2fnVj&I@CF_fZGw+Pv&haGn1}VbjF;_(x!l@tK?sNx0fwj~#FSV|d zmVefOAy1w@O}X9MQr!uTZjMdJP?gwub{f|~@3&%Dax>8$QQvghwM#s2RyR5B7F|dd zpH>E|hk1L4iW~rDl@UpT@)ff25>W*n*X2jPYU$fHe4m)$?K;Wn9}3qz)*IH$962@4 zJKA6pj`O^R=C7w_c%2R>32{CzgyFjG8Vn5dvf`OO=co1&ohFH&D%-bjzcFfV9PbXy zl99;Cx$lJr!&8jU(_KV70gL4WalSlu9@xKMJU3&SM7wW>CY5z3G?a?8bE?wOv}xA7 zk7o^TYHUzTn;W?|02B{I)R_DEMk`uw@RPU?@13rblBUbQMXD~z><=G4R81WHrSCVI zIS<1SuC0$W?kO4>&6}Dm?8H!1#jGyIj}847GI0PYZaiwT&-Oc+i2q3(^Rdo>^rte4 zn1Mto)MqZ*u+ca5*fH5{B0XDB>K`;1?|kbh=oxYtf5xQM%95SC9L+M9`(s@H90bfn zF(e?YS?1^Yi~{Vao4#RSGxR+$ME#jN9AmFq^0H*@l6Peb$hwUCf)k*0yWMr)6}6>B zt4pY@S&WLY-j0rr62j>puzH5uZhHnds>YZzhfB}lYrl5Z&oBL<4|7U3Oqnf;&rx$K zXzQuI7pP83KfiXDJ@KT4jCT8|d=!cH+`EJcoM^cg7oEhV!oR#h4Df^=#AL(N{xY>eyomak0^^7IxIvzxxej z#{=7^^djaWA`#~TFB2c|BfGR?hK=|A^yY43euL6?JBv2to_Qvk4C+W)nJIV#faZxC z1&b|~C)FFWptpIR1khx$Gq7?wmS!0%EFTWR)*+F;2slyqa0=czhPyf9vuFy*FJ7A0 z{gbYj6=WfoM&7-{d`6_&Rbm)PsnO?^Bo+r(b==4JaFh;>m=|iY${R|9U3!i#TX`F% zQdNtOjsWiR07F*2`*G#&j8Rvqna}RsPv~EN+J;}J*jK>qIeF^Tm1r=Z@(KhE4w_TA zOG(_QC@59)faJ)qYHy%ZYBg8B*YSnv1(lCOGbU+%@@`10i; z37~$k`%>_FNxza(OVKOA83-`Ge7*?)cAEFgMzDtg1gx~C=QT0L-3fd#zA=%&;+FV* zw!W58()q`0i)xCG^)cg-Q{OpRDV;>`M^9T3RV%wIkK*Z^%2!{|iajRNL}8oOVfDIo z^(5+>c3%tI5*{rM4ZLU+ALy=Yd{A%9I$mrOe zT}q(FfC#&=bck5<)s&(cy$EJIH_O)7$Ahs>pg&yY*5^RW9d1}>9pDn7lsSx4k~NHaLC)%4PuKycVRt=>aX6_ohw z{$Qw^01B5^d^IY59XgWYzF;9}LNwU@NV?U9<383y>Tr^Gifiz|vI|vKuu^c>a^s#Z z*481#Nu8u`Y}d+M<(e@RpH&}WSn$kX!CAkw>7SpOw)f)~4XbFqKdgR1h-B^p7v!iJ zvI~`FU;H)A5n|KA$bv5lCaZ-ruYBD?LxjAKdGj&wrPI^U*|sR^WWU>2bH{CZ*OT^L z8Q3Jz?nZ+R&+Y zm0qTOZM&+Uh$CM{oU3&sqmnZ=^_x~jI@{2>m-at(^5ouQ!!3^gJ$l>d?txr(Fm^@u z-nw$jmS(tdLt6QMTd{AtJ$1&s#!G*H^(x-XBd-CD)09&|jq z+k8gC>)drUCTtMnqJXyO?(L1m`PCZ-!q@M^MqhCc0r3z~dGUh1o~3#8!HpqtN55Ev z3ENJ@J0h}B)KFHQKf3pkD%ZFK){FkB^W<1l@lxL=;Qrh+ep|Hls~h?*h^2!L!7@E41pOC`UV?F zX-ogai|E@m2^G!c&JP_v=|=I|qO&$sJ!XefV;yXRJzsoH&S1Ph=upenqb}hrvV$ve z)qCTfJ#&B(Wlgl`BRQ?-d6emo-k>~V8E)cdLtje&rgcxMnl$0m;T-28vckkJ=H0t;ii*7IL%u}e=YPigIE|{E6qFa$Y^K1ansCDWX7StUu zIf$sPmhFc(v0Pff1RD2KyH7r+kfDVWG#LM0vA5TMUGI*|GW9Crei)VZn@&|R5^tF)EUw5^wR9vTDM-3*y1Ar*z8V;SS>{^9Z7*{1i10ioP@>ez+ zA1oZ?F~5F^e2nvc*v{jBRuFM##2CKBt%AOyPcjfU<7gO1*I(-+4!X zS7m@Hi7_|J{P_iIpe%0R8!`f!xTSUD!t(FS#fqNjrkgluL0T60oHODlLgBi%G+pdZ z(+;pG%khMlwbj(||7lf|{5RFa4K=G-PukCKz9u>)a!AT`s|cf>Z^~SUJg|*;(=R?Q zOmA#+)`-Q1ni=`Oxs+PHW?j7_*F)*&=;T^!&YsLlLn^XEl797J*LM#Y*n|tAr3eDF zxZpH-PAaTnAjQ!?tG)>vnfys#x6s{?73n^Qosk%ejqWiX7jm_O1vD1wiRu))pnogJ zLB{UFx{J9Ck3`a_;m@qfI=0Va>d^KNc(fItDgCU&?sIkZ2?u=h=xi(Q-9aNp)Kg66 zDjwf>yP)rx<<4myv6Fq6=k<5Of0*vgP8KK85gBj5Jv|LKB;qH(hJ9WvgA(SdXxFwh z?2V7EiFJ1Lvi?t2aIaeD+sEYAbd6|#$K0`KX>9?TKg6`sxo6&Yd^6j^dV3|{T)s-P zfv1wJ+U~Vo(Tq%OrZXQ=p?qLwQzkHo_PF>{qn+zr$;kBwHX`Rplg?{bt@fh~KtiNz zqty)s?tRL2`ClHH^x<@mY_|@h_QdnH$Xe*EihxGu>YKK2U&|ZfdAov&B9p(teOz@{ zJ_?5gj!;t}&T!zZ@z-X_l8}H0we)+(KLKKH+;$Q#61w3)J=i^qNZO2#4;DR|zwg#=`E`& zEuY({eD8i&KbYYo!FDUxN#62>3l@x-u>}ixc#5=v_Q52HnEEuLiQ%U{U1IFlGWj0u z7bgf4c!+_kMWV!v{>@`Fxa-Wf`uRm`i`w);mkJ7M*%og5==Y~`XI@2B8VfxJSc&XB zBxH(3I{tYT`tKvQoJ>ypoYY}xj-1WoeLkX$Op_p6Ko~da1X|vXA}hHp}`_>}a3&qET(iN9J&CH)G=q;u@wsHTC`36kmNY4Q!vPKJVApN$8pP`eK$a$JbF3#cwRDu~3(n zZ*(9loHn(}w|q!!?FL4c>vSV*;Kxb8jkvnURE(+q0n7XN{#?9f^sXr5OX69U_<-o> z_5&P-wQV%l@!HbjGNteA%`xcQ7|ZjhCRJJ_lmr`J2X#!O($kX+Y};rG;vAxWG!F;z zv>+gWcw}^D4yiq{xw+BC+3`auEI#=c@7~=~=W6(s(VKP*@N+I%z<0lvK0t|6gr-6) zQ2-bBQs>k`6mzhb4Kqwj_zuce3Agg4D_4~Nv3bJ|f4J!|>Em$|t<2)$jI5f|lV*LY zJEu;y?>`+Kt{IKGx6oAG?w)^CQNs04s}||lwpjP!C4($hscr0FQGKvfCP#3OOJQqe zttuCqaE+FU){zPYCa*h3r`MO;t#!{V%hQ};Qa?u&LNh?`SM!BbpZ z=PHR5Wl^*Ed(@ALjq4lGl_N6r8QGzQ(!lF0dw}cVjC|O-#evtu@*26?HQaex(g*AEA;2mt<*IR|5%`>u>*1NMV!>p~%IXxnp zR}gb&&ZNv9ki26L!1S`NTnW2>5L6qim<@+xpLefgle*fAV(pW^hu_fhd2%i&=l;Sa z#NPvL6Z=n#9CUD~JihV1nh*StIB6Oc2+{t5)3n6n&Nq zgSe#=nnjj=)89IFo7ToiHTUEfnVrnz`Zl*qr1RwdjHona^b0B?(@m#i$6<4S90w%Q zRJ~Pn^QL4uvX`p51k5bW)NI<+3L)b5PyJK4eMi}3+HduygR}3uvpml4&u?}Xll9I; zEGsvq-W8dmt~xyLQU7;;!cx~R(Mx=DRHt8P$MrF_<(a-UmuD)AHUCBz^lWA0H_$w= zF*Gb6Ko+jyWT1R%Y%Y-w_`fzU(rdKj3UyHGaAtQ%&tsBXGbMK3I8F(r`V@@oT zRn&xoV-IM}`+lk3@*fB)AC~LSgT$2G4#W}3cZj;@EWAUL`F!%`g z44Uj|SYq|@SZU0z&Y8ns4IbFpZS=jVdxrO&^F2%zk(5Owulc3UuV22@H;ubTEM=P8 zswqUdl;_xaqkITLXe7GO>b6}QIiiLS`!*_*#9}0p`#SGvUu*hW?Oydnj?e03pdeyQ zs7qsR&2QbWpX^^MK2{eS=jX|D%nDi&TA-qnM*xty{aWYQ9j8+S1D^e3T`h1?K$pSzB=^kuPYUx*V|tT$sc} zIkk~lFzI9bbb#;q`H3%+ck8`8)}m8>UTN}MN@5_aGwx_Yj0^RGY&a`W(e9XPcUyaNr=U&ZL3C`84MQ-%6pWhn7IYempe9FP?-0eZa z&`oKN*K!A-rIf4M&0xT_*{_-&ANs}grYl;3mB_alcWNUpfCr~Pn8Ih3pr`Apn>KFj ziW#o>tlv(P5y%xJcY2JScjlNIFB`7uns7FjyFO~~+)-P->Flh_262z*1x!i)yrMxQ z?6Ce#(_%NE0P-_#Mzv~{aM9+0(XTZ9HUu6>bH*+m%DB>{^&5328j~@@#}Kkusd~+G z05XTW@7v16>>u!JIpGErmd`n#*$kN()*$xs`~AmW=EeVo5fI5NUoO8I<<{WJQ~D7P zPtPaot}_A|ozb+TaD{K%RtJjYS#D$W0;{8!aRprX^{4Rq_0jGJ$J<12*;1Pc8T-mP ztQ$;0Y)YJ;ocfu#EfG0$rD8P_y$2#CdncE-u{O7j%~dww1FaG`$nr|wK?0xtBBvi)Wh5R+0TXn%kCw+~8$i>+Yk5c*N2U6<^-`v-U} z0#s#zrrx1`zt&8<%a&#CS~p~S2K6s1{K(R zAAzRpv}scvSn%%M^c(b^62!~>Ujnx#hE$~EfqW~;hmUc%jFfJc6^va_Q2$fWZ=1F`PWTf{=3cL8m21|CYPUMQ4c3<0 z)=GYFUAE_CZX;xLnoMhRJ;SNOFwBrYLj*3BFkoD!WxX3fC~e(MiOF|efu!5i`AeX zk3iJ{w>5u^hz$#w4#bv+el>{ofxhN2L^W&p0{04tnx> z;xDOTdHSaH5L$FxBvNzK)KJy>Zct(gSR9p3x(nRn?0KoD&@iS~J-4Q)o?nuh#s!=l z;9{fTU@+0IIgx>Kr35NUt}CC>9yRqtysbPqnMe@3&momvmJKet@A(zOc!3)wh zI!>5SU2pMo7PJ`5|3zDoclv{}KgB|C`y@%Rnmu#oUW=i%byAp8R`E8Mh=80crcRjY zkMy`Rn|3!dUECG8bPWQS3P)G-V-HcM1a%1*yfJkJHZHqlZ+-*F#HylO6;G(5qxFo@ zLEdIy7_m$$%Hv(Fs&_|%#aphl!KwynR63zf#DhEP&B-y>HfWm?-{{W0d-&hs6!D=E zg16Ln*(PK11y9uD?aFu-8?vE|Q5X>;Cw}q+rQ8IgH;rk zBPS4;1O%sq2vzoOp_?#oa>6M#HafCR<&(3V-HfshZymq+Jlwk0%@HAY%xzPH@$Un9 zOxa|mg6Pp;OOgW>n8c=ps2JwZ#%l}j%CqDUy0Z% zfVpw&)KBM`7r*;?RM|mY2B$Y_(xkoB8M=#rXTySbUyC}ahi|%&_aAxDomDJsXSL`MckR1Ad;cmoecZE~-+i@L;c!0W3p9i5 zxs%^cNsC7+@sqOVij_~Gbvu`SjEsp{N6jo@qPfFmIkz}Xr`Nn==!Td|4W4yrs5IvM zBU?wF!jpeJ1vjV;$)M4+|AUx8%xY9vTK$@Q>tdlq8tw4z>nxP(6eRuaBRxF2o&b-R zr4M7?)zk@8Pv)AP>G$Q|j`apMZRshyRI5^Ujho~CT{odQx=tCBh&)#)TDEM-M>}C% zAz>_ldi51ujg93s`<$^CKcshBh+%%p-*pJ6xVtisNKi=f93^Fq`%p%)D0MR;wz!M3 zX$r%uLV*GuC<%3YbaV#G4FnMa+oqF&w65EQ=iWI{rS~Pl4A8guemq7;TgoylbJnOHtl-k zc8$AdPn|mDm1!Cx@zdNY7v^L`cM=!3SKstn?^pHe)^KmTGGuKvyJ17Vq#eh(NtvAF zG=LQlOzg_gznfBblEUe|)wh^7zQG)s7?U83>M$|U#LKVv{(ZqPeqev;2VomQY&>u? zIhO;S!>d=9nH)A5gepkuYr;;BdwnXhEHy`&U^y2Nn$^(RbLOPK91NgI*CX6`4CvMhW<<4LzbiW*jGS0u$}$BP1o#T0 zvr`I~UIFwFCk34$Ad04260;=x6sg0*!D>ZD>Nny{Ro$=E<$Hb=zG>4+^ac%OD*W2$ zRMN>}-0q_5NsO@*7reZ^3bD!nXrHWSKiO!blHDFYf=c(qd%VCmV)hv1{vM&x&o|}l z47c^^#x;(!8P4IYPxrbjQ57Whn`HBar?HEJWUjfAwpUF3an)#?WsCq$1YAH}E-DP? za#c08Ak;$&1|Fg$n`!-eViue-Ye~U620rqrg>C=yyVB4ThW}ZtlHJ zAx*b$h=>TJ2LMQ@qYyAWJUqO6f&On8VDd@l-|tCgiG@Yrh58QtfUyXf6`#|?bgx=M zN6l!-9xU-eEqX{R*n}c&H>0IRB_k~^UlgdAJ+x)j1Kqen0bfZdxt8?W{KHu}IhNry zEza1^J6?nfxglVWtO60gwcZV@nNz1$C!m+4YuOuf>51yPfbkJSlscETB0X&VE04qq zWFq1R;q@`6DoH_sWcu7qU1C82-SMQH^cl);+F!cj-3c ze%g)Vx}frGsT`yKo`#>jVTt$3O(5mS)eS}xvbXa{ctcl#<5V#W)&T}3U~s($7S-ZV zk1=^{FGQ(FY!|P!?7HA*RBVjyYqhe*jiJ|D9d;8J2cvd2b0Qk5nhH;iCU<9yq{~Ny z_4v&jC!Li&5-R#tSFSver{kA{2;8(M>D*`nMY<#oVf^6;ah_k@!!dlql>*mmpZf(b z+%;Bl;LK9e>pfSle*Ta#Da*+dSYSLVw`Ldj{C?^R>L?4Jl>Ges;A{bG9NsN`x%?R7 zR51o&KAQ}1VqurBOd)8ld@RDxes#xw6oXOy7rZPAx*dInd6hLnos4c8{4%a>*Zjet zNS=M(QMSc`d2;9b6ud?k>9nV(P5)`M6(g#ik_4}#JSCpk{oXNsNqLy%TNFDCOBphE zD7>r0uEf=veN%QKFe-^%fwv?*x-EIdl9VGs&R{guE^=acafL$0A|U(7=d##y zuCF`71GKVLLM$17_TN}Wfvq@zS_<92T@^9xdPi36B>pF27S!CIcvG;@PSCs*&i z1x~=3sfQYJVi%9$Ut(suy4HaC;2o3DO!FoVww!S6y836CFP3=!(MfRGVAU4h14B@^ zCr_SGmczS;k9fUzE<+{piK;0DEnY2&OniCUm@bGhq7EIh+{yF43z>*C|01xV(eKR- z&4c?^#ULF>h{(prr40Jh|C4;XQ+M|a?+M@w# zt)#wv00&YUW}x{*Z^K8>qmm9Sd{wWvH3*3zpNjTP@*gALSJ%0>_PNjVkpT4^R1m^5 z=50m~T`yu5Zhtzx1D^-3n`Az)b4aXzR5_$fqG&=4)=5$j2rc%p4V5PROuSu!BsDYu zPpqHPX0R4vBYuB=%OXa2g~eBHkqqqAveZY2AQ~yzMxgwf$GNep zj~#c!MVVt)Yt70dxOR!uofU=h}h)wh3sYg_4mfKp6;;W;350D*fd@ATp7 zs?O_%Z|*vV{O{0 zQw}f6G&#I=IlT4%9^SZL0Mj2pbf17^SS~E*S2&N+PS#sZsbj5jZAnhpotu1_jP)+S zHA&--?^U$xc-D32P4Ngbmh3b7P)VH;+Kkmr=StZ&b9KRa-9W;GtIoIBbG)8kR?6-w z{|M{7^Je+NrkVkga~;XrK~dnz^KNmVq+<1KdQa#uRox9%@nuuH7=pd8Mpj_fsfkxV z=a6k*66g>m;YvDVtf*5|_nZH(?zi#XR{!KTltl{fD$2^%8jZ`wm&2f`@W<$abjLHZ zsb%6+$0qbkk@r(~Z#unOzq?x@3BhTlG*z>H+V^I)0ypdY>!U>$SjbLN@y3jA_AIH> zKLt_cUoY@62Mos|tOTi7{8{iK{9!&0i7-6cvg+5o$)*OQR*+>qs8bU{RpJU4vE3A5 z0?zOt7*&_qrZC%20(q)}c#RnH@R)=2FFNmvH0wRy}Lc zt_nHGyDbU-5+HK>Ck)GKDisqE0bvcrg?Q-u$CnghcAyQSc$spp1mctJC=GDYJs<3liUhYO0 z0-ChaLf6iqMT@DxWC>!h-RnzAXX(EDF zZk;oGHi~Iw+Q;5~`jGuOP>f5A(B{*owO_PMw5-jWJMErvfYiseOuwuQ8PuigQ_QBK z%U1pBPcyGw)prSZjA~U$L8s0Nj3=F~d%-A^j8{C!mgf!yac-FNs|cn#P+88mn2XIy zYRzUCVX{%s>=L%u+0)7-E=|OykR(ECX~ny(Pfoq8iFkXi929N^=9)Bi&ma1}oKOf6 zb#v(deR;kDTkG*t)*Q2$gJ=7r*QJzBbaY+$0w4?&8Bi$>Lt^ z!Y{{wtJfkzNxgYxMuuri>mkR;wreWsn@p+cH4Rm}>$OxquD7n*_!|I=Zg~9-M;oi0 zI5rU0pehBX}^b{sqpN@xR*p+kK}<0w5|miv!x z@5{2G-*tCfxJ&zq3l?rMveJ2p+AO9rnre}3MUHqGt54?{MADybuU^roQrGhouxEF= z`Q@NfdVqW-xK@@v?14%LRbP@bu$Xm~&H3@arz2AZ`=mq?LMQ^)1#vf9@?V2sjsF_Z z1ByU8Fr-%>-aRl;1ZjWA)hbn-H-bTY(xoRegLpKu6P$HmRmz{Uvj%wx_aaL=8BM{R zOL!kLp5I1cv7NvnWPE_?-`E1{e-Qxv}Q1F3r`Hw0^-Y(LxcfkrQ<>fSwa{CKHWEaMla-zRs--vLZ@)y!eThd*n#vmC{v zh_hCV>vRE9Lv_r#g!*?|^!4_qCeq8+#E_#CvbQHKyKLT-ssN)BZq-(6(#MJayBj^Q z2?kbBJHp^}NU*^(s!-sH8aH7=JY<8Uexkq7@fdn@Wk8Q_;$)H-po0y+SLkn1eD@D4 zSo+fU_m16@%2fQoIw%5lxz@LvELx}id_oj-5pk~~H{+Te#J#<%C~BnojZwzxg;|Ek zZwMp#9slN1GxTd{L8P|*m+$snpYeT7e+kQp<{Xlxj??&2F}co9Zw#C;cv0t$TAh;@ z)7De2v*93REZF6-Yw*8&sEJeVO%l;4Hly4Yp~x>~tEp<@`n78<5O@>t#Waq0hbEz{ z^WiO2rxsOE1KSZ+L&qpc$rlaWKnlD4RtX~%1a(Xd?^!5X+a#m)%U=5lr&3^PnWwgHRaFE9tbm$-{Pd}8M3Vff zb45`D{`sEb`rEI5>R@>&J96Q(x-+!E`Svcvzl4cF6Za*jhfUuf2zrsV#ZZ(@(h@8h zUqa&W`ZFi!#nwZ?7!~BE1P*{p>g4U)y>4~x;p4L$#BPJYL6jIt$6o2&H#S&a?>|xQ|-XNG`6q~7!M zBXJHam8#p)Xx$W9fUYeU1s9=tUz? zDOUXGZp?*-pYQ3_vxrJRL}}{pltThS_w3zDPNOTC%W3b`XCzn;8#Wc?QThVP>5Vbo z&d9o|T%&1zwAyyP-=eFivVBfcXf7D=fC+K^Qr+#0YG3H2-f&nP+4bjAT~$6LKlU)E z&|EXUICfW5O00jA66|QF4Mw$C7tkZPP!ecQrl)Hx_2crakCgw9>cru3v99murTmcy zB_zPBxDO5#THdhszGB#7jYU?Bsn5pGA3jW0Q&THFxaMAYacnR?hWrP0I6!y6iOgTa zsfHd;O^upx`_7%s{k)#9Ra8B{_%@#|wFq?IM9YTTKfH3;+j~7l_Fr|mcl8np5a^!> z+2Q^@vsV*iR+|KSTeoXHH=?isHInj9yp)1Qo&40G@nsopOLx90yg_S;w(xQ1L&B2c z*u0ccTgR;eunGcTtM*3k(!5CP`u%23o3;wVA(nm$RtEK=#8btm&7A+_d~ImuT7RX& z5{MC0BnvU1XDt8^dL<-(+8OcY5gH@zr1Tk6j}0U!$@g1HEy3p>Yt;1n^;uKkvY(57H`CIRNloI?_$DO0ukceJ)7ZY-m)Nyf z*Wfn?hv7;^Y%%Wc?y>sA(RGD#zU9V7tky8?n8j$BN_mB9O(^`nsZtYV24hPs1AkM9 zK|r{{qvtiq>{hor3@pt7*u2AwCo0c*H7>9rAZ?*PdJl;5)~2T0C!l08kyIa{nJQP| z*c7J%oNS$OGPSfcI%23}aD(wuoOR5>kF}!tEf|9(+ZK#ga^D1cfJY219*lzR;H0zb z_^^yl`KNm;skUrSlq7Pyg*o^G&dMS z@7K3%V)<9DWb+C>dlwB8ibP7TFa))P(S#)>C7CN}@sIgga0V*#7e0Mg`B&r3EEGg8FolI%a!|o)iv!MQeZ%PGCWCA&{_!cfFsEVj z?5?!F77*-`giN5#mMv?*<|Jr{xlb|$>Rc8eh(!ECA_rnaN2=(IUpZyKybWywBkuld z3_m*CQ2C`F-olz#CkYc)D9CN$8qDkYOe1ogn`5VmC++9n*SL$z31$Bg^ku@#zcZQ9 zhSakf)GlQM9eqP`!|Q|oE0hf%0vW}Z+jZvH6z+HI@oo`rgmEl$w@ma2FCRH%&t%)~ z|8XlPzIJf^%eDw2!)_efh)c`G`o$S$jm5{Gz(%Z&uz&M~O)ne5!xg}FvaW)3owB3f zGLl$<0KA8xP$qr$Jw|SRj>{G?b$`At|A5Lwni0extoUtvVlE$;C;9PM>73fNWf4T_Eg)q6e?{-0&0bdY7hw`ad;zXy5}Xd9hb^X6-jd3_k~ zJEn4(ALm?lB1LbWOEiM`E*JH4ZCVrjwadcAY?s>^$M=fgq*hg|KBMv8`we?;Qo9Nu z=?{0p@$>Pk?aofW*I&M)9?XmPZ*s+AL2qCC8-1J8WG~qC@X(4szEkA}fQnk&;`@HS znT$Ex+;?L<_KMfIl)CBCIs>==i73Yx_B^{7^d#`?*#QV)RwCad?xWYg8`qtx1KcKa z_`n!N7?VD=tx3B_bil$ntr%ua=)!9B)Frb0g}Hw~>)ySak~8`;cD!R$5N)J*p+R=W z-qCNAFi7VIlN6JuXTyS&$Wv77K@4WhLg-ixX zH|A3Hd^CO9v@9kPIb#&tv%Sr);$1#G>vk7rXHnK6Jj#fgbW<=0ZkuWfupMf8Y+?Qb zy7{Ldg2ktg0{`}=9B5v*sy@{};M(f$-MdeAcOUnBd4qu2;VVwADxBVQ)wV42(f<+! z(9`t9_mtVp;#dFrSMfDftTQ$(Yv5}+c(7PAscnOJqW)Pcw3uU8S+xzov9VyBp3=mM zs?}HQ7XZ?CYF${yN+`3|%z4tK%kY4fUZMwBPGGR;4=w&ha0xMXtQf|AmwQ#KCss0h zO{Ep>&0d$Riu**Qox6Ec)*lTafxxHBl<(eO-%VtqZNpZfs$A6plW*^LYz$w3(9Eec zFg#8)hKPmnrPs9e=;AnjyzCccb#p{fzp>EHM|gJ2p!gYqN>XRIy9YBE6mEZ_5s_l( z>((keugGJ3LoHW|eOJgh9x`L(6H+~cZY8r*1Hc5}TNd>ZUwHV=TQ#sJpce``hfEe0 zpP&A5Ocy>HKpx$@Y}ex!c6QmyYsu_f{o8xrC1?qt{g!u|NH;{aOg93#7(@qcpVI18 z`^utfdM9t6+B;)Vygq=@`fB5i1iS++{ezV{Ztl(jWs08zUKjQmqAvCU7!a;^v4aF% zwIKpkzZ0`)9VnLbc3TFUo%!*qqZnWm{*0_#BB;0O@xWz2t!-iW%>TGe;A2*W zTQ(WaWlDONEOq1?0t#wO3oD`p(C|1e%9%@tni)-iI6oqrkv4Z-VKV)H+rD^Jgyvth zJOezVdbzdbm#8jt`{}!BkGT0O&fYzx-+CtJiO>3?g%>hKp@6S6Fz!u=+vw9Qz+Hs30-i~ zn0-yyPjBdvoZpIj8*Q-F3l|mL_U&u(b#T0exb@FcYQ{w-;sI)8$+Kr|UFR#)j`d2s z)vJ5=C+p;DBseCNqh@fN{z0hx#3H;yma?QHX$x`^7`#9WCAdiAVk*UwFBP(BCbn%2a`fg=eDqmTkCIjy%(=4KEonppsSL>6FwXmj1= zTrj92|Jfn%D68Tz9a(Wo+*gac*LniGBIBAedGapXduZppSU^w`+|v#Y_(jkzRoj1r zIVJ=Jbrsh1HwxK8uX@)QGU^f8+DyD9)CAGP^J)oQ->*mHIq^{_Pp6Af=J8SmLF61> zzj_r1mnQTVc{&j-n>lE+m^{}0Yjw6A{Qk8R-plT#H&i5h28BViuoWjewOzWHT_38a z{>NEkH0Pduux_Ky74+M&L=+JthA((#Y;=2EJ>WiE*prsU>Q?UHo|b0jANgZxWYxC2 z!ts2`PX1m#z)nv36dhdCA6<3J*2_Fp+gM9m!FeVJD)v{Z?!ip(AISW>OS^=m_v5PW zPKiV3?##1EH#6vRWLzijhR4P;@TXnVra^axo`61bFG`TbE%g<>X@sbThy|;m)swlD zrxK<8{Q?&uoWQFG4K>2wF8|4X*`OkJPUb!EimZJ#d`JFf=V}h7|L8^_Pzh_rdxdbz zdJpexXx8hFeG^+EGOdOG9q*jUVe*y*M}G07ftt(Y$qK?#DNk8gdMcNCdZy%6@SI0| zH@>+hrjDdnxQsh#-PpWFLVSF7&$dHb9X@^fG&@ELW}V5)2-m&$mPuXMT>$n(k9b+| z<}`6)H}(gK1g)%5Bl84_No0Vlt^gJPfEmNQ-+s|r)6WTj>;Ln^)|7!FafKO5^D3*y zsI~jTY1=UTIxeVd6rwFd&_yN+GFJ$e`IY3v^Ry0qFwOYO&C`z?&n*rnO7Gz1N;72B zJ-EEP6+Nw?_dCGCIBD3H=Y8)|2F%*=5H#tWh1j4|;{7HUb<~T#d*b6)@(CpgQ&6|- z@xF-`ztaaz%q+lgF-LW)JNF4ntoYnU=D{ukP3!6{IoG~)C^<=Rp^|cl8RDJW9!|;r z0|!LXRV9j?d80vELZJO{T3;}=)fT!+3c={9xk0x~2%(|n{QySdGUQA$b|f~H7ZSgG zCiF@uedLpUht5z&CxK)%$L-_v=EqrL)dV4U7c(KzS|eyku|jp#T2k(?Qr$t^NBPUk zs4vp>#uVu`+S_C6!SJc*uSYae8{8SNSQs_M?^Di0Ck^XOM*+nKM9);R+dr{g>9w-2 zqZedcjjPnB2=xD_2z>Ggy8SL+x0Xwlx4dfFaUZBX5f7{j7Dso`(JLCy=Ix~{-uU9uhEV*@e^Mt z5lCzh#cU5ro5ey2k=SYuaT6yLqnMZ47^szO$z@`0N&ozu$Z|QLpFQibHI{S{^}I3$ zy5f@rFiCE1bS2?aj0iM^9Y+y{%t|BWQh@EET`MBmKN_TeX9HT@ z)YvumRdfOfv}yQEeiy5pDKz|GC$XWiohK-eI zkVS{wn$|?53Qshq_fBvpj~UuV?j)mP{>uc{2```Zy}VR{Py7R>_Zl$ZZ1-Bz6lUOT zvKJKmIqLowP9h?q^6vAtxu+rvW+WqS>k0if{&83Fef%GoL$4bx9w+63%Xg{Tg4`f7 zj^I+>C3Ab6{}bmrdm|~LA=5nN%O4VXRSp|<+i=YH;4gm#79DPX`|k5}+w7-$L;)}* zG6*LW%BpED#GY7Q6bfTPL?6-dGR4(Wz>JVm z8Z)>cY1e;)Dl65u)JPwrXDr%I#P*eeuX#ojg_nZ%`jr zGu*3h+ij#~gn2!fH3CS}kp0Bn<;t(c5$mEi=!@w_wHqSPhUDTx*Lavpo#B7JIA=O- z03lhncDZ74GY{T7`(Z{iSo)d@9*Ve(pjAiDX!$SkX51nggJO5B1Q-2-^;-W_~f8dwhFu&cli7Nl}x3+ z{v+3wJ!pL&KYw16(MQOGt%oR&hwgW<(2khqeJhTj7RG#-8;tu+nBIj=8=pEmdE?v0 zf4e$yCm0Y9UKf%1+#72lTN!tf&U>e1HVZM`I#fH8>bHT!DN0lHfybp5|Fu4CFIn$8 zxG4T_)gC@IQKX>SBesoAn^%?}bU4(~CQ;mA4Mx?Y&?8&Or zpMVnX+?qCyc1pQoRBfxm;`H71+XbrI+`L}4?6>y3;!od(@C*-{ zK+1~*OyPkp-~bIw+?^?W*9pr=NR-^Sl^hFCLn}OSW1mm0pyUO+QIb1v^T1ivg-Bmza(ZfdPZv)GCH1Lc4khT_mmi?OMI%GM-g5`<14@T0+Tau&UY9@1(i@j{<^16~!9684in30t+rl2Bq6cSP>fpgdL zH>eu)>=`-Xcah$*y)Lt6&A{XEfK(am74Qz|TkYUWr=Y_@C*2YVCS_C(^q(q=GJERAR>h_9JQK5yy~Jg2yO0&z5csJOl$% zP1_Iq7qL`0a(x#M(hk{hxV5r*)21r$-xBMX9iq{(D}pQjCYkuTsjNLN%ZUSekmh z%X(`4`t|GGx38zqU`xvxmL3MHIz}Xor@me*?t~uc9KWZH1>GrloljkXrI6dum zmMcrQqJ5y=J7!FY9L~zl4xPv{&puK(<pf~u=R8 z`1EXTTnF=fhE7_`sX=F09>x6thm~Fm*ZWcSnWm6DqV5GIBq`6RTb)kZq^@gUc>?(w zGp}q%&C=K$ULf^PIG?N6nW5`UY|CS8Dot*~;h3qhiKZ7wN=#=%Isw+OZWHQ`ok}M% zRCF{p&92{*nJ%$tyrbj%KTt0k)@|;q7?@l%N?uH~&`QX%7RdRF!{aP}q3Po<|4LTd zc^~1zc5jV)xjnaZRr*}llU4{NeDe44`2p~1DVAYlM)R6cYMFNkHAFSNZ>J$?KKxjk zo1eeQqyX3?*Ea9ds9U#g=1q1q=riEwhw3M` z6aIuv#NiweNAPpW0tueXJAd)Y=-DXe`KyX32kIFz`Wgx#zURBEkuNspJ1*7}PDBDH z7M+1>ppCcW|$=!bL4ZGNNPVHsBS=TUgQ78k6K7{He!Yn7Y~JJP_XojUcDUVbp*TG`gj zl%y>$!Sm|T#)>ZE4x7a<(IM5QA#xgZ4B}ax9rlgIL?+IT(vZFnCR*|wC(>1U6a={( z*?i*PY+rfz;|+E0y@I162j=N3SA|fp=xLevs^Lq{q$)_hEXWa1^@J|Vp=-iMnB{5Q zo!;~ALIZAt95LqyPj+2h5a~ZVk-_ zP8NO6e91*c5yI=e8izSF$x6oSsWc%X*P{fbeFDbGp*98@^rtOQgFDDUQle+_VB!pP zE|C*F#W$>332dYep)91{Y==zYq zugdQ-gg=5ka3?Z=4xVXEAVwo+f^)*pzL?9gZC?c_jE#*&u0P|*Yt$XxAb!C#J74^zbiQn zv;a}7lhgLRd?p@tCv!j3EUv@J;lRy&|JeK)LiD>4d-0Y}4u<`>VEjuKbhqqAP3w5Qsrz!~W!n(GW$3{}5xhQj% zryf1p_^=UgPM<0)rRiOL1^G$VH*I$%(bK?Ocw0|%zsNm_j$4u9~ zJvALBo*z8Xp~dMKsm=AN&Cg#SI{Q2d_B(ybo9}|g0U6ETsnKj`sQV$YCyHcM*5VVTZqmGT+6?8vyw6P!~rJ8;$h>(oa@M)EAt71p z33|4qN7Pt!-(r1YK=+D#1eGLWWCOiEGUQn0%KgLTb7<0lxfaKXny&-)Wbe~o5U5Fh zzuJABZKb04`RV#dLGYji9C~f2Ipd!e1T8jK?qe`tK|kmW(Et`G2l&NYWuD{SJ)hxI z5%{nlUsSav5DpUO!T@pbY51|DjW|;q|3SR?{9qjA#>!?ao3T+cQ?Uyb$f&~fXjSJD z$3p~}*j4-?E5yaI2|w0?HCqxA*>fFx0HzR7vbT#AX7fv$a;U)96spyFBCGAjx&a`Y zeX}dkSbQiiHz!H<>stLQn;IyH;!+4m2Vr%vp3X0u+~1oFUYX3$;=kk6Z_E_c#A7{G zQbo28lk+5dN=<4|tXlf=2SP!q#Ki_w8TLm=;~r4?a4rX-zbEgP=0h ztsSnd*X9|gASrqxwmhVnUOOKY7mMQxj3#=on(7XAftz^*b z(XXF0g{X;OTe&=FS9f9T_Z)43M^lsJheY4sG#}Y4sggJt#bf8S;!i?m<>2;lO;ku| z=pecd6eZzAA=qT1Dv%gws(lMY&}Lv$HM;2Q++!$Sed_F6JagdAs)-$EJ#ZNPI)i}c zf}<%0uS_@9oJNl76cABT$^W13uKb_swEcgKX_?2wl$M90jUk0lWHQUL}s#u zgE$>o!c;R}(dG!@SXvYb*^YgwXDT}>%RyB3z3j3*@0-)>`#*gB;Pv60^ZDH8zVGXO zE%*Dnt}A-OTi`)nf&aYuO2DmK`{Ax)ARO2 zE$1G|RB8MAR9#tqdcc^z(rC-3f9VM~iFUAJK1E`*}9J^yp?_Y=i zV&|_NRJWIFwvtYJE_zhqy-{L)i2wCduh#E@I?fLDLha(DO>qMKTyAW%VQNWO zWY~ne^ClQQ2}x}(y-ksK=;yiy@6BJ zu0NNF7e6yeOSLRvCBDqno}a36MY3V&H#MrkJ<&C*uBc_a;cd2)ThvJEIWSHtK9q}w-$G) z`I#>5ks^MfMJJ(fYy3OT0bUo{uHLsbN z;J{38Y;44FhYW%_m2L$iku|+kfA2LVUpdlVCcF1(LZhRTy?`&Yy5q>T6Mfh7=$gsL z;@iXRlem94_*RZLyH;7jGt0)QPAY1^GBvwCUKOe=k&0&P%jfTcbt;?rLN2D<$<0CE zO=$h4LMl$#HETTg)8Fe|raD5Mz7BMZUz5xnyH^+c56)Af@S$|}-+bN9oz9HERb4w7 z#d~y*T4M4_LkmV@NBipK{lu83{B8(*DKx+;99z5N)EyKFv4K@Cmb#><#d9bU^{>sC z0#uBrM@R6<;f8a!tx!?7skti~F=SK+X9%iw$H=%fW(#Ufel*ZLT5I>pe@&!$+E$}B zQyGJPzAv;+=YfgaJ~LajXFdsy{3C2A!DVi03;HdX{$b*#lh;<2X6JpR68ImP6l0gG zIi-VfxPYt_xIHOri(B)B1E)`)X4=M&qg5IbGPEGG)6KJ`zqNJ}nFRix9 zAFa_3Q#iLD34(W#dy{QeC9=oa!a%j-?ir7sSP@0bgC$}=P*im(ZVe-EGu-&fhqtxD zVT00>;_3>0=5L5=s=e9ZdWhLX&y;cdnWBm+f2pK${=z$$k6!9lCZ=)!x(;2^> zOMb#T7JhtJ)f%A#A@2{fFz2PbYn|&J#)~gB)RISro^hd}l5w=eHvcW!)7&aWgbyP{ zCy*denNe>ZQ5Weyfju+RIzM~X&dzI@07b=Y7p|lzU&rqIe&SE{K7h|{6e~wDR6yJ- zA3~BoghL{G@^gTj+Py#t#9m6pJ^KS&=f*SX=Qg^x2dbi-kDG@BRjOoL9b=GevRW5B z$(Ejre7^MUGKmVFctBHJeEj*`P47(n9~^{9_N4Ad-@$IoODWEThtac3igrt_H%=Z$^je=l}QhCPr@g}fRhAf}+)hZ)yV zF^)G<8(T(AdZBNno9WJn(*EdTp1<#^FamVt2s1S~CZEob0e7nZ`By$^!=tMshxsXn z=1UQ*)=kxvWt3S}VrEOY<-67%I82>cZMXWqh!W@mpC|L_PdfEdr-Z2ah{d_Kms0DZe8b&t0Hwa&a4 zAgUj~q^J_-lU(bfIZ)6HVSrYu4}m>db~b6R4W(m~lK2z3E#nSalB$(FmOU(}Ia*zF zMTC}$W<(v;`-JGZ`CwSp*zrAgB?t;tMbDTm2DS%9&$LUaV*e?5&rCL7tT*GrlgVgW zLg{T{FYG(9_6n6&2lW<2T=bLSOBQn4K%l>CF61kA9A!kn&WM;xZH+}QfHZaUt(tCE%mKzQ0!VcZSbE$aKA_; z%i=qSB9vv5^M>^XbNZ5q9Ah01u{gE6a2gTWmy|09d#efuHzR)9Gn)Z^j=}27C}zl_ zSTOFu{^pRb@S)~IVCpqn)2SQ|0D;YHJU=xOJfdO!e>hLNna9aefaN zU5q)zth-(c$d?l!v^=h}l}4tk*`-WY_3J;vL_p5CR_vZaVB4&M!{<97!JFDgKJ#-l zWwS%31zzqwW9GmTP8)d4!I%sevCE@x!&F_Z=P+(dL^4$ev)UR5?LUlL;>X2Hhx$l#&BK@!n<0XyosJ%L8=v4pc$WSkQ%o+BTIyj9R}#j~kZ z6zT!6w{V{tsCBm2a0y+%s_LG?R-9Y(d{J{Mn6kDmzZdvG`q*k+1GSz<2qHZEZy!~C z5oiQ==n*|k z`clg54LAeg_R0CmD0@``(pogi<91#Z(u!l6QbkM|nM9>MWMCFgGSbs6L9KA_i48p# zZ-jBk<#a!o8FVY`yCc~RvHje9sqH4deXyRS-v*b1Q9g6-WljA3d$qoGKWx76Mht|( zVw4Ap2G7=O*adxD4;$?HbnSg@ZAZCM)ghSu3GZd8YHMk- zL0&WNtsbg2DDo4g;qHP|DE3uBwK)F|KTu8k_VANO>3zAnab^?Hz4`N%M#tQN5GSAM zrtB{-1(Y!3{RPv5(W_Cb(Y{|#}=-oO7U7a%y$*A zE8IK7Y*-Fr#VEXQSgA2mPQD}{Gyyug=oW8@R>79Efpc$y7CUR#HJrY>hCnj}Fc5L_ zSPn-S#uSZ(x}WOAhC;6VaI(+r_KJ!MTB@$}L)bhKtuxkxc_Rj84g8FKXrYJs@~lCk z3z5u(s&93LB1Ewgm>uWZ*uZ9aYF$PZ8Soy@LYB4ybhdjE9A?=oYa;AtXQk+U7ZMAn z-`@7V3rI28$sZG8!f`MWMceo?*-T3uP|-MomPid(!Hk>Z&Wg~mb{qkUv1t(&q`633 zi_(|NmcZ(o{-vNbFxt!!uZNyLgUk`tzGrEAeQSJgsbEv@QdYJ`*=!nTu73!HG7L1w z-ji8Me`lZ%;D7q|SL(QAfE;zC=!`)-?vdpayyXg?QKHfXTlA|He8{jDcHA)ySunO^rJK0SEFGlA&i<{DOO8 z8WX8%!3d|{Yelpvo%*+FjVl)3G%x{;)(-pcTyJ0BNPCc>ajqD})Zr%BEC=(Ve(NG) zekg*(-UI2a*?W4x1VO^J>sr+8uX|Ch5Kyr#%9DoAyzH!Jp+T0rmtVN_Rkx@AwWR9(ePjL zUYp@`mtpgnb|gVDPVV5%!xD|@{xl-t_jWr;G?q5yHrZ7ehADF%&aR48g-1_;h10GMABw zV)z5HC2(XLrtRZRw2VZ=UJ>1+>0(kFlkXi}txGIo5b})@iK=CuIh?Vce9Cuz2^O>x z29Xj%zZ*RJQ$SBLfVRq~FI6zmnrffvG4$~tI)P{Pi}I#K5=qV_K+#Argm#?C+6fYL z^OhTdUU_=2fA+N(grvkm#XaJyyzIbTyps;L%v(6gzA7O!}{lv34>{nPxl*B)Y3QtJN zkpbuGnafE$)F@!laJU<@J0=B8YywrKcejb4i5nBeq4kRk%_1ETiuX+(Bf=+Ml7xJMp`+ktD0pIjD#y;eAfhPc$P zeQ^VGK=6)$EYurW5n}ZKN=^*^8*7znqRIp8I#VYr9FLPZ1YI`BAv=73-vDuN#MG5R zqA9_gWHyZdQ%Y1xE>G&Exlj#?h~1V)sK~=eVX_YY8BnoLwB#|}5X~0upf!;t4ha@^ zZ|-s)Qb>@rR^*SqL!O|YxQA^5-H>(rswmNK=3ZZ4aatGVk#Tl$&mUO=D!UAAE#ef(adGFRGppCgw!k`v5lCp!X!1;8$6}J7D2J(SQ z)h;%?;ir0H_~Os=uPKpPiwS_9RZ+w#F2PW4suhisAHJW3S`M3_0UXI^z-7u)TY=A1 z%YBzdO?VSwnx0G&>_Egqa%wc@HBmNbCT9HPC~m72$}<*!AIhIF2hAfC`nM!9|F{Wz2sTXNLS+a#1Ie%l`+rp4+(P$dhZXr)wZH6h+fD L_)pSai!1*F%rN`. + Some of the results of this diagnostics can also be rproduced utilizing + python diagnostics: + Portrait plot: :ref:`recipe_portrait` + Monitoring: :ref:`recipe_monitor` Overview -------- diff --git a/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst b/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst deleted file mode 100644 index 8f3f7f173d..0000000000 --- a/doc/sphinx/source/recipes/recipe_perfmetrics_python.rst +++ /dev/null @@ -1,146 +0,0 @@ -.. _recipe_perfmetrics_python: - -Performance metrics for essential climate parameters in python -============================================================== - -.. note:: - - This recipe uses python diagnostics to reproduce parts of the evaluation - done in the - :ref:`original recipe based on NCL diagnostics `: - It aims for a complete replacement of all involved NCL diagnostics. So - far, only portrait plots (including performance metrics) are supported. - -Overview --------- - -The goal is to create a standard recipe for the calculation of performance metrics to quantify the ability of the models to reproduce the climatological mean annual cycle for selected "Essential Climate Variables" (ECVs) plus some additional corresponding diagnostics and plots to better understand and interpret the results. - -The recipe can be used to calculate performance metrics at different vertical levels (e.g., 5, 30, 200, 850 hPa as in `Gleckler et al. (2008) `_) and in different regions. As an additional reference, we consider `Righi et al. (2015) `_. -Brief description of the diagnostic. - - -Available recipes and diagnostics ---------------------------------- - -Recipes are stored in esmvaltool/recipes/ - - * recipe_perfmetrics_python.yml - * recipe_perfmetrics_CMIP5_python.yml - -Diagnostics are stored in esmvaltool/diag_scripts/perfmetrics/ - - * portrait_plot.py: Plot metrics for any variable for multiple datasets and - up to four references. - - -User settings in recipe ------------------------ - -#. Script perfmetrics/portrait_plot.py - - This plot expects a scalar value in each input file and at most one input - file for each subset of metadata that belongs to a cell or part of cell in - the figure. - By default cells are plotted for combinations of `short_name`, - `dataset`, `project` and `split`. - Where `split` is an optional extra_facet for variables. - However, all this can be customized using the `x_by`, - `y_by`, `group_by` and `split_by` script settings. - For a complete and detailed list of settings see the - :ref:`API documentation `. - While this allows very flexible use for any kind of data, there are some - limitations as well: The grouping (separated - plots in the figure) and normalization is always applied along the x-axis. - With default settings this means normalizing all metrics for each variable - and grouping all datasets by project. - - To plot distance metrics like RMSE, pearson R, bias etc. the - :func:`distance_metrics ` preprocessor or - custom diagnostics can be used. - - - -Variables ---------- - -.. note:: - - The recipe generally works for any variable that is preprocessed correctly. - To use different preprocessors or reference datasets it could be useful - to create different variable groups and link them with the same extra_facet - like `variable_name` See recipe for examples. Below listed are the variables - needed to produce the example figures. - - -#. recipe_perfmetrics_CMIP5.yml - - * clt (atmos, monthly mean, longitude latitude time) - * hus (atmos, monthly mean, longitude latitude lev time) - * od550aer, od870aer, od550abs, od550lt1aer (aero, monthly mean, longitude latitude time) - * pr (atmos, monthly mean, longitude latitude time) - * rlut, rlutcs, rsut, rsutcs (atmos, monthly mean, longitude latitude time) - * sm (land, monthly mean, longitude latitude time) - * ta (atmos, monthly mean, longitude latitude lev time) - * tas (atmos, monthly mean, longitude latitude time) - * toz (atmos, monthly mean, longitude latitude time) - * ts (atmos, monthly mean, longitude latitude time) - * ua (atmos, monthly mean, longitude latitude lev time) - * va (atmos, monthly mean, longitude latitude lev time) - * zg (atmos, monthly mean, longitude latitude lev time) - - -Observations and reformat scripts ---------------------------------- - -*Note: (1) obs4MIPs data can be used directly without any preprocessing; -(2) see headers of reformat scripts for non-obs4MIPs data for download -instructions.* - -The following list shows the currently used observational data sets for this recipe with their variable names and the reference to their respective reformat scripts in parentheses. Please note that obs4MIPs data can be used directly without any reformatting. For non-obs4MIPs data use `esmvaltool data info DATASET` or see headers of cmorization scripts (in `/esmvaltool/cmorizers/data/formatters/datasets/ -`_) for downloading and processing instructions. - -#. recipe_perfmetrics_CMIP5.yml - - * AIRS (hus - obs4MIPs) - * CERES-EBAF (rlut, rlutcs, rsut, rsutcs - obs4MIPs) - * ERA-Interim (tas, ta, ua, va, zg, hus - esmvaltool/cmorizers/data/formatters/datasets/era-interim.py) - * ESACCI-AEROSOL (od550aer, od870aer, od550abs, od550lt1aer - esmvaltool/cmorizers/data/formatters/datasets/esacci-aerosol.ncl) - * ESACCI-CLOUD (clt - esmvaltool/cmorizers/data/formatters/datasets/esacci-cloud.ncl) - * ESACCI-OZONE (toz - esmvaltool/cmorizers/data/formatters/datasets/esacci-ozone.ncl) - * ESACCI-SOILMOISTURE (sm - esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.ncl) - * ESACCI-SST (ts - esmvaltool/ucmorizers/data/formatters/datasets/esacci-sst.py) - * GPCP-SG (pr - obs4MIPs) - * HadISST (ts - esmvaltool/cmorizers/data/formatters/datasets/hadisst.ncl) - * MODIS (od550aer - esmvaltool/cmorizers/data/formatters/datasets/modis.ncl) - * NCEP-NCAR-R1 (tas, ta, ua, va, zg - esmvaltool/cmorizers/data/formatters/datasets/ncep_ncar_r1.py) - * NIWA-BS (toz - esmvaltool/cmorizers/data/formatters/datasets/niwa_bs.ncl) - * PATMOS-x (clt - esmvaltool/cmorizers/data/formatters/datasets/patmos_x.ncl) - - -References ----------- - - -* Gleckler, P. J., K. E. Taylor, and C. Doutriaux, Performance metrics for climate models, J. - Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). - -* Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., - and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, - Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). - - -Example plots -------------- - -.. _fig_perfmetrics_python_portrait_plot: - -.. figure:: /recipes/figures/perfmetrics/perfmetrics_fig_5_python.png - :width: 90% - :align: center - - - Relative space-time root-mean-square deviation (RMSD) calculated from the climatological - seasonal cycle of CMIP5 simulations. A relative performance is displayed, with blue shading - indicating better and red shading indicating worse performance than the median of all model results. - A diagonal split of a grid square shows the relative error with respect to the reference data set diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst new file mode 100644 index 0000000000..cdade3e8e0 --- /dev/null +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -0,0 +1,107 @@ +.. _recipe_portrait: + +Portrait plot +============= + + +Overview +-------- +Portrait plots are a flexible way to visualize performance metrics for multiple +datasets and up to four references. In this recipe the normalized Root Mean +Squared Deviation (RMSD) of global mean seasonal climatologies is shown for a +selection of CMIP models is plotted. For each variable up to two observation +based datasets are used as reference (see :ref:`_datasets` for complete list). + + +Available recipes and diagnostics +--------------------------------- + +Recipes are stored in esmvaltool/recipes/ + + * recipe_portrait_CMIP.yml + +Diagnostics are stored in esmvaltool/diag_scripts/perfmetrics/ + + * portrait_plot.py: Plot metrics for any variable for multiple datasets and + up to four references. + + +User settings in recipe +----------------------- + +#. Script perfmetrics/portrait_plot.py + + This plot expects a scalar value in each input file and at most one input + file for each subset of metadata that belongs to a cell or part of cell in + the figure. + By default cells are plotted for combinations of `short_name`, + `dataset`, `project` and `split`. + Where `split` is an optional extra_facet for variables. + However, all this can be customized using the `x_by`, + `y_by`, `group_by` and `split_by` script settings. + For a complete and detailed list of settings see the + :ref:`API documentation `. + While this allows very flexible use for any kind of data, there are some + limitations as well: The grouping (separated + plots in the figure) and normalization is always applied along the x-axis. + With default settings this means normalizing all metrics for each variable + and grouping all datasets by project. + + To plot distance metrics like RMSE, pearson R, bias etc. the + :func:`distance_metrics ` preprocessor or + custom diagnostics can be used. + + + +.. _datasets: + +Variables and References +--------- + +.. note:: + + The recipe generally works for any variable that is preprocessed correctly. + To use different preprocessors or reference datasets it could be useful + to create different variable groups and link them with the same extra_facet + like `variable_name` See recipe for examples. Below listed are the variables + used to produce the example figure. + + +The following list shows which observational dataset is used as reference for +each variable in this recipe. All variables are atmospheric monthly means. +For 3D variables the selected pressure level is given in parentheses. + +* clt (Ref1: ESACCI-CLOUD, Ref2: PATMOS-x) +* pr (Ref1: GPCP-V2.2) +* rlut, rsut (Ref1: CERES-EBAF) +* tas (Ref1: ERA-Interim, Ref2: NCEP-NCAR-R1) +* ts (Ref1: ESACCI-SST, Ref2: HadISST) +* ua (200 hPa, Ref1: ERA-Interim, Ref2: NCEP-NCAR-R1) +* zg (500 hPa, Ref1: ERA-Interim, Ref2: NCEP-NCAR-R1) + + +References +---------- + +* Gleckler, P. J., K. E. Taylor, and C. Doutriaux, Performance metrics for climate models, J. + Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). + +* Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., + and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, + Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). + + +Example plots +------------- + +.. _fig_portrait_plot: + +.. figure:: /recipes/figures/portrait/portrait_plot.png + :width: 90% + :align: center + + + Relative space-time root-mean-square deviation (RMSD) calculated from the climatological + seasonal cycle of CMIP5 and CMIP6 simulations. A relative performance is displayed, with blue shading + indicating better and red shading indicating worse performance than the median of all model results. + A diagonal split of a grid square shows the relative error with respect to the reference data set From 5e337b39d367eb11776a655004cac01f6c517f99 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Nov 2024 14:41:03 +0100 Subject: [PATCH 38/56] fix rst error --- doc/sphinx/source/recipes/recipe_portrait.rst | 64 ++++++++----------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index cdade3e8e0..b803e2d4de 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -7,56 +7,44 @@ Portrait plot Overview -------- Portrait plots are a flexible way to visualize performance metrics for multiple -datasets and up to four references. In this recipe the normalized Root Mean -Squared Deviation (RMSD) of global mean seasonal climatologies is shown for a -selection of CMIP models is plotted. For each variable up to two observation -based datasets are used as reference (see :ref:`_datasets` for complete list). +datasets and up to four references. In this recipe `recipe_portrait_CMIP.yml` +the normalized Root Mean Squared Deviation (RMSD) of global mean seasonal +climatologies is shown for a selection of CMIP models is plotted. +For each variable up to two observation based datasets are used as reference. +See :ref:`variables` for complete list of references. -Available recipes and diagnostics ---------------------------------- - -Recipes are stored in esmvaltool/recipes/ - - * recipe_portrait_CMIP.yml - -Diagnostics are stored in esmvaltool/diag_scripts/perfmetrics/ - - * portrait_plot.py: Plot metrics for any variable for multiple datasets and - up to four references. User settings in recipe ----------------------- -#. Script perfmetrics/portrait_plot.py - - This plot expects a scalar value in each input file and at most one input - file for each subset of metadata that belongs to a cell or part of cell in - the figure. - By default cells are plotted for combinations of `short_name`, - `dataset`, `project` and `split`. - Where `split` is an optional extra_facet for variables. - However, all this can be customized using the `x_by`, - `y_by`, `group_by` and `split_by` script settings. - For a complete and detailed list of settings see the - :ref:`API documentation `. - While this allows very flexible use for any kind of data, there are some - limitations as well: The grouping (separated - plots in the figure) and normalization is always applied along the x-axis. - With default settings this means normalizing all metrics for each variable - and grouping all datasets by project. +This plot expects a scalar value in each input file and at most one input +file for each subset of metadata that belongs to a cell or part of cell in +the figure. +By default cells are plotted for combinations of `short_name`, +`dataset`, `project` and `split`. +Where `split` is an optional extra_facet for variables. +However, all this can be customized using the `x_by`, +`y_by`, `group_by` and `split_by` script settings. +For a complete and detailed list of settings see the +:ref:`API documentation `. +While this allows very flexible use for any kind of data, there are some +limitations as well: The grouping (separated +plots in the figure) and normalization is always applied along the x-axis. +With default settings this means normalizing all metrics for each variable +and grouping all datasets by project. - To plot distance metrics like RMSE, pearson R, bias etc. the - :func:`distance_metrics ` preprocessor or - custom diagnostics can be used. +To plot distance metrics like RMSE, pearson R, bias etc. the +:func:`distance_metrics ` preprocessor or +custom diagnostics can be used. -.. _datasets: +.. _variables: -Variables and References ---------- +Variables and Datasets +------------------------ .. note:: From bf61ccf30ea2adc302391cf93581b0213d84a5e9 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Wed, 20 Nov 2024 16:50:29 +0100 Subject: [PATCH 39/56] set msissing facets to "unknown" --- .../diag_scripts/perfmetrics/portrait_plot.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py index 54c259baa3..8cd177be77 100644 --- a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py +++ b/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py @@ -46,7 +46,8 @@ default_split: str, optional Data labeled with this string, will be used as main rectangles. All other splits will be plotted as overlays. This can be used to choose the base - reference, while all references are labeled for the legend. + reference, while all references are labeled for the legend. If None, the + first split will be used as default. By default None. plot_legend: bool, optional If True, a legend is plotted, when multiple splits are given. @@ -180,11 +181,13 @@ def remove_reference(metas): metas.remove(meta) -def add_split_none(cfg, metas): - """List of metadata with split=None if no split is given.""" +def add_missing_facets(cfg, metas): + """Ensure that all facets are present in metadata.""" for meta in metas: - if cfg["split_by"] not in meta: - meta[cfg["split_by"]] = None + facet_config = ["x_by", "y_by", "group_by", "split_by"] + facets = [cfg[key] for key in facet_config] + for facet in facets: + meta.setdefault(facet, "unknown") def open_file(metadata, **selection): @@ -231,10 +234,8 @@ def load_data(cfg, metas): for coord_tuple in itertools.product(*coords.values()): selection = dict(zip(coords.keys(), coord_tuple)) data['var'].loc[selection] = open_file(metas, **selection) - if None in data.coords[cfg["split_by"]].values: - cfg.setdefault("default_split", None) - else: - cfg.setdefault("default_split", data.coords[cfg["split_by"]].values[0]) + if cfg["default_split"] is None: + cfg["default_split"] = data.coords[cfg["split_by"]].values[0] log.debug("using %s as default split", cfg["default_split"]) log.debug("Loaded Data:") log.debug(data) @@ -539,7 +540,7 @@ def main(cfg): set_defaults(cfg) metas = list(cfg["input_data"].values()) remove_reference(metas) - add_split_none(cfg, metas) + add_missing_facets(cfg, metas) dataset = load_data(cfg, metas) dataset = sort_data(cfg, dataset) if cfg["normalize"] is not None: From 78a81464e2733d3f29402f103bb9552ef5a10aca Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 11:42:42 +0100 Subject: [PATCH 40/56] seperate portrait from performance metrics folder/docs --- .../esmvaltool.diag_scripts.perfmetrics.rst | 20 ------------------- .../portrait_plot.rst | 6 ------ .../esmvaltool.diag_scripts.portrait_plot.rst | 7 +++++++ doc/sphinx/source/api/esmvaltool.rst | 2 +- .../{perfmetrics => }/portrait_plot.py | 0 esmvaltool/recipes/recipe_portrait_CMIP.yml | 2 +- 6 files changed, 9 insertions(+), 28 deletions(-) delete mode 100644 doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst delete mode 100644 doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst create mode 100644 doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst rename esmvaltool/diag_scripts/{perfmetrics => }/portrait_plot.py (100%) diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst deleted file mode 100644 index 7dbae3e390..0000000000 --- a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _api.esmvaltool.diag_scripts.perfmetrics: - -Performance Metrics -=================== - -This module contains various reusable diagnostics and plot scripts. - - -Examples --------- - -* :ref:`recipe_portrait ` - - -Diagnostic scripts ------------------- -.. toctree:: - :maxdepth: 1 - - esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst deleted file mode 100644 index 3679d7015b..0000000000 --- a/doc/sphinx/source/api/esmvaltool.diag_scripts.perfmetrics/portrait_plot.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _api.esmvaltool.diag_scripts.perfmetrics.portrait_plot: - -Plot performance metrics of multiple datasets vs up to four references -====================================================================== - -.. automodule:: esmvaltool.diag_scripts.perfmetrics.portrait_plot diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst new file mode 100644 index 0000000000..60afc89ee8 --- /dev/null +++ b/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst @@ -0,0 +1,7 @@ +Plot performance metrics of multiple datasets vs up to four references +====================================================================== + +A :doc:`documented ` example recipe to use this diagnostic +is provided as `recipes/recipe_portrait_CMIP.yml`. + +.. automodule:: esmvaltool.diag_scripts.portrait_plot diff --git a/doc/sphinx/source/api/esmvaltool.rst b/doc/sphinx/source/api/esmvaltool.rst index f3553d7056..6eda5f5912 100644 --- a/doc/sphinx/source/api/esmvaltool.rst +++ b/doc/sphinx/source/api/esmvaltool.rst @@ -29,4 +29,4 @@ Diagnostic Scripts esmvaltool.diag_scripts.ocean esmvaltool.diag_scripts.psyplot_diag esmvaltool.diag_scripts.seaborn_diag - esmvaltool.diag_scripts.perfmetrics + esmvaltool.diag_scripts.portrait_plot diff --git a/esmvaltool/diag_scripts/perfmetrics/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py similarity index 100% rename from esmvaltool/diag_scripts/perfmetrics/portrait_plot.py rename to esmvaltool/diag_scripts/portrait_plot.py diff --git a/esmvaltool/recipes/recipe_portrait_CMIP.yml b/esmvaltool/recipes/recipe_portrait_CMIP.yml index 3d10d3e7df..3a507a146d 100644 --- a/esmvaltool/recipes/recipe_portrait_CMIP.yml +++ b/esmvaltool/recipes/recipe_portrait_CMIP.yml @@ -203,7 +203,7 @@ diagnostics: scripts: portrait: - script: perfmetrics/portrait_plot.py + script: portrait_plot.py x_by: dataset y_by: variable # extra_facet group_by: project From 6fe20a123b06bf122fff8db664a1c23a7c9e1e2a Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 15:28:07 +0100 Subject: [PATCH 41/56] fix rst style issues and documentation links --- .../esmvaltool.diag_scripts.portrait_plot.rst | 13 ++++++++---- doc/sphinx/source/recipes/index.rst | 2 ++ doc/sphinx/source/recipes/recipe_portrait.rst | 2 +- esmvaltool/diag_scripts/portrait_plot.py | 20 ++++++++++--------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst b/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst index 60afc89ee8..100ac704e5 100644 --- a/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst +++ b/doc/sphinx/source/api/esmvaltool.diag_scripts.portrait_plot.rst @@ -1,7 +1,12 @@ -Plot performance metrics of multiple datasets vs up to four references -====================================================================== -A :doc:`documented ` example recipe to use this diagnostic -is provided as `recipes/recipe_portrait_CMIP.yml`. +.. _api.esmvaltool.diag_scripts.portrait_plot: + +Portrait Plot +============= + +Plot performance metrics of multiple datasets vs up to four references +A :doc:`documented example recipe ` to use this +diagnostic is provided as `recipes/recipe_portrait_CMIP.yml`. .. automodule:: esmvaltool.diag_scripts.portrait_plot + :members: diff --git a/doc/sphinx/source/recipes/index.rst b/doc/sphinx/source/recipes/index.rst index a2ca9da811..037cc338c6 100644 --- a/doc/sphinx/source/recipes/index.rst +++ b/doc/sphinx/source/recipes/index.rst @@ -65,6 +65,8 @@ Atmosphere Climate metrics ^^^^^^^^^^^^^^^ +.. _climate_metrics: + .. toctree:: :maxdepth: 1 diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index b803e2d4de..6f8fccc11f 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -28,7 +28,7 @@ Where `split` is an optional extra_facet for variables. However, all this can be customized using the `x_by`, `y_by`, `group_by` and `split_by` script settings. For a complete and detailed list of settings see the -:ref:`API documentation `. +:doc:`diagnostic documentation `. While this allows very flexible use for any kind of data, there are some limitations as well: The grouping (separated plots in the figure) and normalization is always applied along the x-axis. diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index 8cd177be77..a23b765b70 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -6,13 +6,13 @@ The multi model overview heatmap might be useful for different tasks and therefore this diagnostic tries to be as flexible as possible. X and Y axis, grouping parameter and slits for each rectangle can be -configured in the recipe. All *_by parameters can be set to any metadata +configured in the recipe. All `*_by` parameters can be set to any metadata key. To split by 'reference' this key needs to be set as extra_facet in recipe. Author ------ -Lukas Ruhe (Universität Bremen, Germany) -Diego Cammarano +- Lukas Ruhe (Universität Bremen, Germany) +- Diego Cammarano Configuration parameters through recipe: ---------------------------------------- @@ -58,9 +58,10 @@ (i.e. lengths of labels, aspect ratio of the plots...). And might require manual adjustment of `x`, `y` and `size` to fit the figure layout. Keys (each optional) that will be handled are: + position: str or None, optional - Position of the legend. Can be 'right' or 'left'. Or set to None to - disable plotting the legend. By default 'right'. + Position of the legend. Can be 'right' or 'left'. + Or set to None to disable plotting the legend. By default 'right'. x_offset: float, optional Manually adjust horizontal position to save space or fix overlap. Number given in Inches. By default 0. @@ -69,6 +70,7 @@ Number given in Inches. By default 0. size: float, optional Size of the legend in Inches. By default 0.3. + plot_kwargs: dict, optional Dictionary that gets passed as kwargs to `matplotlib.pyplot.imshow()`. Colormaps will be converted to 11 discrete steps automatically. Default @@ -91,9 +93,9 @@ no triagnles are plotted for NaN values. By default 'white'. figsize: list(float), optional - [width, height] of the figure in inches. The final figure will be saved with - bbox_inches="tight", which can change the resulting aspect ratio. - By default [5, 3]. + [width, height] of the figure in inches. The final figure will be saved + with bbox_inches="tight", which can change the resulting aspect ratio. + By default [5, 3]. dpi: int, optional Dots per inch for the figure. By default 300. domain: str, optional @@ -385,7 +387,7 @@ def plot(cfg, data): """Create figure with subplots for each group and save to NetCDF. sets same color range and overlays additional references based on - the content of data (xr.DataArray) + the content of data (xr.DataArray). """ save_to_netcdf(cfg, data) From e592aa32f6168812fb585a9c2f9933132d5de4b2 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 16:41:39 +0100 Subject: [PATCH 42/56] remove preproc distance metrics call from diagnostic --- esmvaltool/diag_scripts/portrait_plot.py | 32 ------------------------ 1 file changed, 32 deletions(-) diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index a23b765b70..0cade74425 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -104,14 +104,11 @@ import itertools import logging -from pathlib import Path -import iris import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import xarray as xr -from esmvalcore import preprocessor as pp from matplotlib import patches from mpl_toolkits.axes_grid1 import ImageGrid @@ -444,35 +441,6 @@ def normalize(array, method, dims): return normalized -def apply_distance_metric(cfg, metas): - """Optionally apply preproc method. - - reference_for_metric facet required. - """ - if not cfg["distance_metric"]: - return - for y_metas in group_metadata(metas, cfg["y_by"]).values(): - try: # TODO: add select_single_metadata to shared? - reference = select_metadata(y_metas, reference_for_metric=True)[0] - except IndexError as exc: - raise IndexError("No reference found for metric.") from exc - ref_cube = iris.load_cube(reference["filename"]) - for meta in y_metas: - if meta.get("reference_for_metric", False): - continue # skip distance to itself - cube = iris.load_cube(meta["filename"]) - distance = pp.distance_metric([cube], - reference=ref_cube, - metric=cfg["distance_metric"]) - basename = f"{Path(meta['filename']).stem}" - basename += f"{cfg['distance_metric']}" - fname = get_diagnostic_filename(basename, cfg) - iris.save(distance, fname) - log.info("Distance metric saved: %s", fname) - # TODO: adjust all relevant meta data - meta["filename"] = fname - - def set_defaults(cfg): """Set default values for most important config parameters.""" cfg.setdefault("normalize", "centered_median") From 05c03ebe9c7d0826b49b6d868a6732acd1951089 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 16:44:03 +0100 Subject: [PATCH 43/56] remove old comments --- esmvaltool/diag_scripts/portrait_plot.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index 0cade74425..29480a8894 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -467,17 +467,11 @@ def set_defaults(cfg): def sort_data(cfg, dataset): """Sort the dataset along by custom or alphabetical order.""" - # TODO: decide on (default) strategies and options. - # sort alphabetically (caseinsensitive) dataset = dataset.sortby([ dataset[cfg["x_by"]].str.lower(), dataset[cfg["y_by"]].str.lower(), dataset[cfg["group_by"]].str.lower(), dataset[cfg["split_by"]].str.lower() ]) - # apply custom orders if given: - # if cfg.get("x_order"): - # dataset = dataset.reindex({cfg["x_by"]: cfg["x_order"]}) - # move MMM to begin: if cfg["x_by"] in ["alias", "dataset"]: # NOTE: not clean, but it works for many cases mm_stats = [ @@ -495,7 +489,6 @@ def sort_data(cfg, dataset): def save_to_netcdf(cfg, data): """Save the final dataset to a NetCDF file.""" - # Define the output filename for the NetCDF file basename = "portrait" fname = get_diagnostic_filename(basename, cfg, extension='nc') data.to_netcdf(fname) From 051513c1bd5f4babe4737a979f5f944ec579c14a Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 16:53:26 +0100 Subject: [PATCH 44/56] add test recipe, removed comment --- esmvaltool/recipes/recipe_portrait_CMIP.yml | 2 +- .../testing/recipe_portrait_CMIP_fast.yml | 159 ++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml diff --git a/esmvaltool/recipes/recipe_portrait_CMIP.yml b/esmvaltool/recipes/recipe_portrait_CMIP.yml index 3a507a146d..14be113022 100644 --- a/esmvaltool/recipes/recipe_portrait_CMIP.yml +++ b/esmvaltool/recipes/recipe_portrait_CMIP.yml @@ -111,7 +111,7 @@ diagnostics: variables: zg: &zg <<: *var_default - short_name: zg # find cmor table for custom var name + short_name: zg variable: zg500 preprocessor: pp500 additional_datasets: diff --git a/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml b/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml new file mode 100644 index 0000000000..88104ecb2d --- /dev/null +++ b/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml @@ -0,0 +1,159 @@ +# ESMValTool +# +--- +documentation: + title: Performance metrics plots. + description: > + Test recipe for the performance comparison of CMIP simulations to a reference dataset. + authors: + - lindenlaub_lukas + maintainer: + - lindenlaub_lukas + references: + - eyring21ipcc + - gleckler08jgr + +cmip5: &CMIP5 + project: CMIP5 + ensemble: r1i1p1 + +datasets: + # cmip5 + - {<<: *CMIP5, dataset: ACCESS1-0} + - {<<: *CMIP5, dataset: CESM1-BGC} + - {<<: *CMIP5, dataset: GFDL-ESM2M} + - {<<: *CMIP5, dataset: MIROC-ESM} + - {<<: *CMIP5, dataset: MRI-CGCM3} + # cmip6 + - {dataset: ACCESS-ESM1-5, institute: CSIRO} + - {dataset: CESM2, institute: NCAR} + - {dataset: GFDL-CM4, grid: gr1} + - {dataset: MIROC-ES2L, ensemble: r1i1p1f2} + - {dataset: MRI-ESM2-0} + + +preprocessors: + default: &default # common preprocessor settings + regrid: + target_grid: 3x3 + scheme: linear + distance_metric: + metric: weighted_rmse + climate_statistics: + operator: mean + period: month + mask_fillvalues: + threshold_fraction: 0.95 + multi_model_statistics: + span: overlap + statistics: [mean, median] + groupby: ['project'] + # exclude all possible reference datasets + exclude: [ + AIRS-2-1, + CERES-EBAF, + ERA-Interim, + ESACCI-AEROSOL, + ESACCI-CLOUD, + ESACCI-OZONE, + ESACCI-SOILMOISTURE, + ESACCI-SST, + GPCP-V2.2, + HadISST, + NCEP-NCAR-R1, + MODIS, + NIWA-BS, + PATMOS-x] + pp200: # only add/overwrite var specific settings + <<: *default + extract_levels: + levels: 20000 + scheme: linear + coordinate: air_pressure + pp500: # only add/overwrite var specific settings + <<: *default + extract_levels: + levels: 50000 + scheme: linear + coordinate: air_pressure + thr10: # only add/overwrite var specific settings + <<: *default + mask_fillvalues: + threshold_fraction: 0.10 + +# datasets can't be defined here as there is one variable that do not use own datasets +# is there a way to overwrite datasets rather than just put ADDITIONAL ones? + +var_default: &var_default + mip: Amon + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + preprocessor: default + grid: gn + start_year: 2000 + end_year: 2002 + split: Ref1 # first triangle + +diagnostics: + portrait_rmse: + themes: [aerosols, phys, clouds, atmDyn, chem, ghg] + realms: [atmos, land, atmosChem, ocean] + variables: + tas: &tas + <<: *var_default + short_name: tas + variable: tas + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + tas_2: + <<: *tas + split: Ref2 + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + pr: + <<: *var_default + variable: pr + split: Ref1 + additional_datasets: + - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, tier: 1, reference_for_metric: true} + rsut: + <<: *var_default + variable: rsut + start_year: 2001 + end_year: 2003 + additional_datasets: + - {dataset: CERES-EBAF, project: obs4MIPs, level: L3B, tier: 1, reference_for_metric: true} + ua200: &ua200 + <<: *var_default + variable: ua200 + short_name: ua + preprocessor: pp200 + split: Ref1 + additional_datasets: + - {dataset: ERA-Interim, project: OBS6, type: reanaly, + version: 1, tier: 3, reference_for_metric: true} + ua200_2: + <<: *ua200 + split: Ref2 + additional_datasets: + - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, + version: 1, tier: 2, reference_for_metric: true} + + scripts: + portrait: + script: portrait_plot.py + x_by: dataset + y_by: variable # extra_facet + group_by: project + normalize: "centered_median" + default_split: Ref1 + nan_color: null + plot_kwargs: + vmin: -0.5 + vmax: +0.5 + cbar_kwargs: + label: Relative RMSE + extend: both From 9863272492f921021537e30c8809de3ab47a9b33 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 21 Nov 2024 16:56:55 +0100 Subject: [PATCH 45/56] grammar [skip ci] Co-authored-by: Bettina Gier --- doc/sphinx/source/recipes/recipe_portrait.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index 6f8fccc11f..d06b7da2ed 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -9,7 +9,7 @@ Overview Portrait plots are a flexible way to visualize performance metrics for multiple datasets and up to four references. In this recipe `recipe_portrait_CMIP.yml` the normalized Root Mean Squared Deviation (RMSD) of global mean seasonal -climatologies is shown for a selection of CMIP models is plotted. +climatologies is calculated for a selection of CMIP models. For each variable up to two observation based datasets are used as reference. See :ref:`variables` for complete list of references. From c8f2136f2d0ee310425796964c9fd9df3dcfb2d0 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 21 Nov 2024 16:57:35 +0100 Subject: [PATCH 46/56] typo [skip ci] Co-authored-by: Bettina Gier --- doc/sphinx/source/recipes/recipe_perfmetrics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_perfmetrics.rst b/doc/sphinx/source/recipes/recipe_perfmetrics.rst index b501185ee6..428cbfc98f 100644 --- a/doc/sphinx/source/recipes/recipe_perfmetrics.rst +++ b/doc/sphinx/source/recipes/recipe_perfmetrics.rst @@ -4,7 +4,7 @@ Performance metrics for essential climate parameters ==================================================== .. note:: - Some of the results of this diagnostics can also be rproduced utilizing + Some of the results of this diagnostics can also be reproduced utilizing python diagnostics: Portrait plot: :ref:`recipe_portrait` Monitoring: :ref:`recipe_monitor` From 216a7af408966997870201fc6e4b060a6d512517 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 17:03:03 +0100 Subject: [PATCH 47/56] remove comment, change yaml formatting for dataset --- esmvaltool/recipes/recipe_portrait_CMIP.yml | 13 ++----------- .../recipes/testing/recipe_portrait_CMIP_fast.yml | 3 --- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/esmvaltool/recipes/recipe_portrait_CMIP.yml b/esmvaltool/recipes/recipe_portrait_CMIP.yml index 14be113022..2fbff77470 100644 --- a/esmvaltool/recipes/recipe_portrait_CMIP.yml +++ b/esmvaltool/recipes/recipe_portrait_CMIP.yml @@ -34,13 +34,11 @@ datasets: - {dataset: CESM2, institute: NCAR} - {dataset: CNRM-CM6-1, ensemble: r1i1p1f2, grid: gr} - {dataset: GFDL-CM4, grid: gr1} - # - {dataset: IPSL-CM6A-LR, grid: gr} - {dataset: MIROC-ES2L, ensemble: r1i1p1f2} - {dataset: MPI-ESM1-2-LR} - {dataset: MRI-ESM2-0} - {dataset: UKESM1-0-LL, ensemble: r1i1p1f2} - preprocessors: default: &default # common preprocessor settings regrid: @@ -90,9 +88,6 @@ preprocessors: mask_fillvalues: threshold_fraction: 0.10 -# datasets can't be defined here as there is one variable that do not use own datasets -# is there a way to overwrite datasets rather than just put ADDITIONAL ones? - var_default: &var_default mip: Amon project: CMIP6 @@ -128,12 +123,8 @@ diagnostics: short_name: clt variable: clt additional_datasets: - - dataset: ESACCI-CLOUD - project: OBS - type: sat - version: AVHRR-AMPM-fv3.0 - tier: 2 - reference_for_metric: true + - {dataset: ESACCI-CLOUD, project: OBS, type: sat, + version: AVHRR-AMPM-fv3.0, tier: 2, reference_for_metric: true} clt_2: <<: *clt split: Ref2 diff --git a/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml b/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml index 88104ecb2d..6f661a3df3 100644 --- a/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml +++ b/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml @@ -81,9 +81,6 @@ preprocessors: mask_fillvalues: threshold_fraction: 0.10 -# datasets can't be defined here as there is one variable that do not use own datasets -# is there a way to overwrite datasets rather than just put ADDITIONAL ones? - var_default: &var_default mip: Amon project: CMIP6 From fb73bac5c4d21d93619ca74b46730eefff3dc8d1 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 21 Nov 2024 17:08:48 +0100 Subject: [PATCH 48/56] clearify docs about example vs diag feature [skip ci] Co-authored-by: Bettina Gier --- doc/sphinx/source/recipes/recipe_portrait.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index d06b7da2ed..42d7c9766f 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -10,7 +10,7 @@ Portrait plots are a flexible way to visualize performance metrics for multiple datasets and up to four references. In this recipe `recipe_portrait_CMIP.yml` the normalized Root Mean Squared Deviation (RMSD) of global mean seasonal climatologies is calculated for a selection of CMIP models. -For each variable up to two observation based datasets are used as reference. +In the example recipe, for each variable up to two observation based datasets are used as reference. See :ref:`variables` for complete list of references. From 22e1c5d7df51a919033132bc6a407b51dfd0d857 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 21 Nov 2024 17:10:06 +0100 Subject: [PATCH 49/56] grammar [skip ci] Co-authored-by: Bettina Gier --- doc/sphinx/source/recipes/recipe_portrait.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index 42d7c9766f..5f43733860 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -23,8 +23,8 @@ This plot expects a scalar value in each input file and at most one input file for each subset of metadata that belongs to a cell or part of cell in the figure. By default cells are plotted for combinations of `short_name`, -`dataset`, `project` and `split`. -Where `split` is an optional extra_facet for variables. +`dataset`, `project` and `split`, +where `split` is an optional extra_facet for variables. However, all this can be customized using the `x_by`, `y_by`, `group_by` and `split_by` script settings. For a complete and detailed list of settings see the From 5df02dbc967698546fd83bd616645f9855cebe43 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Thu, 21 Nov 2024 17:15:01 +0100 Subject: [PATCH 50/56] seperate save_netcdf from plot() --- esmvaltool/diag_scripts/portrait_plot.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index 29480a8894..d5246ce01c 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -383,11 +383,9 @@ def plot_overlays(cfg, grid, data): def plot(cfg, data): """Create figure with subplots for each group and save to NetCDF. - sets same color range and overlays additional references based on + Sets same color range and overlays additional references based on the content of data (xr.DataArray). """ - save_to_netcdf(cfg, data) - fig = plt.figure(1, cfg.get("figsize", (5.5, 3.5))) group_count = len(data.coords[cfg["group_by"]]) grid = ImageGrid( @@ -510,8 +508,6 @@ def main(cfg): dataset["var"] = normalize(dataset["var"], cfg["normalize"], [cfg["x_by"], cfg["group_by"]]) plot(cfg, dataset["var"]) - - -if __name__ == "__main__": + save_to_netcdf(cfg, dataset["var"]) with run_diagnostic() as config: main(config) From 11fc9d96a17a827f3ee2d257faff8b337bf72910 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 22 Nov 2024 10:16:07 +0100 Subject: [PATCH 51/56] rephrase first paragraph --- doc/sphinx/source/recipes/recipe_portrait.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index 5f43733860..34c6a635ad 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -10,18 +10,18 @@ Portrait plots are a flexible way to visualize performance metrics for multiple datasets and up to four references. In this recipe `recipe_portrait_CMIP.yml` the normalized Root Mean Squared Deviation (RMSD) of global mean seasonal climatologies is calculated for a selection of CMIP models. -In the example recipe, for each variable up to two observation based datasets are used as reference. +In the example recipe, for each variable up to two observation based datasets +are used as reference. See :ref:`variables` for complete list of references. - - +The recipe uses preprocessor functions (distance metrics, global mean, +climate statistics) to calculate a scalar metric for each combination of +dataset, variable and reference, which is plotted by the `portrait_plot.py` +diagnostic script. User settings in recipe ----------------------- -This plot expects a scalar value in each input file and at most one input -file for each subset of metadata that belongs to a cell or part of cell in -the figure. By default cells are plotted for combinations of `short_name`, `dataset`, `project` and `split`, where `split` is an optional extra_facet for variables. From 2b5781ba41daf4be46816a70f0ee8627595be623 Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 22 Nov 2024 10:37:05 +0100 Subject: [PATCH 52/56] phrasing [skip ci] Co-authored-by: Bettina Gier --- doc/sphinx/source/recipes/recipe_portrait.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index 34c6a635ad..bb96aa92d3 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -25,7 +25,7 @@ User settings in recipe By default cells are plotted for combinations of `short_name`, `dataset`, `project` and `split`, where `split` is an optional extra_facet for variables. -However, all this can be customized using the `x_by`, +However, this can be customized using the `x_by`, `y_by`, `group_by` and `split_by` script settings. For a complete and detailed list of settings see the :doc:`diagnostic documentation `. From 9707d2959ddf6b72f41ac3505ad3e48620ff196a Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 22 Nov 2024 10:41:59 +0100 Subject: [PATCH 53/56] more grammar and typos --- doc/sphinx/source/recipes/recipe_portrait.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_portrait.rst b/doc/sphinx/source/recipes/recipe_portrait.rst index 34c6a635ad..dbde704918 100644 --- a/doc/sphinx/source/recipes/recipe_portrait.rst +++ b/doc/sphinx/source/recipes/recipe_portrait.rst @@ -25,13 +25,13 @@ User settings in recipe By default cells are plotted for combinations of `short_name`, `dataset`, `project` and `split`, where `split` is an optional extra_facet for variables. -However, all this can be customized using the `x_by`, +However, this can be customized using the `x_by`, `y_by`, `group_by` and `split_by` script settings. -For a complete and detailed list of settings see the +For a complete and detailed list of settings, see the :doc:`diagnostic documentation `. While this allows very flexible use for any kind of data, there are some -limitations as well: The grouping (separated -plots in the figure) and normalization is always applied along the x-axis. +limitations as well: The grouping (subplots) and normalization is always +applied along the x-axis. With default settings this means normalizing all metrics for each variable and grouping all datasets by project. @@ -51,13 +51,13 @@ Variables and Datasets The recipe generally works for any variable that is preprocessed correctly. To use different preprocessors or reference datasets it could be useful to create different variable groups and link them with the same extra_facet - like `variable_name` See recipe for examples. Below listed are the variables + like `variable_name`. See recipe for examples. Listed below are the variables used to produce the example figure. The following list shows which observational dataset is used as reference for each variable in this recipe. All variables are atmospheric monthly means. -For 3D variables the selected pressure level is given in parentheses. +For 3D variables the selected pressure level is specified in parentheses. * clt (Ref1: ESACCI-CLOUD, Ref2: PATMOS-x) * pr (Ref1: GPCP-V2.2) @@ -75,7 +75,7 @@ References Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). * Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., - and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, + and Cionni, I.: Quantitative evaluation of ozone and selected climate parameters in a set of EMAC simulations, Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). @@ -92,4 +92,4 @@ Example plots Relative space-time root-mean-square deviation (RMSD) calculated from the climatological seasonal cycle of CMIP5 and CMIP6 simulations. A relative performance is displayed, with blue shading indicating better and red shading indicating worse performance than the median of all model results. - A diagonal split of a grid square shows the relative error with respect to the reference data set + A diagonal split of a grid square shows the relative error with respect to the reference data set. From 0f5af792051df0ce99ae2f8b6a4a24343f9acb88 Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 22 Nov 2024 10:50:02 +0100 Subject: [PATCH 54/56] remove distance metric parameter, change name --- esmvaltool/diag_scripts/portrait_plot.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index d5246ce01c..efc42107f2 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -9,9 +9,9 @@ configured in the recipe. All `*_by` parameters can be set to any metadata key. To split by 'reference' this key needs to be set as extra_facet in recipe. -Author ------- -- Lukas Ruhe (Universität Bremen, Germany) +Authors +------- +- Lukas Lindenlaub (Universität Bremen, Germany) - Diego Cammarano Configuration parameters through recipe: @@ -20,10 +20,6 @@ ('mean', 'median', 'centered_mean', 'centered_median', None). Subtract median/mean if centered. Divide by median/mean if not None. By default 'centered_median'. -distance_metric: str or None, optional - A method for the distance_metric preprocessor can be set, to apply it to - the input data along all axis before plotting. If set to None, the input - is expected to contain scalar values for each input file. By default, None. x_by: str, optional Metadata key for x coordinate. By default 'alias'. From 404ad27ca4782cf749f3f97048ddd75cbe24d8bb Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 22 Nov 2024 11:03:36 +0100 Subject: [PATCH 55/56] more typos --- esmvaltool/diag_scripts/portrait_plot.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index efc42107f2..36faa4f0fc 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -1,11 +1,10 @@ -"""Overview plot for performance metrics. +"""Portrait Plot Diagnostic. Description ----------- -This diagnostic provides plot functionalities for performance metrics. -The multi model overview heatmap might be useful for different -tasks and therefore this diagnostic tries to be as flexible as possible. -X and Y axis, grouping parameter and slits for each rectangle can be +This diagnostic provides plot functionalities for performance metrics, +and is written to be as flexible as possible to be adaptable to further use +cases. X and Y axis, grouping parameter and splits for each rectangle can be configured in the recipe. All `*_by` parameters can be set to any metadata key. To split by 'reference' this key needs to be set as extra_facet in recipe. @@ -18,7 +17,8 @@ ---------------------------------------- normalize: str or None, optional ('mean', 'median', 'centered_mean', 'centered_median', None). - Subtract median/mean if centered. Divide by median/mean if not None. + Divide by median or mean if not None. Subtract median/mean afterwards if + centered. By default 'centered_median'. x_by: str, optional Metadata key for x coordinate. @@ -51,7 +51,7 @@ legend: dict, optional Customize, if, how and where the legend is plotted. The 'best' position and size of the legend depends on multiple parameters of the figure - (i.e. lengths of labels, aspect ratio of the plots...). And might require + (i.e. lengths of labels, aspect ratio of the plots...). Might require manual adjustment of `x`, `y` and `size` to fit the figure layout. Keys (each optional) that will be handled are: @@ -160,8 +160,6 @@ def plot_matrix(data, row_labels, col_labels, axe, plot_kwargs): va="center", rotation_mode="anchor", ) - # Turn spines off and create white grid. - # ax.spines[:].set_visible(False) axe.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) axe.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) axe.grid(which="minor", color="black", linestyle="-", linewidth=0.8) @@ -195,9 +193,9 @@ def open_file(metadata, **selection): if len(metas) > 1: raise ValueError(f"Multiple files found for {selection}") if len(metas) < 1: - log.debug("No Metadata found for %s", selection) + log.debug("No files found for %s", selection) return np.nan - log.debug("Metadata found for %s", selection) + log.debug("File found for %s", selection) das = xr.open_dataset(metas[0]["filename"]) varname = list(das.data_vars.keys())[0] try: @@ -209,7 +207,7 @@ def open_file(metadata, **selection): def load_data(cfg, metas): - """Load all nc files from metadata into xarray dataset. + """Load all netcdf files from metadata into xarray dataset. The dataset contains all relevant information for the plot. Coord names are metadata keys, ordered as x, y, group, split. The default From 7484d5a02fea997b43b38824ee5c345303d3850c Mon Sep 17 00:00:00 2001 From: Lukas Ruhe Date: Fri, 22 Nov 2024 11:15:21 +0100 Subject: [PATCH 56/56] more typos --- doc/sphinx/source/recipes/recipe_perfmetrics.rst | 2 +- esmvaltool/diag_scripts/portrait_plot.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/sphinx/source/recipes/recipe_perfmetrics.rst b/doc/sphinx/source/recipes/recipe_perfmetrics.rst index 428cbfc98f..d2f82dbffc 100644 --- a/doc/sphinx/source/recipes/recipe_perfmetrics.rst +++ b/doc/sphinx/source/recipes/recipe_perfmetrics.rst @@ -229,7 +229,7 @@ References Geophys. Res., 113, D06104, doi: 10.1029/2007JD008972 (2008). * Righi, M., Eyring, V., Klinger, C., Frank, F., Gottschaldt, K.-D., Jöckel, P., - and Cionni, I.: Quantitative evaluation of oone and selected climate parameters in a set of EMAC simulations, + and Cionni, I.: Quantitative evaluation of ozone and selected climate parameters in a set of EMAC simulations, Geosci. Model Dev., 8, 733, doi: 10.5194/gmd-8-733-2015 (2015). diff --git a/esmvaltool/diag_scripts/portrait_plot.py b/esmvaltool/diag_scripts/portrait_plot.py index 36faa4f0fc..63cf47d472 100644 --- a/esmvaltool/diag_scripts/portrait_plot.py +++ b/esmvaltool/diag_scripts/portrait_plot.py @@ -86,7 +86,7 @@ By default {}. nan_color: str or None, optional Matplotlib named color or hexcode for NaN values. If set to None, - no triagnles are plotted for NaN values. + no triangles are plotted for NaN values. By default 'white'. figsize: list(float), optional [width, height] of the figure in inches. The final figure will be saved @@ -203,7 +203,6 @@ def open_file(metadata, **selection): except ValueError as exc: msg = f"Expected scalar in input file {metas[0]['filename']}." raise ValueError(msg) from exc - # iris.load_cube(metas[0]["filename"]).data def load_data(cfg, metas): @@ -229,9 +228,8 @@ def load_data(cfg, metas): data['var'].loc[selection] = open_file(metas, **selection) if cfg["default_split"] is None: cfg["default_split"] = data.coords[cfg["split_by"]].values[0] - log.debug("using %s as default split", cfg["default_split"]) - log.debug("Loaded Data:") - log.debug(data) + log.debug("Using %s as default split", cfg["default_split"]) + log.debug("Loaded Data: %s", data) return data @@ -240,7 +238,7 @@ def split_legend(cfg, grid, data): Mpl handles axes positions in relative figure coordinates. To anchor the legend to the origin of the first graph (bottom left) with fixed size, - without messing up the layout for changing figure sizes a few extra steps + without messing up the layout for changing figure sizes, a few extra steps are required. NOTE: maybe `mpl_toolkits.axes_grid1.axes_divider.AxesDivider` simplifies this a bit by using `append_axes`. @@ -312,7 +310,7 @@ def overlay_reference(cfg, axe, data, triangle): def plot_group(cfg, axe, data, title=None): """Create matrix for one subplot in ax using plt.imshow. - by default split None is used, if all splits are named the first is + By default split None is used, if all splits are named the first is used. Other splits will be added by overlaying triangles. """ split = data.sel({cfg["split_by"]: cfg["default_split"]}) @@ -331,7 +329,7 @@ def plot_group(cfg, axe, data, title=None): def get_triangle_nodes(position, total_count=2): """Return list of nodes with relative x, y coordinates. - The nodes of the triangle are given as list of three tuples. Each tuples + The nodes of the triangle are given as list of three tuples. Each tuple contains relative coordinates (-0.5 to +0.5). For total of <= 2 a top left (position=0) and bottom right (position=1) rectangle is returned. For higher counts (3 or 4) one quartile is returned for each position.