From 713337c6395ceb53e484fa05ded2c13cfaf63550 Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Thu, 10 Aug 2023 12:45:05 +0200 Subject: [PATCH 01/14] v1 of hyper-param search implemented --- .gitignore | 5 +- env.yml | 3 +- expts/hydra-configs/architecture/toymix.yaml | 2 +- expts/hydra-configs/hparam_search/optuna.yaml | 54 +++++++++++++++++++ .../training/model/toymix_gcn.yaml | 1 + .../training/model/toymix_gin.yaml | 2 + expts/main_run_multitask.py | 30 +---------- graphium/cli/__init__.py | 6 +-- graphium/cli/__main__.py | 4 +- graphium/cli/data.py | 46 ++++------------ graphium/cli/finetune_utils.py | 30 ++++------- graphium/cli/main.py | 9 ++-- graphium/cli/train_finetune.py | 19 ++++--- graphium/finetuning/__init__.py | 3 ++ graphium/finetuning/utils.py | 1 + graphium/hyper_param_search/__init__.py | 3 ++ graphium/hyper_param_search/results.py | 44 +++++++++++++++ pyproject.toml | 2 +- 18 files changed, 158 insertions(+), 106 deletions(-) create mode 100644 expts/hydra-configs/hparam_search/optuna.yaml create mode 100644 graphium/hyper_param_search/__init__.py create mode 100644 graphium/hyper_param_search/results.py diff --git a/.gitignore b/.gitignore index 77cd466fc..e2d3cd745 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ *.datacache.gz lightning_logs/ logs/ +multirun/ +hparam-search-results/ models_checkpoints/ outputs/ out/ @@ -39,7 +41,8 @@ graphium/data/neurips2023/dummy-dataset/ graphium/data/make_data_splits/*.csv* graphium/data/make_data_splits/*.pt* graphium/data/make_data_splits/*.parquet* - +*.csv.gz +*.pt # Others expts_untracked/ diff --git a/env.yml b/env.yml index e49d071a4..dade0638b 100644 --- a/env.yml +++ b/env.yml @@ -5,7 +5,7 @@ channels: dependencies: - python >=3.8 - pip - - click + - typer - loguru - omegaconf >=2.0.0 - tqdm @@ -73,3 +73,4 @@ dependencies: - pip: - lightning-graphcore # optional, for using IPUs only - hydra-core>=1.3.2 + - hydra-optuna-sweeper diff --git a/expts/hydra-configs/architecture/toymix.yaml b/expts/hydra-configs/architecture/toymix.yaml index 6927f4e66..381b5e5c8 100644 --- a/expts/hydra-configs/architecture/toymix.yaml +++ b/expts/hydra-configs/architecture/toymix.yaml @@ -78,7 +78,7 @@ datamodule: featurization_n_jobs: 30 featurization_progress: True featurization_backend: "loky" - processed_graph_data_path: "../datacache/neurips2023-small/" + processed_graph_data_path: ${constants.datacache_path} num_workers: 30 # -1 to use all persistent_workers: False featurization: diff --git a/expts/hydra-configs/hparam_search/optuna.yaml b/expts/hydra-configs/hparam_search/optuna.yaml new file mode 100644 index 000000000..4395bb4b4 --- /dev/null +++ b/expts/hydra-configs/hparam_search/optuna.yaml @@ -0,0 +1,54 @@ +# @package _global_ +# +# For running a hyper-parameter search, we use the Optuna plugin for hydra. +# This makes optuna available as a sweeper in hydra and integrates easily with the rest of the codebase. +# For more info, see https://hydra.cc/docs/plugins/optuna_sweeper/ +# +# To run a hyper-param search, +# (1) Update this config, specifically the hyper-param search space; +# (2) Run `graphium-train +hparam_search=optuna` from the command line. + + +defaults: + - override /hydra/sweeper: optuna + # Optuna supports various sweepers (e.g. grid search, random search, TPE sampler) + - override /hydra/sweeper/sampler: tpe + +hyper_param_search: + # For the sweeper to work, the main process needs to return + # the objective value(s) (as a float) we are trying to optimize. + + # Assuming this is a metric, the `objective` key specifies which metric. + # Optuna supports multi-parameter optimization as well. + # If configured correctly, you can specify multiple keys. + objective: loss/test + + # Where to save results to + # NOTE (cwognum): Ideally, we would use the `hydra.sweep.dir` key, but they don't support remote paths. + # save_destination: gs://path/to/bucket + # overwrite_destination: false + +hydra: + # Run in multirun mode by default (i.e. actually use the sweeper) + mode: MULTIRUN + + # Changes the working directory + sweep: + dir: hparam-search-results/${constants.name} + subdir: ${hydra.job.num} + + # Sweeper config + sweeper: + sampler: + seed: ${constants.seed} + direction: minimize + study_name: ${constants.name} + storage: null + n_trials: 20 + n_jobs: 1 + + # The hyper-parameter search space definition + # See https://hydra.cc/docs/plugins/optuna_sweeper/#search-space-configuration for the options + params: + constants.seed: choice(0, 42) + diff --git a/expts/hydra-configs/training/model/toymix_gcn.yaml b/expts/hydra-configs/training/model/toymix_gcn.yaml index 48eabe003..3c7a13d05 100644 --- a/expts/hydra-configs/training/model/toymix_gcn.yaml +++ b/expts/hydra-configs/training/model/toymix_gcn.yaml @@ -6,6 +6,7 @@ constants: max_epochs: 100 data_dir: expts/data/neurips2023/small-dataset raise_train_error: true + datacache_path: ../datacache/neurips2023-small/ trainer: model_checkpoint: diff --git a/expts/hydra-configs/training/model/toymix_gin.yaml b/expts/hydra-configs/training/model/toymix_gin.yaml index ed2885efb..459694c9a 100644 --- a/expts/hydra-configs/training/model/toymix_gin.yaml +++ b/expts/hydra-configs/training/model/toymix_gin.yaml @@ -3,8 +3,10 @@ constants: name: neurips2023_small_data_gin seed: 42 + max_epochs: 100 data_dir: expts/data/neurips2023/small-dataset raise_train_error: true + datacache_path: ../datacache/neurips2023-small/ trainer: model_checkpoint: diff --git a/expts/main_run_multitask.py b/expts/main_run_multitask.py index c68663a08..d854c2c3e 100644 --- a/expts/main_run_multitask.py +++ b/expts/main_run_multitask.py @@ -1,33 +1,5 @@ -# General imports -import os -from os.path import dirname, abspath -from omegaconf import DictConfig, OmegaConf -import timeit -from loguru import logger -from datetime import datetime -from lightning.pytorch.utilities.model_summary import ModelSummary - -# Current project imports -import graphium -from graphium.config._loader import ( - load_datamodule, - load_metrics, - load_architecture, - load_predictor, - load_trainer, - save_params_to_wandb, - load_accelerator, -) -from graphium.utils.safe_run import SafeRun - import hydra - -# WandB -import wandb - -# Set up the working directory -MAIN_DIR = dirname(dirname(abspath(graphium.__file__))) -os.chdir(MAIN_DIR) +from omegaconf import DictConfig @hydra.main(version_base=None, config_path="hydra-configs", config_name="main") diff --git a/graphium/cli/__init__.py b/graphium/cli/__init__.py index 8928b9836..0bea27c69 100644 --- a/graphium/cli/__init__.py +++ b/graphium/cli/__init__.py @@ -1,3 +1,3 @@ -from .data import data_cli -from .finetune_utils import finetune_cli -from .main import main_cli +from .data import data_app +from .finetune_utils import finetune_app +from .main import app diff --git a/graphium/cli/__main__.py b/graphium/cli/__main__.py index 0baa7638c..3e6e96f3a 100644 --- a/graphium/cli/__main__.py +++ b/graphium/cli/__main__.py @@ -1,4 +1,4 @@ -from .main import main_cli +from .main import app if __name__ == "__main__": - main_cli() + app() diff --git a/graphium/cli/data.py b/graphium/cli/data.py index a6003b58d..ccfcaf427 100644 --- a/graphium/cli/data.py +++ b/graphium/cli/data.py @@ -1,41 +1,17 @@ -import click +import typer +import graphium from loguru import logger -import graphium +from .main import app + + +data_app = typer.Typer(help="Graphium datasets.") +app.add_typer(data_app, name="data") + -from .main import main_cli - - -@main_cli.group(name="data", help="Graphium datasets.") -def data_cli(): - pass - - -@data_cli.command(name="download", help="Download a Graphium dataset.") -@click.option( - "-n", - "--name", - type=str, - required=True, - help="Name of the graphium dataset to download.", -) -@click.option( - "-o", - "--output", - type=str, - required=True, - help="Where to download the Graphium dataset.", -) -@click.option( - "--progress", - type=bool, - is_flag=True, - default=False, - required=False, - help="Whether to extract the dataset if it's a zip file.", -) -def download(name, output, progress): +@data_app.command(name="download", help="Download a Graphium dataset.") +def download(name: str, output: str, progress: bool = True): args = {} args["name"] = name args["output_path"] = output @@ -49,7 +25,7 @@ def download(name, output, progress): logger.info(f"Dataset available at {fpath}.") -@data_cli.command(name="list", help="List available Graphium dataset.") +@data_app.command(name="list", help="List available Graphium dataset.") def list(): logger.info("Graphium datasets:") logger.info(graphium.data.utils.list_graphium_datasets()) diff --git a/graphium/cli/finetune_utils.py b/graphium/cli/finetune_utils.py index 80d437f98..c1218f1f1 100644 --- a/graphium/cli/finetune_utils.py +++ b/graphium/cli/finetune_utils.py @@ -1,40 +1,28 @@ +from typing import List, Optional import yaml -import click import fsspec +import typer from loguru import logger from hydra import compose, initialize from datamol.utils import fs -from .main import main_cli +from .main import app from .train_finetune import run_training_finetuning -@main_cli.group(name="finetune", help="Utility CLI for extra fine-tuning utilities.") -def finetune_cli(): - pass +finetune_app = typer.Typer(help="Utility CLI for extra fine-tuning utilities.") +app.add_typer(finetune_app, name="finetune") -@finetune_cli.command(name="admet") -@click.argument("save_dir") -@click.option("--wandb/--no-wandb", default=True, help="Whether to log to Weights & Biases.") -@click.option( - "--name", - "-n", - multiple=True, - help="One or multiple benchmarks to filter on. See also --inclusive-filter/--exclusive-filter.", -) -@click.option( - "--inclusive-filter/--exclusive-filter", - default=True, - help="Whether to include or exclude the benchmarks specified by `--name`.", -) -def benchmark_tdc_admet_cli(save_dir, wandb, name, inclusive_filter): +@finetune_app.command(name="admet") +def benchmark_tdc_admet_cli( + save_dir, wandb: bool = True, name: Optional[List[str]] = None, inclusive_filter: bool = True +): """ Utility CLI to easily fine-tune a model on (a subset of) the benchmarks in the TDC ADMET group. The results are saved to the SAVE_DIR. """ - try: from tdc.utils import retrieve_benchmark_names except ImportError: diff --git a/graphium/cli/main.py b/graphium/cli/main.py index 2161514e7..7cce5fce8 100644 --- a/graphium/cli/main.py +++ b/graphium/cli/main.py @@ -1,11 +1,8 @@ -import click +import typer -@click.group() -@click.version_option() -def main_cli(): - pass +app = typer.Typer(add_completion=False) if __name__ == "__main__": - main_cli() + app() diff --git a/graphium/cli/train_finetune.py b/graphium/cli/train_finetune.py index e6ead5122..190c76528 100644 --- a/graphium/cli/train_finetune.py +++ b/graphium/cli/train_finetune.py @@ -2,6 +2,8 @@ import wandb import timeit +from hydra.core.hydra_config import HydraConfig +from hydra.types import RunMode from omegaconf import DictConfig, OmegaConf from loguru import logger from datetime import datetime @@ -16,19 +18,17 @@ load_accelerator, save_params_to_wandb, ) -from graphium.finetuning import modify_cfg_for_finetuning, GraphFinetuning +from graphium.hyper_param_search import process_results_for_hyper_param_search, HYPER_PARAM_SEARCH_CONFIG_KEY +from graphium.finetuning import modify_cfg_for_finetuning, GraphFinetuning, FINETUNING_CONFIG_KEY from graphium.utils.safe_run import SafeRun -FINETUNING_CONFIG_KEY = "finetuning" - - @hydra.main(version_base=None, config_path="../../expts/hydra-configs", config_name="main") def cli(cfg: DictConfig) -> None: """ The main CLI endpoint for training and fine-tuning Graphium models. """ - run_training_finetuning(cfg) + return run_training_finetuning(cfg) def run_training_finetuning(cfg: DictConfig) -> None: @@ -116,7 +116,14 @@ def run_training_finetuning(cfg: DictConfig) -> None: if wandb_cfg is not None: wandb.finish() - return trainer.callback_metrics + results = trainer.callback_metrics + + # When part of of a hyper-parameter search, we are very specific about how we save our results + # NOTE (cwognum): We also check if the we are in multi-run mode, as the sweeper is otherwise not active. + if HYPER_PARAM_SEARCH_CONFIG_KEY in cfg and HydraConfig.get().mode == RunMode.MULTIRUN: + results = process_results_for_hyper_param_search(results, cfg[HYPER_PARAM_SEARCH_CONFIG_KEY]) + + return results if __name__ == "__main__": diff --git a/graphium/finetuning/__init__.py b/graphium/finetuning/__init__.py index 5ef566743..0bfaf0587 100644 --- a/graphium/finetuning/__init__.py +++ b/graphium/finetuning/__init__.py @@ -1,3 +1,6 @@ from .utils import modify_cfg_for_finetuning from .finetuning import GraphFinetuning from .finetuning_architecture import FullGraphFinetuningNetwork + + +FINETUNING_CONFIG_KEY = "finetuning" diff --git a/graphium/finetuning/utils.py b/graphium/finetuning/utils.py index 605ca536f..ce7e556a6 100644 --- a/graphium/finetuning/utils.py +++ b/graphium/finetuning/utils.py @@ -57,6 +57,7 @@ def modify_cfg_for_finetuning(cfg: Dict[str, Any]): # Load pretrained model pretrained_model_name = cfg_finetune["pretrained_model_name"] + print(GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name]) pretrained_predictor = PredictorModule.load_from_checkpoint( GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name], device="cpu" ) diff --git a/graphium/hyper_param_search/__init__.py b/graphium/hyper_param_search/__init__.py new file mode 100644 index 000000000..9b206feb6 --- /dev/null +++ b/graphium/hyper_param_search/__init__.py @@ -0,0 +1,3 @@ +from .results import process_results_for_hyper_param_search + +HYPER_PARAM_SEARCH_CONFIG_KEY = "hyper_param_search" diff --git a/graphium/hyper_param_search/results.py b/graphium/hyper_param_search/results.py new file mode 100644 index 000000000..0b2d92dcf --- /dev/null +++ b/graphium/hyper_param_search/results.py @@ -0,0 +1,44 @@ +import os +import enum +import fsspec +import hydra +import torch +import yaml + +from datamol.utils import fs + + +class _Keys(enum.Enum): + OBJECTIVE = "objective" + SAVE_DESTINATION = "save_destination" + FORCE = "overwrite_destination" + + +def process_results_for_hyper_param_search(results: dict, cfg: dict): + """Processes the results in the context of a hyper-parameter search.""" + + # Save the results to the current work directory + hydra_cfg = hydra.core.hydra_config.HydraConfig.get() + output_dir = hydra_cfg["runtime"]["output_dir"] + + results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()} + with fsspec.open(fs.join(output_dir, "trial_results.json"), "w") as f: + yaml.dump(results, f) + + # Copy the current working directory to remote + dst_dir = cfg.get(_Keys.SAVE_DESTINATION.value) + if dst_dir is not None: + relpath = os.path.relpath(output_dir, os.getcwd()) + dst = fs.join(dst_dir, relpath) + fs.copy_dir(output_dir, dst, force=cfg.get(_Keys.FORCE, False)) + + # Extract the objectives + objectives = cfg[_Keys.OBJECTIVE.value] + if isinstance(objectives, str): + objectives = [objectives] + + # Extract the objective values + objective_values = [results[k] for k in objectives] + if len(objective_values) == 1: + objective_values = objective_values[0] + return objective_values diff --git a/pyproject.toml b/pyproject.toml index 9e55eb5f9..831aac314 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ ] [project.scripts] -graphium = "graphium.cli.main:main_cli" +graphium = "graphium.cli.main:app" graphium-train = "graphium.cli.train_finetune:cli" [project.urls] From 994d2d46c2d90c36c90f1fd88318019f5a377439 Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Wed, 16 Aug 2023 13:12:47 +0200 Subject: [PATCH 02/14] Further changes to finetuning + hparam search pipeline --- expts/hydra-configs/finetuning/admet.yaml | 8 +-- expts/hydra-configs/hparam_search/optuna.yaml | 4 +- graphium/cli/train_finetune.py | 68 ++++++++++++++++--- graphium/config/_loader.py | 36 +++++----- graphium/config/dummy_finetuning.yaml | 2 +- graphium/data/dataset.py | 23 ++++--- .../finetuning/finetuning_architecture.py | 49 ++++++------- graphium/finetuning/utils.py | 16 ++--- graphium/hyper_param_search/__init__.py | 2 +- graphium/hyper_param_search/results.py | 29 ++------ graphium/trainer/predictor.py | 50 ++++++++------ tests/test_finetuning.py | 23 +++---- 12 files changed, 162 insertions(+), 148 deletions(-) diff --git a/expts/hydra-configs/finetuning/admet.yaml b/expts/hydra-configs/finetuning/admet.yaml index 80fb20e35..f601bd76d 100644 --- a/expts/hydra-configs/finetuning/admet.yaml +++ b/expts/hydra-configs/finetuning/admet.yaml @@ -29,16 +29,16 @@ constants: # For now, we assume a model is always fine-tuned on a single task at a time. # You can override this value with any of the benchmark names in the TDC benchmark suite. # See also https://tdcommons.ai/benchmark/admet_group/overview/ - task: &task lipophilicity_astrazeneca + task: lipophilicity_astrazeneca name: finetuning_${constants.task}_gcn wandb: name: ${constants.name} - project: *task + project: ${constants.task} entity: multitask-gnn save_dir: logs/${constants.task} seed: 42 - max_epochs: 10 + max_epochs: 100 data_dir: expts/data/admet/${constants.task} raise_train_error: true @@ -57,7 +57,7 @@ finetuning: level: graph # Pretrained model - pretrained_model_name: dummy-pretrained-model + pretrained_model: dummy-pretrained-model finetuning_module: task_heads # gnn sub_module_from_pretrained: zinc # optional new_sub_module: lipophilicity_astrazeneca # optional diff --git a/expts/hydra-configs/hparam_search/optuna.yaml b/expts/hydra-configs/hparam_search/optuna.yaml index 4395bb4b4..47811f3ec 100644 --- a/expts/hydra-configs/hparam_search/optuna.yaml +++ b/expts/hydra-configs/hparam_search/optuna.yaml @@ -44,11 +44,11 @@ hydra: direction: minimize study_name: ${constants.name} storage: null - n_trials: 20 + n_trials: 100 n_jobs: 1 # The hyper-parameter search space definition # See https://hydra.cc/docs/plugins/optuna_sweeper/#search-space-configuration for the options params: - constants.seed: choice(0, 42) + predictor.optim_kwargs.lr: tag(log, interval(0.00001, 0.001)) diff --git a/graphium/cli/train_finetune.py b/graphium/cli/train_finetune.py index 190c76528..b0a601eb2 100644 --- a/graphium/cli/train_finetune.py +++ b/graphium/cli/train_finetune.py @@ -1,25 +1,38 @@ -import hydra -import wandb +import os +import time import timeit +from datetime import datetime +import fsspec +import hydra +import torch +import wandb +import yaml +from datamol.utils import fs from hydra.core.hydra_config import HydraConfig from hydra.types import RunMode -from omegaconf import DictConfig, OmegaConf -from loguru import logger -from datetime import datetime from lightning.pytorch.utilities.model_summary import ModelSummary +from loguru import logger +from omegaconf import DictConfig, OmegaConf from graphium.config._loader import ( + load_accelerator, + load_architecture, load_datamodule, load_metrics, - load_architecture, load_predictor, load_trainer, - load_accelerator, save_params_to_wandb, ) -from graphium.hyper_param_search import process_results_for_hyper_param_search, HYPER_PARAM_SEARCH_CONFIG_KEY -from graphium.finetuning import modify_cfg_for_finetuning, GraphFinetuning, FINETUNING_CONFIG_KEY +from graphium.finetuning import ( + FINETUNING_CONFIG_KEY, + GraphFinetuning, + modify_cfg_for_finetuning, +) +from graphium.hyper_param_search import ( + HYPER_PARAM_SEARCH_CONFIG_KEY, + extract_main_metric_for_hparam_search, +) from graphium.utils.safe_run import SafeRun @@ -38,6 +51,18 @@ def run_training_finetuning(cfg: DictConfig) -> None: cfg = OmegaConf.to_container(cfg, resolve=True) + dst_dir = cfg["constants"].get("results_dir") + hydra_cfg = HydraConfig.get() + output_dir = hydra_cfg["runtime"]["output_dir"] + + if dst_dir is not None and fs.exists(dst_dir) and len(fs.get_mapper(dst_dir).fs.ls(dst_dir)) > 0: + logger.warning( + "The destination directory is not empty. " + "If files already exist, this would lead to a crash at the end of training." + ) + # We pause here briefly, to make sure the notification is seen as there's lots of logs afterwards + time.sleep(5) + # Modify the config for finetuning if FINETUNING_CONFIG_KEY in cfg: cfg = modify_cfg_for_finetuning(cfg) @@ -102,6 +127,12 @@ def run_training_finetuning(cfg: DictConfig) -> None: with SafeRun(name="TRAINING", raise_error=cfg["constants"]["raise_train_error"], verbose=True): trainer.fit(model=predictor, datamodule=datamodule) + # Save validation metrics - Base utility in case someone doesn't use a logger. + results = trainer.callback_metrics + results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()} + with fsspec.open(fs.join(output_dir, "val_results.yaml"), "w") as f: + yaml.dump(results, f) + # Determine the max num nodes and edges in testing predictor.set_max_nodes_edges_per_graph(datamodule, stages=["test"]) @@ -116,12 +147,27 @@ def run_training_finetuning(cfg: DictConfig) -> None: if wandb_cfg is not None: wandb.finish() + # Save test metrics - Base utility in case someone doesn't use a logger. results = trainer.callback_metrics + results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()} + with fsspec.open(fs.join(output_dir, "test_results.yaml"), "w") as f: + yaml.dump(results, f) # When part of of a hyper-parameter search, we are very specific about how we save our results # NOTE (cwognum): We also check if the we are in multi-run mode, as the sweeper is otherwise not active. - if HYPER_PARAM_SEARCH_CONFIG_KEY in cfg and HydraConfig.get().mode == RunMode.MULTIRUN: - results = process_results_for_hyper_param_search(results, cfg[HYPER_PARAM_SEARCH_CONFIG_KEY]) + if HYPER_PARAM_SEARCH_CONFIG_KEY in cfg and hydra_cfg.mode == RunMode.MULTIRUN: + results = extract_main_metric_for_hparam_search(results, cfg[HYPER_PARAM_SEARCH_CONFIG_KEY]) + + # Copy the current working directory to remote + # By default, processes should just write results to Hydra's output directory. + # However, this currently does not support remote storage, which is why we copy the results here if needed. + # For more info, see also: https://github.com/facebookresearch/hydra/issues/993 + + if dst_dir is not None: + src_dir = hydra_cfg["runtime"]["output_dir"] + dst_dir = fs.join(dst_dir, fs.get_basename(src_dir)) + fs.mkdir(dst_dir, exist_ok=True) + fs.copy_dir(src_dir, dst_dir) return results diff --git a/graphium/config/_loader.py b/graphium/config/_loader.py index 3c7a654e9..813251985 100644 --- a/graphium/config/_loader.py +++ b/graphium/config/_loader.py @@ -1,36 +1,35 @@ -from typing import Dict, Mapping, Tuple, Type, Union, Any, Optional, Callable - # Misc import os -import omegaconf from copy import deepcopy -from loguru import logger -import yaml +from typing import Any, Callable, Dict, Mapping, Optional, Tuple, Type, Union + import joblib -import pathlib -import warnings +import mup +import omegaconf # Torch import torch -import mup +import yaml # Lightning from lightning import Trainer from lightning.pytorch.callbacks import EarlyStopping, ModelCheckpoint -from lightning.pytorch.loggers import WandbLogger, Logger +from lightning.pytorch.loggers import Logger, WandbLogger +from loguru import logger -# Graphium -from graphium.utils.mup import set_base_shapes +from graphium.data.datamodule import BaseDataModule, MultitaskFromSmilesDataModule +from graphium.finetuning.finetuning_architecture import FullGraphFinetuningNetwork from graphium.ipu.ipu_dataloader import IPUDataloaderOptions -from graphium.trainer.metrics import MetricWrapper +from graphium.ipu.ipu_utils import import_poptorch, load_ipu_options from graphium.nn.architectures import FullGraphMultiTaskNetwork -from graphium.finetuning.finetuning_architecture import FullGraphFinetuningNetwork from graphium.nn.utils import MupMixin +from graphium.trainer.metrics import MetricWrapper from graphium.trainer.predictor import PredictorModule +from graphium.utils.command_line_utils import get_anchors_and_aliases, update_config + +# Graphium +from graphium.utils.mup import set_base_shapes from graphium.utils.spaces import DATAMODULE_DICT -from graphium.ipu.ipu_utils import import_poptorch, load_ipu_options -from graphium.data.datamodule import MultitaskFromSmilesDataModule, BaseDataModule -from graphium.utils.command_line_utils import update_config, get_anchors_and_aliases def get_accelerator( @@ -264,12 +263,12 @@ def load_architecture( if model_class is FullGraphFinetuningNetwork: finetuning_head_kwargs = config["finetuning"].pop("finetuning_head", None) pretrained_overwriting_kwargs = config["finetuning"].pop("overwriting_kwargs") - pretrained_model_name = pretrained_overwriting_kwargs.pop("pretrained_model_name") + pretrained_model = pretrained_overwriting_kwargs.pop("pretrained_model") model_kwargs = { "pretrained_model_kwargs": deepcopy(model_kwargs), "pretrained_overwriting_kwargs": pretrained_overwriting_kwargs, - "pretrained_model_name": pretrained_model_name, + "pretrained_model": pretrained_model, "finetuning_head_kwargs": finetuning_head_kwargs, } @@ -406,7 +405,6 @@ def load_trainer( # Define the early model checkpoing parameters if "model_checkpoint" in cfg_trainer.keys(): - cfg_trainer["model_checkpoint"]["dirpath"] += str(cfg_trainer["seed"]) + "/" callbacks.append(ModelCheckpoint(**cfg_trainer["model_checkpoint"])) # Define the logger parameters diff --git a/graphium/config/dummy_finetuning.yaml b/graphium/config/dummy_finetuning.yaml index 38cdf029b..d780f2931 100644 --- a/graphium/config/dummy_finetuning.yaml +++ b/graphium/config/dummy_finetuning.yaml @@ -30,7 +30,7 @@ finetuning: level: graph # Pretrained model - pretrained_model_name: dummy-pretrained-model + pretrained_model: dummy-pretrained-model finetuning_module: task_heads sub_module_from_pretrained: zinc # optional new_sub_module: lipophilicity_astrazeneca # optional diff --git a/graphium/data/dataset.py b/graphium/data/dataset.py index 180e3275f..9872fdce0 100644 --- a/graphium/data/dataset.py +++ b/graphium/data/dataset.py @@ -1,16 +1,15 @@ -from typing import Type, List, Dict, Union, Any, Callable, Optional, Tuple, Iterable - -from multiprocessing import Manager -import numpy as np -from functools import lru_cache -from loguru import logger -from copy import deepcopy import os -import numpy as np +from copy import deepcopy +from functools import lru_cache +from multiprocessing import Manager +from typing import Any, Dict, List, Optional, Tuple, Union +import fsspec +import numpy as np import torch +from loguru import logger from torch.utils.data.dataloader import Dataset -from torch_geometric.data import Data, Batch +from torch_geometric.data import Batch, Data from graphium.data.smiles_transform import smiles_to_unique_mol_ids from graphium.features import GraphDict @@ -247,7 +246,8 @@ def _load_metadata(self): "_num_edges_list", ] path = os.path.join(self.data_path, "multitask_metadata.pkl") - attrs = torch.load(path) + with fsspec.open(path, "rb") as f: + attrs = torch.load(path) if not set(attrs_to_load).issubset(set(attrs.keys())): raise ValueError( @@ -409,7 +409,8 @@ def load_graph_from_index(self, data_idx): filename = os.path.join( self.data_path, format(data_idx // 1000, "04d"), format(data_idx, "07d") + ".pkl" ) - data_dict = torch.load(filename) + with fsspec.open(filename, "rb") as f: + data_dict = torch.load(f) return data_dict def merge( diff --git a/graphium/finetuning/finetuning_architecture.py b/graphium/finetuning/finetuning_architecture.py index 8fd6263b5..21b28374d 100644 --- a/graphium/finetuning/finetuning_architecture.py +++ b/graphium/finetuning/finetuning_architecture.py @@ -1,33 +1,23 @@ -from typing import Iterable, List, Dict, Tuple, Union, Callable, Any, Optional, Type - from copy import deepcopy - -from loguru import logger +from typing import Any, Dict, List, Optional, Union import torch import torch.nn as nn - +from loguru import logger from torch import Tensor from torch_geometric.data import Batch -from graphium.data.utils import get_keys -from graphium.nn.base_graph_layer import BaseGraphStructure -from graphium.nn.architectures.encoder_manager import EncoderManager -from graphium.nn.architectures import FullGraphMultiTaskNetwork, FeedForwardNN, FeedForwardPyg, TaskHeads -from graphium.nn.architectures.global_architectures import FeedForwardGraph -from graphium.trainer.predictor_options import ModelOptions from graphium.nn.utils import MupMixin - from graphium.trainer.predictor import PredictorModule -from graphium.utils.spaces import GRAPHIUM_PRETRAINED_MODELS_DICT, FINETUNING_HEADS_DICT +from graphium.utils.spaces import FINETUNING_HEADS_DICT class FullGraphFinetuningNetwork(nn.Module, MupMixin): def __init__( self, - pretrained_model_name: str, - pretrained_model_kwargs: Dict[str, Any], - pretrained_overwriting_kwargs: Dict[str, Any], + pretrained_model: Union[str, "PretrainedModel"], + pretrained_model_kwargs: Dict[str, Any] = {}, + pretrained_overwriting_kwargs: Dict[str, Any] = {}, finetuning_head_kwargs: Optional[Dict[str, Any]] = None, num_inference_to_average: int = 1, last_layer_is_readout: bool = False, @@ -41,8 +31,8 @@ def __init__( Parameters: - pretrained_model_name: - Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT + pretrained_model: + A PretrainedModel or an identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT or a valid .ckpt checkpoint path pretrained_model_kwargs: Key-word arguments to instantiate a model of the same class as the pretrained model (e.g., FullGraphMultitaskNetwork)) @@ -79,16 +69,17 @@ def __init__( self.num_inference_to_average = num_inference_to_average self.last_layer_is_readout = last_layer_is_readout self._concat_last_layers = None - self.pretrained_model_name = pretrained_model_name + self.pretrained_model = pretrained_model self.pretrained_overwriting_kwargs = pretrained_overwriting_kwargs self.finetuning_head_kwargs = finetuning_head_kwargs self.max_num_nodes_per_graph = None self.max_num_edges_per_graph = None self.finetuning_head = None - self.pretrained_model = PretrainedModel( - pretrained_model_name, pretrained_model_kwargs, pretrained_overwriting_kwargs - ) + if not isinstance(self.pretrained_model, PretrainedModel): + self.pretrained_model = PretrainedModel( + self.pretrained_model, pretrained_model_kwargs, pretrained_overwriting_kwargs + ) if finetuning_head_kwargs is not None: self.finetuning_head = FinetuningHead(finetuning_head_kwargs) @@ -147,7 +138,7 @@ def make_mup_base_kwargs(self, divide_factor: float = 2.0) -> Dict[str, Any]: Dictionary with the kwargs to create the base model. """ kwargs = dict( - pretrained_model_name=self.pretrained_model_name, + pretrained_model=self.pretrained_model, pretrained_model_kwargs=None, finetuning_head_kwargs=None, num_inference_to_average=self.num_inference_to_average, @@ -186,18 +177,18 @@ def set_max_num_nodes_edges_per_graph(self, max_nodes: Optional[int], max_edges: class PretrainedModel(nn.Module, MupMixin): def __init__( self, - pretrained_model_name: str, + pretrained_model: str, pretrained_model_kwargs: Dict[str, Any], pretrained_overwriting_kwargs: Dict[str, Any], ): r""" - Flexible class allowing to finetune pretrained models from GRAPHIUM_PRETRAINED_MODELS_DICT. + Flexible class allowing to finetune pretrained models from GRAPHIUM_PRETRAINED_MODELS_DICT or from a ckeckpoint path. Can be any model that inherits from nn.Module, MupMixin and comes with a module map (e.g., FullGraphMultitaskNetwork) Parameters: - pretrained_model_name: - Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT + pretrained_model: + Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT or from a checkpoint path pretrained_model_kwargs: Key-word arguments to instantiate a model of the same class as the pretrained model (e.g., FullGraphMultitaskNetwork)) @@ -210,9 +201,7 @@ def __init__( super().__init__() # Load pretrained model - pretrained_model = PredictorModule.load_from_checkpoint( - GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name] - ).model + pretrained_model = PredictorModule.load_pretrained_models(pretrained_model).model pretrained_model.create_module_map() # Initialize new model with architecture after desired modifications to architecture. diff --git a/graphium/finetuning/utils.py b/graphium/finetuning/utils.py index ce7e556a6..b43dba7c5 100644 --- a/graphium/finetuning/utils.py +++ b/graphium/finetuning/utils.py @@ -1,10 +1,9 @@ -from typing import Union, List, Dict, Any - from copy import deepcopy +from typing import Any, Dict, List, Union + from loguru import logger -from graphium.trainer import PredictorModule -from graphium.utils.spaces import GRAPHIUM_PRETRAINED_MODELS_DICT +from graphium.trainer import PredictorModule def filter_cfg_based_on_admet_benchmark_name(config: Dict[str, Any], names: Union[List[str], str]): @@ -56,11 +55,8 @@ def modify_cfg_for_finetuning(cfg: Dict[str, Any]): cfg_finetune = cfg["finetuning"] # Load pretrained model - pretrained_model_name = cfg_finetune["pretrained_model_name"] - print(GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name]) - pretrained_predictor = PredictorModule.load_from_checkpoint( - GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name], device="cpu" - ) + pretrained_model = cfg_finetune["pretrained_model"] + pretrained_predictor = PredictorModule.load_pretrained_models(pretrained_model, device="cpu") # Inherit shared configuration from pretrained # Architecture @@ -147,7 +143,7 @@ def modify_cfg_for_finetuning(cfg: Dict[str, Any]): pretrained_overwriting_kwargs.pop(key, None) finetuning_training_kwargs = deepcopy(cfg["finetuning"]) - drop_keys = ["task", "level", "pretrained_model_name", "sub_module_from_pretrained", "finetuning_head"] + drop_keys = ["task", "level", "pretrained_model", "sub_module_from_pretrained", "finetuning_head"] for key in drop_keys: finetuning_training_kwargs.pop(key, None) diff --git a/graphium/hyper_param_search/__init__.py b/graphium/hyper_param_search/__init__.py index 9b206feb6..f7be11aff 100644 --- a/graphium/hyper_param_search/__init__.py +++ b/graphium/hyper_param_search/__init__.py @@ -1,3 +1,3 @@ -from .results import process_results_for_hyper_param_search +from .results import extract_main_metric_for_hparam_search HYPER_PARAM_SEARCH_CONFIG_KEY = "hyper_param_search" diff --git a/graphium/hyper_param_search/results.py b/graphium/hyper_param_search/results.py index 0b2d92dcf..c26352b1c 100644 --- a/graphium/hyper_param_search/results.py +++ b/graphium/hyper_param_search/results.py @@ -1,39 +1,20 @@ -import os import enum + import fsspec import hydra import torch import yaml - from datamol.utils import fs +from hydra.core.hydra_config import HydraConfig - -class _Keys(enum.Enum): - OBJECTIVE = "objective" - SAVE_DESTINATION = "save_destination" - FORCE = "overwrite_destination" +_OBJECTIVE_KEY = "objective" -def process_results_for_hyper_param_search(results: dict, cfg: dict): +def extract_main_metric_for_hparam_search(results: dict, cfg: dict): """Processes the results in the context of a hyper-parameter search.""" - # Save the results to the current work directory - hydra_cfg = hydra.core.hydra_config.HydraConfig.get() - output_dir = hydra_cfg["runtime"]["output_dir"] - - results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()} - with fsspec.open(fs.join(output_dir, "trial_results.json"), "w") as f: - yaml.dump(results, f) - - # Copy the current working directory to remote - dst_dir = cfg.get(_Keys.SAVE_DESTINATION.value) - if dst_dir is not None: - relpath = os.path.relpath(output_dir, os.getcwd()) - dst = fs.join(dst_dir, relpath) - fs.copy_dir(output_dir, dst, force=cfg.get(_Keys.FORCE, False)) - # Extract the objectives - objectives = cfg[_Keys.OBJECTIVE.value] + objectives = cfg[_OBJECTIVE_KEY] if isinstance(objectives, str): objectives = [objectives] diff --git a/graphium/trainer/predictor.py b/graphium/trainer/predictor.py index 3f2d2e676..ce6e9bb90 100644 --- a/graphium/trainer/predictor.py +++ b/graphium/trainer/predictor.py @@ -1,25 +1,29 @@ -from graphium.trainer.metrics import MetricWrapper -from typing import Dict, List, Any, Union, Any, Callable, Tuple, Type, Optional -from collections import OrderedDict -import numpy as np -from copy import deepcopy import time -from loguru import logger +from copy import deepcopy +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union -import torch -from torch import nn, Tensor import lightning -from torch_geometric.data import Data, Batch +import numpy as np +import torch +from loguru import logger from mup.optim import MuAdam +from torch import Tensor, nn +from torch_geometric.data import Batch, Data from graphium.config.config_convert import recursive_config_reformating -from graphium.trainer.predictor_options import EvalOptions, FlagOptions, ModelOptions, OptimOptions -from graphium.trainer.predictor_summaries import TaskSummaries from graphium.data.datamodule import BaseDataModule +from graphium.trainer.metrics import MetricWrapper +from graphium.trainer.predictor_options import ( + EvalOptions, + FlagOptions, + ModelOptions, + OptimOptions, +) +from graphium.trainer.predictor_summaries import TaskSummaries +from graphium.utils import fs from graphium.utils.moving_average_tracker import MovingAverageTracker -from graphium.utils.tensor import dict_tensor_fp16_to_fp32 - from graphium.utils.spaces import GRAPHIUM_PRETRAINED_MODELS_DICT +from graphium.utils.tensor import dict_tensor_fp16_to_fp32 class PredictorModule(lightning.LightningModule): @@ -669,22 +673,28 @@ def list_pretrained_models(): return GRAPHIUM_PRETRAINED_MODELS_DICT @staticmethod - def load_pretrained_models(name: str, device: str = None): + def load_pretrained_models(name_or_path: str, device: str = None): """Load a pretrained model from its name. Args: - name: Name of the model to load. List available + name: Name of the model to load or a valid checkpoint path. List available from `graphium.trainer.PredictorModule.list_pretrained_models()`. """ - if name not in GRAPHIUM_PRETRAINED_MODELS_DICT: + name = GRAPHIUM_PRETRAINED_MODELS_DICT.get(name_or_path) + + if name is not None: + return PredictorModule.load_from_checkpoint( + GRAPHIUM_PRETRAINED_MODELS_DICT[name_or_path], map_location=device + ) + + if name is None and not (fs.exists(name_or_path) and fs.get_extension(name_or_path) == "ckpt"): raise ValueError( - f"The model '{name}' is not available. Choose from {set(GRAPHIUM_PRETRAINED_MODELS_DICT.keys())}." + f"The model '{name_or_path}' is not available. Choose from {set(GRAPHIUM_PRETRAINED_MODELS_DICT.keys())} " + "or pass a valid checkpoint (.ckpt) path." ) - return PredictorModule.load_from_checkpoint( - GRAPHIUM_PRETRAINED_MODELS_DICT[name], map_location=device - ) + return PredictorModule.load_from_checkpoint(name_or_path, map_location=device) def set_max_nodes_edges_per_graph(self, datamodule: BaseDataModule, stages: Optional[List[str]] = None): datamodule.setup() diff --git a/tests/test_finetuning.py b/tests/test_finetuning.py index 7eeaf5f3f..932741e2f 100644 --- a/tests/test_finetuning.py +++ b/tests/test_finetuning.py @@ -1,31 +1,24 @@ import os -from os.path import dirname, abspath - import unittest as ut - -import torch from copy import deepcopy +from os.path import abspath, dirname +import torch from lightning.pytorch.callbacks import Callback - from omegaconf import OmegaConf -import graphium - -from graphium.finetuning import modify_cfg_for_finetuning -from graphium.trainer import PredictorModule - -from graphium.finetuning import GraphFinetuning +import graphium from graphium.config._loader import ( + load_accelerator, + load_architecture, load_datamodule, load_metrics, - load_architecture, load_predictor, load_trainer, save_params_to_wandb, - load_accelerator, ) - +from graphium.finetuning import GraphFinetuning, modify_cfg_for_finetuning +from graphium.trainer import PredictorModule MAIN_DIR = dirname(dirname(abspath(graphium.__file__))) CONFIG_FILE = "graphium/config/dummy_finetuning.yaml" @@ -96,7 +89,7 @@ def test_finetuning_pipeline(self): # Load pretrained & replace in predictor pretrained_model = PredictorModule.load_pretrained_models( - cfg["finetuning"]["pretrained_model_name"], device="cpu" + cfg["finetuning"]["pretrained_model"], device="cpu" ).model pretrained_model.create_module_map() From 5550d7bfdfe3bd8e9aa9369bc5e23b204c2cd1fa Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Tue, 22 Aug 2023 12:45:08 +0200 Subject: [PATCH 03/14] Started a notebook for analysis --- expts/hydra-configs/finetuning/admet.yaml | 2 +- graphium/cli/finetune_utils.py | 36 +- graphium/hyper_param_search/results.py | 9 - ...e-pretraining-finetuning-performance.ipynb | 326 ++++++++++++++++++ 4 files changed, 345 insertions(+), 28 deletions(-) create mode 100644 notebooks/compare-pretraining-finetuning-performance.ipynb diff --git a/expts/hydra-configs/finetuning/admet.yaml b/expts/hydra-configs/finetuning/admet.yaml index f601bd76d..7360707df 100644 --- a/expts/hydra-configs/finetuning/admet.yaml +++ b/expts/hydra-configs/finetuning/admet.yaml @@ -60,7 +60,7 @@ finetuning: pretrained_model: dummy-pretrained-model finetuning_module: task_heads # gnn sub_module_from_pretrained: zinc # optional - new_sub_module: lipophilicity_astrazeneca # optional + new_sub_module: ${constants.task} # optional # keep_modules_after_finetuning_module: # optional # graph_output_nn/graph: {} diff --git a/graphium/cli/finetune_utils.py b/graphium/cli/finetune_utils.py index c1218f1f1..1bf993ea1 100644 --- a/graphium/cli/finetune_utils.py +++ b/graphium/cli/finetune_utils.py @@ -1,27 +1,29 @@ from typing import List, Optional -import yaml + import fsspec import typer - -from loguru import logger -from hydra import compose, initialize +import yaml from datamol.utils import fs +from hydra import compose, initialize +from hydra.core.hydra_config import HydraConfig +from loguru import logger from .main import app from .train_finetune import run_training_finetuning - finetune_app = typer.Typer(help="Utility CLI for extra fine-tuning utilities.") app.add_typer(finetune_app, name="finetune") @finetune_app.command(name="admet") def benchmark_tdc_admet_cli( - save_dir, wandb: bool = True, name: Optional[List[str]] = None, inclusive_filter: bool = True + overrides: list[str], + name: Optional[List[str]] = None, + inclusive_filter: bool = True, ): """ Utility CLI to easily fine-tune a model on (a subset of) the benchmarks in the TDC ADMET group. - The results are saved to the SAVE_DIR. + A major limitation is that we cannot use all features of the Hydra CLI, such as multiruns. """ try: from tdc.utils import retrieve_benchmark_names @@ -29,23 +31,18 @@ def benchmark_tdc_admet_cli( raise ImportError("TDC needs to be installed to use this CLI. Run `pip install PyTDC`.") # Get the benchmarks to run this for - if name is None: + if len(name) == 0: name = retrieve_benchmark_names("admet_group") - elif not inclusive_filter: - name = [n for n in name if n not in retrieve_benchmark_names("admet_group")] + if not inclusive_filter: + name = [n for n in retrieve_benchmark_names("admet_group") if n not in name] + + logger.info(f"Running fine-tuning for the following benchmarks: {name}") results = {} # Use the Compose API to construct the config for n in name: - overrides = [ - "+finetuning=admet", - f"finetuning.task={n}", - f"finetuning.finetuning_head.task={n}", - ] - - if not wandb: - overrides.append("~constants.wandb") + overrides += ["+finetuning=admet", f"constants.task={n}"] with initialize(version_base=None, config_path="../../expts/hydra-configs"): cfg = compose( @@ -58,6 +55,9 @@ def benchmark_tdc_admet_cli( ret = {k: v.item() for k, v in ret.items()} results[n] = ret + # Save to the results_dir by default or to the Hydra output_dir if needed. + # This distinction is needed, because Hydra's output_dir cannot be remote. + save_dir = cfg["constants"].get("results_dir", HydraConfig.get()["runtime"]["output_dir"]) fs.mkdir(save_dir, exist_ok=True) path = fs.join(save_dir, "results.yaml") logger.info(f"Saving results to {path}") diff --git a/graphium/hyper_param_search/results.py b/graphium/hyper_param_search/results.py index c26352b1c..6aab001f3 100644 --- a/graphium/hyper_param_search/results.py +++ b/graphium/hyper_param_search/results.py @@ -1,12 +1,3 @@ -import enum - -import fsspec -import hydra -import torch -import yaml -from datamol.utils import fs -from hydra.core.hydra_config import HydraConfig - _OBJECTIVE_KEY = "objective" diff --git a/notebooks/compare-pretraining-finetuning-performance.ipynb b/notebooks/compare-pretraining-finetuning-performance.ipynb new file mode 100644 index 000000000..0f971e1b4 --- /dev/null +++ b/notebooks/compare-pretraining-finetuning-performance.ipynb @@ -0,0 +1,326 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import fsspec\n", + "import yaml\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "from tqdm import tqdm\n", + "from typing import Literal\n", + "from dataclasses import dataclass, field\n", + "from collections import defaultdict\n", + "from graphium.utils import fs" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "ROOT_DIR = \"gs://graphium-private/pretrained-models/ToyMix/cas\"\n", + "\n", + "PT_TASKS = [\"qm9\", \"tox21\", \"zinc\"]\n", + "PT_FT_RELS = {\n", + " \"toymix_gcn_1\": {\"caco2\": \"finetuning_caco2_wang_gcn_1\", \"lipophilicity\": \"finetuning_lipophilicity_astrazeneca_gcn_1\"},\n", + " \"toymix_gcn_2\": {\"caco2\": \"finetuning_caco2_wang_gcn_2\", \"lipophilicity\": \"finetuning_lipophilicity_astrazeneca_gcn_2\"},\n", + "}\n", + "FT_METRICS = {\"caco2\": \"graph_caco2_wang/mae/test\", \"lipophilicity\": \"graph_lipophilicity_astrazeneca/r2_score/test\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@dataclass\n", + "class FinetuningResult: \n", + " name: str\n", + " scores: dict[str, list[float]] = field(default_factory=lambda: defaultdict(list))\n", + "\n", + " def best(self, metric, minimize: bool = False):\n", + " return max(self.scores[metric]) if not minimize else min(self.scores[metric])\n", + "\n", + "\n", + "@dataclass\n", + "class PretrainingResult: \n", + " name: str\n", + " loss: dict[Literal[\"qm9\", \"zinc\", \"tox21\", \"all\"], float] = field(default_factory=dict)\n", + " ft_results: dict[str, FinetuningResult] = field(default_factory=dict)\n", + "\n", + " @property\n", + " def finetuning_tasks(self):\n", + " return sorted(list(self.ft_results.keys()))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/cas.wognum/micromamba/envs/graphium/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", + " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n", + "100%|██████████| 2/2 [00:20<00:00, 10.05s/it]\n" + ] + } + ], + "source": [ + "globber, _ = fsspec.core.url_to_fs(ROOT_DIR)\n", + "\n", + "results = {}\n", + "\n", + "for pt_dir, ft_dirs in tqdm(PT_FT_RELS.items()):\n", + " \n", + " # Create a new results object\n", + " results[pt_dir] = PretrainingResult(name=pt_dir)\n", + "\n", + " # Parse the pre-training results\n", + " pt_results_path = fs.join(ROOT_DIR, pt_dir, \"results\", \"test_results.yaml\")\n", + " with fsspec.open(pt_results_path, \"r\") as f:\n", + " pt_results = yaml.safe_load(f)\n", + " results[pt_dir].loss = {k: pt_results[f\"graph_{k}/loss/test\"] for k in PT_TASKS}\n", + " results[pt_dir].loss[\"all\"] = pt_results[\"loss/test\"]\n", + "\n", + " # Parse the associated fine-tuning results\n", + " for ft_label, ft_dir in ft_dirs.items():\n", + "\n", + " # Create a new results object\n", + " ft_results = FinetuningResult(name=ft_label)\n", + " \n", + " # Find all results for all trials\n", + " ft_results_pattern = fs.join(ROOT_DIR, ft_dir, \"**\", \"test_results.yaml\")\n", + " paths = globber.glob(ft_results_pattern)\n", + "\n", + " # Save all scores\n", + " for path in paths: \n", + " with globber.open(path, \"r\") as f:\n", + " data = yaml.safe_load(f)\n", + " for k, v in data.items():\n", + " ft_results.scores[k].append(v)\n", + " \n", + " # Save the finetuning results to the pre-training results\n", + " results[pt_dir].ft_results[ft_label] = ft_results\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def draw_boxplot(results: dict[str, FinetuningResult], fine_tuning_task: str, metric_label: str = None, loss_label: str = \"all\", ax=None):\n", + " if ax is None: \n", + " _, ax = plt.subplots()\n", + " if metric_label is None:\n", + " metric_label = FT_METRICS[fine_tuning_task]\n", + " for pt_label, pt_results in results.items():\n", + " positions = [round(pt_results.loss[loss_label], 3)]\n", + " data = pt_results.ft_results[fine_tuning_task].scores[metric_label]\n", + " ax.boxplot(data, positions=positions)\n", + " ax.set_xlabel(f\"Pre-training loss on {loss_label}\")\n", + " ax.set_ylabel(metric_label) " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAGGCAYAAACqvTJ0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABT3klEQVR4nO3dfXzO9f////sxY4ZtcpazNeZsc5KclBhKqFRC75KKkPq+lXKWTsS7dEZnIokiSQmdUd4V8hZJEs2kNKxYpMlZzFnTtufvj36OT2sbx/HaseP13Ha7Xi67vNvreO11PI9jHbf363h0HMc8xhgjAAAAAAAAIIhC3F4AAAAAAAAASh6GUgAAAAAAAAg6hlIAAAAAAAAIOoZSAAAAAAAACDqGUgAAAAAAAAg6hlIAAAAAAAAIOoZSAAAAAAAACDqGUgAAAAAAAAi6ULcXUBDZ2dn69ddfFRERIY/H4/ZyADhkjNHRo0dVs2ZNhYQUn1k5jQKKh+LYKPoEFA/0CYCtfO1TkR5K/frrr4qOjnZ7GQACZPfu3apdu7bbywgYGgUUL8WpUfQJKF7oEwBbna1PRXooFRERIemvGxkZGenyagA4lZ6erujoaO9jurigUUDxUBwbRZ+A4oE+AbCVr30q0kOp0y/njIyMJFhAMVDcXqJNo4DipTg1ij4BxQt9AmCrs/WpeLzxGAAAAAAAAEUKQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEXajbCwCKkhMnTmjr1q25tp88eVKpqamqU6eOwsPDc10eFxencuXKBWOJAIohp+2R6A8Ad3DOBMBW9MkuDKUAP2zdulWtWrXy++cSExPVsmXLQlgRgJLAaXsk+gPAHZwzAbAVfbILQynAD3FxcUpMTMy1PTk5WX379tXcuXMVHx+f588BgFNO23P6ZwEg2DhnAmAr+mQXhlKAH8qVK3fG6Xh8fDzTcwABR3sAFDV0C4Ct6JNd+KBzAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAASd60OpPXv2qG/fvqpcubLKlSunCy64QImJiW4vCwDoEwBr0ScAtqJPAPwR6uaV//7770pISFCnTp20ZMkSVatWTT/99JMqVqzo5rIAgD4BsBZ9AmAr+gTAX64OpZ5++mlFR0dr9uzZ3m116tRxb0EA8P+jTwBsRZ8A2Io+AfCXq2/fW7x4sVq3bq0bbrhB1apVU4sWLTRz5sx898/IyFB6enqOLwAoDP72SaJRAIKDPgGwFX0C4C9Xh1I7duzQ9OnT1aBBAy1btkyDBw/W0KFD9cYbb+S5/4QJExQVFeX9io6ODvKKAZQU/vZJolEAgoM+AbAVfQLgL48xxrh15WXKlFHr1q21du1a77ahQ4dqw4YN+uqrr3Ltn5GRoYyMDO/36enpio6O1pEjRxQZGRmUNQN52bhxo1q1aqXExES1bNnS7eUUOenp6YqKirLqsexvnyQaheCjPcFhW6PoE4oyuhVY9AkIHPoUWL72ydVXStWoUUONGzfOsS0+Pl67du3Kc/+wsDBFRkbm+AKAwuBvnyQaBSA46BMAW9EnAP5y9YPOExIStG3bthzbtm/frpiYGJdWVHydOHFCW7duzbX95MmTSk1NVZ06dRQeHp7r8ri4OJUrVy4YSwSsQp/skV+/JBqGkok+ucPpuZREi1By0Cf30CgUVa4OpUaMGKF27dpp/Pjx6t27t9avX68ZM2ZoxowZbi6rWNq6datatWrl98/x0kWUVPTJHk77JdEwFE/0yR20CDg7+uQeGoWiytWh1IUXXqhFixZp9OjReuyxx1S3bl1NnjxZt9xyi5vLKpbi4uKUmJiYa3tycrL69u2ruXPnKj4+Ps+fA0oi+mSP/Pol0TCUTPTJHU7PpU7/LFAS0Cf30CgUVa4OpSTpmmuu0TXXXOP2Moq9cuXKnXH6HR8fz3Qc+Af6ZIez9UuiYSh56FPwcS4F+IY+uYNGoahy9YPOAQAAAAAAUDIxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNA5GkrFxsbq4MGDubYfPnxYsbGxBV4UADhFnwDYij4BsBmNAuAGR0Op1NRUZWVl5dqekZGhPXv2FHhRAOAUfQJgK/oEwGY0CoAbQv3ZefHixd5/XrZsmaKiorzfZ2VlacWKFapTp07AFgcAvqJPAGxFnwDYjEYBcJNfQ6mePXtKkjwej/r375/jstKlS6tOnTqaOHFiwBYHAL6iTwBsRZ8A2IxGAXCTX0Op7OxsSVLdunW1YcMGValSpVAWBQD+ok8AbEWfANiMRgFwk19DqdN27tyZa9vhw4dVsWLFgq4HAAqEPgGwFX0CYDMaBcANjj7o/Omnn9bbb7/t/f6GG25QpUqVVKtWLX377bcBWxwA+Is+AbAVfQJgMxoFwA2OhlKvvPKKoqOjJUnLly/X//73Py1dulTdunXTfffdF9AFAoA/6BMAW9EnADajUQDc4Ojte2lpad5gffTRR+rdu7cuv/xy1alTR23atAnoAgHAH/QJgK3oEwCb0SgAbnD0SqlzzjlHu3fvliQtXbpUXbp0kSQZY5SVlRW41QGAn+gTAFvRJwA2o1EA3ODolVLXXXedbr75ZjVo0EAHDx5Ut27dJEmbNm1S/fr1A7pAAPAHfQJgK/oEwGY0CoAbHA2lJk2apDp16mj37t165plnVKFCBUl/veTzrrvuCugCAcAf9AmAregTAJvRKABucDSUKl26tEaNGpVr+/Dhwwu6HgAoEPoEwFb0CYDNaBQANzj6TClJevPNN9W+fXvVrFlTP//8syRp8uTJ+vDDDwO2OABwgj4BsBV9AmAzGgUg2BwNpaZPn66RI0eqW7duOnz4sPeD7ypWrKjJkycHcn0A4Bf6BMBW9AmAzWgUADc4Gkq9+OKLmjlzpsaMGaNSpUp5t7du3VrfffddwBYHAP6iTwBsRZ8A2IxGAXCDo6HUzp071aJFi1zbw8LCdPz48QIvCgCcok8AbEWfANiMRgFwg6OhVN26dbVp06Zc25csWaLGjRv7fJxx48bJ4/Hk+KpevbqTJQGAJPoEwF70CYDNaBQANzj663v33XefhgwZoj/++EPGGK1fv17z58/XhAkT9Oqrr/p1rCZNmuh///uf9/u/v1QUAPxFnwDYij4BsBmNAuAGR0OpgQMHKjMzU/fff79OnDihm2++WbVq1dILL7ygPn36+LeA0FAm5wAChj4BsBV9AmAzGgXADY6GUpJ0xx136I477tCBAweUnZ2tatWqOTpOSkqKatasqbCwMLVp00bjx49XbGxsnvtmZGQoIyPD+316erqj6wRQvLnRJ4lGATg7+gTAZjzHAxBsjj5T6rLLLtPhw4clSVWqVPHGKj09XZdddpnPx2nTpo3eeOMNLVu2TDNnztTevXvVrl07HTx4MM/9J0yYoKioKO9XdHS0k+UDKMbc6pNEowCcGX0CYDOe4wFwg6Oh1KpVq3Tq1Klc2//44w998cUXPh+nW7du+te//qVmzZqpS5cu+vjjjyVJc+bMyXP/0aNH68iRI96v3bt3O1k+gGLMrT5JNArAmdEnADbjOR4AN/j19r3Nmzd7//mHH37Q3r17vd9nZWVp6dKlqlWrluPFlC9fXs2aNVNKSkqel4eFhSksLMzx8QEUX273SaJRAPJGnwDYzO1G0SegZPNrKHXBBRd4/6xnXi/hDA8P14svvuh4MRkZGUpOTlaHDh0cHwNAyUSfANiKPgGwGY0C4Ca/hlI7d+6UMUaxsbFav369qlat6r2sTJkyqlatml9/7nPUqFHq3r27zjvvPO3bt09PPPGE0tPT1b9/f3+WBQD0CYC16BMAm9EoAG7yayj1yiuvqGfPnsrOzg7Ilf/yyy+66aabdODAAVWtWlUXX3yx1q1bp5iYmIAcH0DJQZ8A2Io+AbAZjQLgJr+GUmlpabrmmmtUqlQpde/eXT169FCXLl0cvwd4wYIFjn4OAP6JPgGwFX0CYDMaBcBNfv31vdmzZ+u3337TO++8o4oVK+ree+9VlSpVdN111+n111/XgQMHCmudAHBG9AmAregTAJvRKABu8msoJUkej0cdOnTQM888o61bt2r9+vW6+OKLNXPmTNWqVUsdO3bUc889pz179hTGegEgX/QJgK3oEwCb0SgAbvF7KPVP8fHxuv/++/Xll19q9+7d6t+/v7744gvNnz8/EOsDAMfoEwBb0ScANqNRAILFr8+U+qcff/xRP/30kzp27Kjw8HBVrVpVgwYN0qBBgwK1PgBwhD4BsBV9AmAzGgUgmBy9UurgwYPq0qWLGjZsqKuuukppaWmSpNtvv12jRo0K6AIBwB/0CYCt6BMAm9EoAG5wNJQaMWKEQkNDtWvXLpUrV867/cYbb9SSJUsCtjgA8Bd9AmAr+gTAZjQKgBscvX3v008/1bJly1S7du0c2xs0aKCff/45IAsDACfoEwBb0ScANqNRANzg6JVSx48fzzE9P+3AgQMKCwsr8KIAwCn6BMBW9AmAzWgUADc4Gkp17NhRb7zxhvd7j8ej7OxsPfvss+rUqVPAFgcA/qJPAGxFnwDYjEYBcIOjt+89++yzuvTSS/XNN9/o1KlTuv/++7VlyxYdOnRIX375ZaDXCAA+o08AbEWfANiMRgFwg6NXSjVu3FibN2/WRRddpK5du+r48eO67rrrlJSUpHr16gV6jQDgM/oEwFb0CYDNaBQANzh6pZQkVa9eXY8++mgg1wIAAUGfANiKPgGwGY0CEGyOh1KSdOLECe3atUunTp3Ksf38888v0KIAoKDoEwBb0ScANqNRAILJ0VBq//79GjhwoJYsWZLn5VlZWQVaFAA4RZ8A2Io+AbAZjQLgBkefKTV8+HD9/vvvWrduncLDw7V06VLNmTNHDRo00OLFiwO9RgDwGX0CYCv6BMBmNAqAGxy9Uuqzzz7Thx9+qAsvvFAhISGKiYlR165dFRkZqQkTJujqq68O9DoBwCf0CYCt6BMAm9EoAG5w9Eqp48ePq1q1apKkSpUqaf/+/ZKkZs2aaePGjYFbHQD4iT4BsBV9AmAzGgXADY6GUo0aNdK2bdskSRdccIFeeeUV7dmzRy+//LJq1KgR0AUCgD/oEwBb0ScANqNRANzg6O17w4cPV1pamiTpkUce0RVXXKG33npLZcqU0euvvx7I9QGAX+gTAFvRJwA2o1EA3OBoKHXLLbd4/7lFixZKTU3V1q1bdd5556lKlSoBWxwA+Is+AbAVfQJgMxoFwA2OhlL/VK5cObVs2TIQhwKAgKJPAGxFnwDYjEYBCAZHQyljjN577z2tXLlS+/btU3Z2do7LFy5cGJDFAYC/6BMAW9EnADajUQDc4GgoNWzYMM2YMUOdOnXSueeeK4/HE+h1AYAj9AmAregTAJvRKABucDSUmjt3rhYuXKirrroq0OsBgAKhTwBsRZ8A2IxGAXBDiJMfioqKUmxsbKDXAgAFRp8A2Io+AbAZjQLgBkdDqXHjxunRRx/VyZMnA70eACgQ+gTAVvQJgM1oFAA3OHr73g033KD58+erWrVqqlOnjkqXLp3j8o0bNwZkcQDgL/oEwFb0CYDNaBQANzgaSg0YMECJiYnq27cvH4IHwCr0CYCt6BMAm9EoAG5wNJT6+OOPtWzZMrVv3z7Q6wGAAqFPAGxFnwDYjEYBcIOjz5SKjo5WZGRkoNcCAAVGnwDYij4BsBmNAuAGR0OpiRMn6v7771dqamqAlwMABUOfANiKPgGwGY0C4AZHb9/r27evTpw4oXr16qlcuXK5PgTv0KFDAVkcAPiLPgGwFX0CYDMaBcANjoZSkyZN4oPvAFiJPgGwFX0CYDMaBcANfg2lPv30U3Xq1EkDBgwopOUAgDP0CYCt6BMAm9EoAG7y6zOlBg8erKpVq+rGG2/UvHnzdPjw4UJaFgD4hz4BsBV9AmAzGgXATX4NpXbs2KHVq1erWbNmmjx5sqpXr67OnTtrypQpfCAeAFfRJwC2ok8AbEajALjJ77++d/7552vs2LFav369duzYoRtuuEFLly5VfHy8mjdvrocffljffPNNYawVAM6IPgGwFX0CYDMaBcAtfg+l/q5mzZoaPHiwPvnkEx04cED/+c9/lJqaqiuvvFLjx48P1BoBwG/0CYCt6BMAm9EoAMHk6K/v5aV8+fK6/vrrdf311ys7O1sHDx4M1KEBoEDoEwBb0ScANqNRAAqbo6HUlClT8tzu8XhUtmxZNWjQQB06dPDrmBMmTNBDDz2kYcOGafLkyU6WBQCF0ieJRgEoOPoEwGY8xwPgBkdDqUmTJmn//v06ceKEzjnnHBljdPjwYZUrV04VKlTQvn37FBsbq5UrVyo6Ovqsx9uwYYNmzJih888/38lyAMAr0H2SaBSAwKBPAGzGczwAbnD0mVLjx4/XhRdeqJSUFB08eFCHDh3S9u3b1aZNG73wwgvatWuXqlevrhEjRpz1WMeOHdMtt9yimTNn6pxzznGyHADwCmSfJBoFIHDoEwCb8RwPgBscDaXGjh2rSZMmqV69et5t9evX13PPPafRo0erdu3aeuaZZ/Tll1+e9VhDhgzR1VdfrS5dupx134yMDKWnp+f4AoC/C2SfJBoFIHDoEwCb8RwPgBscvX0vLS1NmZmZubZnZmZq7969kv76qw1Hjx4943EWLFigjRs3asOGDT5d74QJE/Too4/6v2AAJUag+iTRKACBRZ8A2IzneADc4OiVUp06ddK///1vJSUlebclJSXpzjvv1GWXXSZJ+u6771S3bt18j7F7924NGzZMc+fOVdmyZX263tGjR+vIkSPer927dztZPoBiLBB9kmgUgMCjTwBsxnM8AG5wNJSaNWuWKlWqpFatWiksLExhYWFq3bq1KlWqpFmzZkmSKlSooIkTJ+Z7jMTERO3bt0+tWrVSaGioQkND9fnnn2vKlCkKDQ1VVlZWrp8JCwtTZGRkji8A+LtA9EmiUQACjz4BsBnP8QC4wdHb96pXr67ly5dr69at2r59u4wxiouLU6NGjbz7dOrU6YzH6Ny5s7777rsc2wYOHKi4uDg98MADKlWqlJOlASjhAtEniUYBCDz6BMBmPMcD4AZHQ6nT4uLiFBcX5+hnIyIi1LRp0xzbypcvr8qVK+faDgD+KkifJBoFoPDQJwA24zkegGByNJTKysrS66+/rhUrVmjfvn3Kzs7Ocflnn30WkMUBgL/oEwBb0ScANqNRANzgaCg1bNgwvf7667r66qvVtGlTeTyegCxm1apVATkOgJKrsPok0SgABUOfANiM53gA3OBoKLVgwQK98847uuqqqwK9HgAoEPoEwFb0CYDNaBQANzj663tlypRR/fr1A70WACgw+gTAVvQJgM1oFAA3OBpK3XvvvXrhhRdkjAn0egCgQOgTAFvRJwA2o1EA3ODo7Xtr1qzRypUrtWTJEjVp0kSlS5fOcfnChQsDsjgA8Bd9AmAr+gTAZjQKgBscDaUqVqyoXr16BXotAFBg9AmAregTAJvRKABucDSUmj17dqDXAQABQZ8A2Io+AbAZjQLgBkefKQUAAAAAAAAUhKNXSknSe++9p3feeUe7du3SqVOncly2cePGAi8MAJyiTwBsRZ8A2IxGAQg2R6+UmjJligYOHKhq1aopKSlJF110kSpXrqwdO3aoW7dugV4jAPiMPgGwFX0CYDMaBcANjoZS06ZN04wZMzR16lSVKVNG999/v5YvX66hQ4fqyJEjgV4jAPiMPgGwFX0CYDMaBcANjoZSu3btUrt27SRJ4eHhOnr0qCSpX79+mj9/fuBWBwB+ok8AbEWfANiMRgFwg6OhVPXq1XXw4EFJUkxMjNatWydJ2rlzp4wxgVsdAPiJPgGwFX0CYDMaBcANjoZSl112mf773/9KkgYNGqQRI0aoa9euuvHGG9WrV6+ALhAA/EGfANiKPgGwGY0C4AZHf31vxowZys7OliQNHjxYlSpV0po1a9S9e3cNHjw4oAsEAH/QJwC2ok8AbEajALjB0VAqJCREISH/9yKr3r17q3fv3gFbFAA4RZ8A2Io+AbAZjQLgBkdDqYSEBF1yySW69NJLlZCQoPLlywd6XQDgCH0CYCv6BMBmNAqAGxx9ptQ111yjjRs36vrrr9c555yjtm3b6sEHH9TSpUt17NixQK8RAHxGnwDYij4BsBmNAuAGR0Op0aNHa+nSpfr999+1evVq9ejRQ5s2bdK1116rypUrB3qNAOAz+gTAVvQJgM1oFAA3OHr73mkpKSn69ttv9e2332rz5s2KjIxUhw4dArU2AHCMPgGwFX0CYDMaBSCYHA2lbrzxRq1evVrZ2dnq2LGjOnbsqNGjR+v8888P9PoAwC/0CYCt6BMAm9EoAG5wNJR69913VaVKFQ0YMECdOnVShw4dVKFChUCvDQD8Rp8A2Io+AbAZjQLgBkefKXXo0CG9+uqryszM1NixY1WlShW1adNGDzzwgJYsWRLoNQKAz+gTAFvRJwA2o1EA3OBoKFWxYkVde+21ev7555WYmKgtW7aocePGev7553XNNdcEeo0A4DP6BMBW9AmAzWgUADc4evveoUOH9Pnnn2vVqlVatWqVtmzZokqVKqlHjx7q1KlToNcIAD6jTwBsRZ8A2IxGAXCDo6FU1apVVaVKFXXo0EF33HGHLr30UjVt2jTQawMAv9EnALaiTwBsRqMAuMHRUOrbb7/1KVBffvmlWrdurbCwMCdXAwB+o08AbEWfANiMRgFwg6PPlPJ1Yt6tWzft2bPHyVUAgCP0CYCt6BMAm9EoAG5wNJTylTGmMA8PAI7RJwC2ok8AbEajAARSoQ6lAAAAAAAAgLwwlAIAAAAAAEDQMZQCAAAAAABA0BXqUMrj8RTm4QHAMfoEwFb0CYDNaBSAQOKDzgGUSPQJgK3oEwCb0SgAgRRamAc/evRoYR4eAByjTwBsRZ8A2IxGAQgkv18p9e233+qJJ57QtGnTdODAgRyXpaen67bbbgvY4gDAH/QJgK3oEwCb0SgAbvFrKPXpp5/qoosu0oIFC/T0008rPj5eK1eu9F5+8uRJzZkzJ+CLBICzoU8AbEWfANiMRgFwk19DqXHjxmnUqFH6/vvvlZqaqvvvv1/XXnutli5dWljrAwCf0CcAtqJPAGxGowC4ya/PlNqyZYvefPNNSX/91YX77rtPtWvX1vXXX6/58+froosuKpRFAsDZ0CcAtqJPAGxGowC4ya+hVFhYmA4fPpxj20033aSQkBD16dNHEydODOTaAMBn9AmAregTAJvRKABu8uvtexdccEGO9xefduONN+rVV1/V0KFD/bry6dOn6/zzz1dkZKQiIyPVtm1bLVmyxK9jAIBEnwDYiz4BsBmNAuAmv14pdeedd2r16tV5XnbTTTdJkmbMmOHz8WrXrq2nnnpK9evXlyTNmTNHPXr0UFJSkpo0aeLP0gCUcPQJgK3oEwCb0SgAbvJrKNWrVy/16tUr38tvuukmb7h80b179xzfP/nkk5o+fbrWrVtHsAD4hT4BsBV9AmAzGgXATX4Npf4pMTFRycnJ8ng8io+PV8uWLR0fKysrS++++66OHz+utm3bFmRZAECfAFiLPgGwGY0CEEyOhlL79u1Tnz59tGrVKlWsWFHGGB05ckSdOnXSggULVLVqVZ+P9d1336lt27b6448/VKFCBS1atEiNGzfOc9+MjAxlZGR4v09PT3eyfADFmFt9kmgUgDOjTwBsxnM8AG7w64POT7vnnnuUnp6uLVu26NChQ/r999/1/fffKz093e8PwmvUqJE2bdqkdevW6c4771T//v31ww8/5LnvhAkTFBUV5f2Kjo52snwAxZhbfZJoFIAzo08AbMZzPABucDSUWrp0qaZPn674+HjvtsaNG+ull17y+y8rlClTRvXr11fr1q01YcIENW/eXC+88EKe+44ePVpHjhzxfu3evdvJ8gEUY271SaJRAM6MPgGwGc/xALjB0dv3srOzVbp06VzbS5curezs7AItyBiT4+WbfxcWFqawsLACHR/wVUpKio4ePerTvsnJyTn+1xcRERFq0KCBo7Uhf271SaJRKDh/uiM5a89pNCj46BOKq8I+Z5JoVjDwHA/FTTDOq2hTwTkaSl122WUaNmyY5s+fr5o1a0qS9uzZoxEjRqhz584+H+ehhx5St27dFB0draNHj2rBggVatWqVli5d6mRZQMCkpKSoYcOGfv9c3759/dp/+/btRCzA6BOKKqfdkfxvz2k0KLjoE4qjYJ0zSTSrsNEoFCfBPK+iTQXjaCg1depU9ejRQ3Xq1FF0dLQ8Ho927dqlZs2aae7cuT4f57ffflO/fv2UlpamqKgonX/++Vq6dKm6du3qZFlAwJyeqM+dOzfHS5jzc/LkSaWmpqpOnToKDw8/6/7Jycnq27evX5N7+IY+oajytzuS/+05jQa5gz6hOCrscyaJZgULjUJxEozzKtoUGI6GUtHR0dq4caOWL1+urVu3yhijxo0bq0uXLn4dZ9asWU6uHggaf/4MbkJCQiGvBr6gTyjq/P3z27Sn6KBPKM44Zyr6aBSKI86r7OdoKHVa165dmXgDsBJ9AmAr+gTAZjQKQDA5+ut7Q4cO1ZQpU3Jtnzp1qoYPH17QNQGAY/QJgK3oEwCb0SgAbnA0lHr//ffzfFlbu3bt9N577xV4UQDgFH0CYCv6BMBmNAqAGxwNpQ4ePKioqKhc2yMjI3XgwIECLwoAnKJPAGxFnwDYjEYBcIOjoVT9+vXz/JOeS5YsUWxsbIEXBQBO0ScAtqJPAGxGowC4wdEHnY8cOVJ333239u/fr8suu0yStGLFCk2cOFGTJ08O5PoAwC/0CYCt6BMAm9EoAG5wNJS67bbblJGRoSeffFKPP/64JKlOnTqaPn26br311oAuEAD8QZ8A2Io+AbAZjQLgBkdDKUm68847deedd2r//v0KDw9XhQoVArkuAHCMPgGwFX0CYDMaBSDYHA+lTqtatWog1oEASklJ0dGjR33aNzk5Ocf/+iIiIkINGjRwtDYgmOhT0eJPu05z0jCJjsF99MluhX0uJdEh2I1G2Y1GoThxPJR677339M4772jXrl06depUjss2btxY4IXBmZSUFDVs2NDvn+vbt69f+2/fvp1IwVr0qehx2q7T/G2YRMfgDvpkv2CdS0l0CPahUfajUShuHA2lpkyZojFjxqh///768MMPNXDgQP3000/asGGDhgwZEug1wg+nJ+Zz585VfHz8Wfc/efKkUlNTVadOHYWHh591/+TkZPXt29fvVzMAwUKfiiZ/23Wavw2T6BjcQ5+KhsI+l5LoEOxEo4oGGoXixtFQatq0aZoxY4ZuuukmzZkzR/fff79iY2P18MMP69ChQ4FeIxyIj49Xy5Ytfdo3ISGhkFcDBA99Ktr8addpNAxFBX0qWjiXQklDo4oWGoXiIsTJD+3atUvt2rWTJIWHh3snqP369dP8+fMDtzoA8BN9AmAr+gTAZjQKgBscDaWqV6+ugwcPSpJiYmK0bt06SdLOnTtljAnc6gDAT/QJgK3oEwCb0SgAbnA0lLrsssv03//+V5I0aNAgjRgxQl27dtWNN96oXr16BXSBAOAP+gTAVvQJgM1oFAA3OPpMqRkzZig7O1uSNHjwYFWqVElr1qxR9+7dNXjw4IAuEAD8QZ8A2Io+AbAZjQLgBkdDqZCQEIWE/N+LrHr37q3evXsHbFEA4BR9AmAr+gTAZjQKgBscvX1v9uzZevfdd3Ntf/fddzVnzpwCLwoAnKJPAGxFnwDYjEYBcIOjodRTTz2lKlWq5NperVo1jR8/vsCLAgCn6BMAW9EnADajUQDc4Ggo9fPPP6tu3bq5tsfExGjXrl0FXhQAOEWfANiKPgGwGY0C4AZHQ6lq1app8+bNubZ/++23qly5coEXBQBO0ScAtqJPAGxGowC4wdFQqk+fPho6dKhWrlyprKwsZWVl6bPPPtOwYcPUp0+fQK8RAHxGnwDYij4BsBmNAuAGR39974knntDPP/+szp07KzT0r0NkZ2fr1ltv5f3GAFxFnwDYij4BsBmNAuAGR0OpMmXK6O2339YTTzyhTZs2KTw8XM2aNVNMTEyg1wcAfqFPAGxFnwDYjEYBcIOjodRpDRo0UIMGDfK9PDIyUps2bVJsbGxBrgYA/EafANiKPgGwGY0CEEyOPlPKV8aYwjw8ADhGnwDYij4BsBmNAhBIhTqUAgAAAAAAAPLCUAoAAAAAAABBx1AKAAAAAAAAQVeoQymPx1OYhwcAx+gTAFvRJwA2o1EAAokPOgdQItEnALaiTwBsRqMABFKhDqWWLFmiWrVqFeZVAIAj9AmAregTAJvRKACBFOrkh7KysvT6669rxYoV2rdvn7Kzs3Nc/tlnn0mS2rdvX/AVAoAf6BMAW9EnADajUQDc4GgoNWzYML3++uu6+uqr1bRpU95XDMAa9AmAregTAJvRKABucDSUWrBggd555x1dddVVgV4PABQIfQJgK/oEwGY0CoAbHH2mVJkyZVS/fv1ArwUACow+AbAVfQJgMxoFwA2OhlL33nuvXnjhBf7yAgDr0CcAtqJPAGxGowC4wee371133XU5vv/ss8+0ZMkSNWnSRKVLl85x2cKFCwOzOgDwAX0CYCv6BMBmNAqA23weSkVFReX4vlevXgFfDAA4QZ8A2Io+AbAZjQLgNp+HUrNnzw74lU+YMEELFy7U1q1bFR4ernbt2unpp59Wo0aNAn5dAIov+gTAVvQJgM0C3Sj6BMBfjj5T6rR9+/bpiy++0Jo1a7Rv3z6/f/7zzz/XkCFDtG7dOi1fvlyZmZm6/PLLdfz48YIsCwDoEwBr0ScANitIo+gTAH/5/Eqpv0tPT9eQIUO0YMECZWVlSZJKlSqlG2+8US+99FKul4HmZ+nSpTm+nz17tqpVq6bExER17NjRydIAlHD0CYCt6BMAmwWiUfQJgL8cvVLq9ttv19dff62PPvpIhw8f1pEjR/TRRx/pm2++0R133OF4MUeOHJEkVapUyfExAJRs9AmAregTAJsVRqPoE4CzcfRKqY8//ljLli1T+/btvduuuOIKzZw5U1deeaWjhRhjNHLkSLVv315NmzbNc5+MjAxlZGR4v09PT3d0XQCKL7f6JNEoAGdGnwDYLNCNok8AfOHolVKVK1fO8+WbUVFROueccxwt5O6779bmzZs1f/78fPeZMGGCoqKivF/R0dGOrgtA8eVWnyQaBeDM6BMAmwW6UfQJgC8cDaXGjh2rkSNHKi0tzbtt7969uu+++/Sf//zH7+Pdc889Wrx4sVauXKnatWvnu9/o0aN15MgR79fu3budLB9AMeZWnyQaBeDM6BMAmwWyUfQJgK8cvX1v+vTp+vHHHxUTE6PzzjtPkrRr1y6FhYVp//79euWVV7z7bty4Md/jGGN0zz33aNGiRVq1apXq1q17xusNCwtTWFiYkyUDKCHc6pNEowCcGX0CYLNANIo+AfCXo6FUz549A3LlQ4YM0bx58/Thhx8qIiJCe/fulfTXS0TDw8MDch0AShb6BMBW9AmAzQLRKPoEwF+OhlKPPPJIQK58+vTpkqRLL700x/bZs2drwIABAbkOACULfQJgK/oEwGaBaBR9AuAvR0OpQDHGuHn1AJAv+gTAVvQJgK3oEwB/ORpKZWVladKkSXrnnXe0a9cunTp1Ksflhw4dCsjiAMBf9AmAregTAJvRKABucPTX9x599FE9//zz6t27t44cOaKRI0fquuuuU0hIiMaNGxfgJQKA7+gTAFvRJwA2o1EA3OBoKPXWW29p5syZGjVqlEJDQ3XTTTfp1Vdf1cMPP6x169YFeo0A4DP6BMBW9AmAzWgUADc4Gkrt3btXzZo1kyRVqFBBR44ckSRdc801+vjjjwO3OgDwE30CYCv6BMBmNAqAGxwNpWrXrq20tDRJUv369fXpp59KkjZs2KCwsLDArQ4A/ESfANiKPgGwGY0C4AZHQ6levXppxYoVkqRhw4bpP//5jxo0aKBbb71Vt912W0AXCAD+oE8AbEWfANiMRgFwg6O/vvfUU095//n6669XdHS0vvzyS9WvX1/XXnttwBYHAP6iTwBsRZ8A2IxGAXCD30OpP//8U//v//0//ec//1FsbKwkqU2bNmrTpk3AFwcA/qBPAGxFnwDYjEYBcIvfb98rXbq0Fi1aVBhrAYACoU8AbEWfANiMRgFwi+PPlPrggw8CvBQAKDj6BMBW9AmAzWgUADc4+kyp+vXr6/HHH9fatWvVqlUrlS9fPsflQ4cODcjiAMBf9AmAregTAJvRKABucDSUevXVV1WxYkUlJiYqMTExx2Uej4dgAXANfQJgK/oEwGY0CoAbHA2ldu7cGeh1AEBA0CcAtqJPAGxGowC4wdFQauTIkXlu93g8Klu2rOrXr68ePXqoUqVKBVocAPiLPgGwFX0CYDMaBcANjoZSSUlJ2rhxo7KystSoUSMZY5SSkqJSpUopLi5O06ZN07333qs1a9aocePGgV4zAOSLPgGwFX0CYDMaBcANjv76Xo8ePdSlSxf9+uuvSkxM1MaNG7Vnzx517dpVN910k/bs2aOOHTtqxIgRgV4vAJwRfQJgK/oEwGY0CoAbHA2lnn32WT3++OOKjIz0bouMjNS4ceP0zDPPqFy5cnr44YdzfUAeABQ2+gTAVvQJgM1oFAA3OBpKHTlyRPv27cu1ff/+/UpPT5ckVaxYUadOnSrY6gDAT/QJgK3oEwCb0SgAbnD89r3bbrtNixYt0i+//KI9e/Zo0aJFGjRokHr27ClJWr9+vRo2bBjItQLAWdEnALaiTwBsRqMAuMHRB52/8sorGjFihPr06aPMzMy/DhQaqv79+2vSpEmSpLi4OL366quBWykA+IA+AbAVfQJgMxoFwA2OhlIVKlTQzJkzNWnSJO3YsUPGGNWrV08VKlTw7nPBBRcEao0A4DP6BMBW9AmAzWgUADc4GkqdVqFCBZ1//vmBWgsABAx9AmAr+gTAZjQKQDA5+kwpAAAAAAAAoCAYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoXB1KrV69Wt27d1fNmjXl8Xj0wQcfuLkcAMiBRgGwFX0CYCv6BMAfrg6ljh8/rubNm2vq1KluLgMA8kSjANiKPgGwFX0C4I9QN6+8W7du6tatm5tLAIB80SgAtqJPAGxFnwD4g8+UAgAAAAAAQNC5+kopf2VkZCgjI8P7fXp6uourAYCcaBQAW9EnALaiT0DJVqReKTVhwgRFRUV5v6Kjo91eEgB40SgAtqJPAGxFn4CSrUgNpUaPHq0jR454v3bv3u32kgDAi0YBsBV9AmAr+gSUbEXq7XthYWEKCwtzexkAkCcaBcBW9AmAregTULK5OpQ6duyYfvzxR+/3O3fu1KZNm1SpUiWdd955Lq4MAGgUAHvRJwC2ok8A/OHqUOqbb75Rp06dvN+PHDlSktS/f3+9/vrrLq0KAP5CowDYij4BsBV9AuAPV4dSl156qYwxbi4BAPJFowDYij4BsBV9AuCPIvVB5wAAAAAAACgeGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOhC3V4AYCNP5h9qUT1E4Ye3S78GfnYbfni7WlQPkSfzj4AfG0DRVNjd+TsaBCBQgtEumgXAX7Sp6GAoBeSh7LFd2vjvCtLqf0urA3/8eEkb/11Bycd2SWoX+CsAUOQUdnf+jgYBCJRgtItmAfAXbSo6GEoBefijwnlq+coxvfXWW4qPiwv48ZO3btUtt9yiWVedF/BjAyiaCrs7f0eDAARKMNpFswD4izYVHQylgDyY0LJK2putkxUbSjUvCPjxT+7NVtLebJnQsgE/NoCiqbC783c0CECgBKNdNAuAv2hT0cEHnQMAAAAAACDoeKVUMcMHdAMoiviQbwC24MNxAdiMRqG4YShVzPAB3QCKIj7kG4At+HBcADajUShuGEoVM3xAN4CiiA/5BmALPhwXgM1oFIobhlLFDB/QDaAo4kO+AdiCD8cFYDMaheKGDzoHAAAAAABA0PFKKSAPJ06ckCRt3LjRp/1Pnjyp1NRU1alTR+Hh4WfdPzk5uUDrA1D8+Nsdyf/2nEaDAARKYZ8zSTQLgP+CcV5FmwKDoVQxwzAlMLZu3SpJuuOOOwr1eiIiIgr1+EBR4eTEQSpeT26C1Z2/o0FAbgxZ/BPMdtEsgEb5ijYVHQylihmGKYHRs2dPSVJcXJzKlSvn3Z6cnKy+ffv6fby5c+cqPj4+x7aIiAg1aNCgQOsEigsGMvl3R3LeHinv/kg0CMgPT2T8E4xzJolmAafRKN8UxnkVz+kKh8cYY9xehFPp6emKiorSkSNHFBkZ6fZyrHDgwAF98MEHDFMKyYkTJ7z/R/B3Z/svEHnFEP+nuD6Wi+vtKgz5tUtiICM5b49EfwKhOD6Wi+NtCpRAn0tJJfN8inOm4CiOj+XieJsCiUYVHH0KDl8fywylSggeeLBZcX0sF9fbFWz59UuiYQiO4vhYLo63qbAxHIaNiuNjuTjepmCgUbANQykARUZxfSwX19sFlDTF8bFcHG8TUBIVx8dycbxNQEnk62M5JIhrAgAAAAAAACQxlAIAAAAAAIALGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoQt1eQEEYYyRJ6enpLq8EQEGcfgyffkwXFzQKKB6KY6PoE1A80CcAtvK1T0V6KHX06FFJUnR0tMsrARAIR48eVVRUlNvLCBgaBRQvxalR9AkoXugTAFudrU8eU4TH6tnZ2fr1118VEREhj8fj9nKKpPT0dEVHR2v37t2KjIx0ezlFFvdjwRhjdPToUdWsWVMhIcXnXcU0qvCV9MdeSb/9wVIcG0WfAofHoX+4vwKLPuFseMz5jvsqsHztU5F+pVRISIhq167t9jKKhcjISB54AcD96Fxx+a97f0ejgqekP/ZK+u0PhuLWKPoUeDwO/cP9FTj0Cb7gMec77qvA8aVPxWOcDgAAAAAAgCKFoRQAAAAAAACCjqFUCRcWFqZHHnlEYWFhbi+lSON+BNxR0h97Jf32Azbgcegf7i8guHjM+Y77yh1F+oPOAQAAAAAAUDTxSikAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQ6kibtq0aapbt67Kli2rVq1a6Ysvvsh334ULF6pr166qWrWqIiMj1bZtWy1btizXPq1bt1bFihVVvnx5XXDBBXrzzTdzHWvPnj3q27evKleurHLlyumCCy5QYmJiwG9fsPhzPw4YMEAejyfXV5MmTfLcf8GCBfJ4POrZs2eO7ePGjct1jOrVqwfyZgHW8+exl5aWpptvvlmNGjVSSEiIhg8fnmufP//8U4899pjq1aunsmXLqnnz5lq6dGmOfTIzMzV27FjVrVtX4eHhio2N1WOPPabs7OxA37yz8uf2r1q1Ks/2bN26Ncd+hw8f1pAhQ1SjRg2VLVtW8fHx+uSTT7yXr169Wt27d1fNmjXl8Xj0wQcfFNbNA4oEfx6Ha9asUUJCgipXrqzw8HDFxcVp0qRJOfbZsmWL/vWvf6lOnTryeDyaPHlyruMU5cdhYZwzna1bnDOhJAv0872/y+95ik3nSv7w576SpIyMDI0ZM0YxMTEKCwtTvXr19Nprr3kv9+W8sij33BYMpYqwt99+W8OHD9eYMWOUlJSkDh06qFu3btq1a1ee+69evVpdu3bVJ598osTERHXq1Endu3dXUlKSd59KlSppzJgx+uqrr7R582YNHDhQAwcOzBGz33//XQkJCSpdurSWLFmiH374QRMnTlTFihUL+yYXCn/vxxdeeEFpaWner927d6tSpUq64YYbcu37888/a9SoUerQoUOex2rSpEmOY3333XcBvW2Azfx97GVkZKhq1aoaM2aMmjdvnuc+Y8eO1SuvvKIXX3xRP/zwgwYPHqxevXrl6NzTTz+tl19+WVOnTlVycrKeeeYZPfvss3rxxRcL5Xbmx9/bf9q2bdtydKNBgwbey06dOqWuXbsqNTVV7733nrZt26aZM2eqVq1a3n2OHz+u5s2ba+rUqYV224Ciwt/HYfny5XX33Xdr9erVSk5O1tixYzV27FjNmDHDu8+JEycUGxurp556Kt/BSVF9HBbGOZMv3ZI4Z0LJVBjP90470/MUW86V/OHkvKp3795asWKFZs2apW3btmn+/PmKi4vzXu7LeWVR7blVDIqsiy66yAwePDjHtri4OPPggw/6fIzGjRubRx999Iz7tGjRwowdO9b7/QMPPGDat2/v32ItVtD7cdGiRcbj8ZjU1NQc2zMzM01CQoJ59dVXTf/+/U2PHj1yXP7II4+Y5s2bF2TpQJFWkMfeJZdcYoYNG5Zre40aNczUqVNzbOvRo4e55ZZbvN9fffXV5rbbbsuxz3XXXWf69u3rx+oLzt/bv3LlSiPJ/P777/kec/r06SY2NtacOnXKpzVIMosWLfJ1yUCxE4hzqV69euXbj5iYGDNp0qQz/nxRehwWxjmTL93inAklVWE93zvb8xRbzpX84e99tWTJEhMVFWUOHjyY7zF9Oa/8u6LUc5vwSqki6tSpU0pMTNTll1+eY/vll1+utWvX+nSM7OxsHT16VJUqVcrzcmOMVqxYoW3btqljx47e7YsXL1br1q11ww03qFq1amrRooVmzpzp/Ma4KBD346xZs9SlSxfFxMTk2P7YY4+patWqGjRoUL4/m5KSopo1a6pu3brq06ePduzY4f+NAIqgQDz28pKRkaGyZcvm2BYeHq41a9Z4v2/fvr1WrFih7du3S5K+/fZbrVmzRldddZXj6/VXQW5/ixYtVKNGDXXu3FkrV67McdnixYvVtm1bDRkyROeee66aNm2q8ePHKysrK+C3ASjqAtGhpKQkrV27VpdccklhLNEqhXXO5Gu3OGdCSVOYz/fO9jzFhnMlfzi5r04/p33mmWdUq1YtNWzYUKNGjdLJkye9+/hyXomCC3V7AXDmwIEDysrK0rnnnptj+7nnnqu9e/f6dIyJEyfq+PHj6t27d47tR44cUa1atZSRkaFSpUpp2rRp6tq1q/fyHTt2aPr06Ro5cqQeeughrV+/XkOHDlVYWJhuvfXWgt+4ICro/ZiWlqYlS5Zo3rx5ObZ/+eWXmjVrljZt2pTvz7Zp00ZvvPGGGjZsqN9++01PPPGE2rVrpy1btqhy5cqObg9QVASiYXm54oor9Pzzz6tjx46qV6+eVqxYoQ8//DDHk5sHHnhAR44cUVxcnEqVKqWsrCw9+eSTuummmxxfr7+c3P4aNWpoxowZatWqlTIyMvTmm2+qc+fOWrVqlfc/HOzYsUOfffaZbrnlFn3yySdKSUnRkCFDlJmZqYcffrjQbxdQlBSkQ7Vr19b+/fuVmZmpcePG6fbbby/MpVqhsM6ZfOkW50woiQrr+Z4vz1NsOFfyh5P7aseOHVqzZo3Kli2rRYsW6cCBA7rrrrt06NAh7+dK+XJeiYJjKFXEeTyeHN8bY3Jty8v8+fM1btw4ffjhh6pWrVqOyyIiIrRp0yYdO3ZMK1as0MiRIxUbG6tLL71U0l8T99atW2v8+PGS/vqv9lu2bNH06dOL3FDqNKf34+uvv66KFSvm+HDAo0ePqm/fvpo5c6aqVKmS789269bN+8/NmjVT27ZtVa9ePc2ZM0cjR470/0YARZDTx15+XnjhBd1xxx2Ki4uTx+NRvXr1NHDgQM2ePdu7z9tvv625c+dq3rx5atKkiTZt2qThw4erZs2a6t+/v+PrdsKf29+oUSM1atTI+33btm21e/duPffcc96hVHZ2tqpVq6YZM2aoVKlSatWqlX799Vc9++yzDKWAfDjp0BdffKFjx45p3bp1evDBB1W/fn1rn6wFWiDPmSTfusU5E0qyQD7f8/V5ik3nSv7w577Kzs6Wx+PRW2+9paioKEnS888/r+uvv14vvfSSwsPDfTqvRMExlCqiqlSpolKlSuWa/O7bty/XhPif3n77bQ0aNEjvvvuuunTpkuvykJAQ1a9fX5J0wQUXKDk5WRMmTPAOpWrUqKHGjRvn+Jn4+Hi9//77BbhF7ijI/WiM0WuvvaZ+/fqpTJky3u0//fSTUlNT1b17d++203+pIjQ0VNu2bVO9evVyHa98+fJq1qyZUlJSCnKTgCKhII+9M6latao++OAD/fHHHzp48KBq1qypBx98UHXr1vXuc9999+nBBx9Unz59JP31BOfnn3/WhAkTgnaiFajbf/HFF2vu3Lne72vUqKHSpUurVKlS3m3x8fHau3evTp06laNVQElXkMfh6aY0a9ZMv/32m8aNG1fsh1KFcc4kOesW50woCQrj+Z6vz1NsOFfyh5P7qkaNGqpVq5Z3ICX91R5jjH755Rc1aNDAp/NKFByfKVVElSlTRq1atdLy5ctzbF++fLnatWuX78/Nnz9fAwYM0Lx583T11Vf7dF3GGGVkZHi/T0hI0LZt23Lss3379lyfqVQUOL0fJenzzz/Xjz/+mOu92HFxcfruu++0adMm79e1116rTp06adOmTYqOjs7zeBkZGUpOTlaNGjUKdqOAIqAgjz1flC1bVrVq1VJmZqbef/999ejRw3vZiRMnFBKS8//+SpUqFdQ/cxyo25+UlJSjGQkJCfrxxx9z3Jbt27erRo0aDKSAfwjU4/Cf50nFVWGcM0nOusU5E0qCwni+5+vzFBvOlfzh5L5KSEjQr7/+qmPHjnm3bd++XSEhIapdu3aOfc90XokAcOPT1REYCxYsMKVLlzazZs0yP/zwgxk+fLgpX7689y+aPPjgg6Zfv37e/efNm2dCQ0PNSy+9ZNLS0rxfhw8f9u4zfvx48+mnn5qffvrJJCcnm4kTJ5rQ0FAzc+ZM7z7r1683oaGh5sknnzQpKSnmrbfeMuXKlTNz584N3o0PIH/vx9P69u1r2rRp49N15PVXLe69916zatUqs2PHDrNu3TpzzTXXmIiIiFx/xQ8orpw89pKSkkxSUpJp1aqVufnmm01SUpLZsmWL9/J169aZ999/3/z0009m9erV5rLLLjN169bN8Rfr+vfvb2rVqmU++ugjs3PnTrNw4UJTpUoVc//99wfldp/m7+2fNGmSWbRokdm+fbv5/vvvzYMPPmgkmffff9+7z65du0yFChXM3XffbbZt22Y++ugjU61aNfPEE0949zl69Kj3fpRknn/+eZOUlGR+/vnn4N14wBL+Pg6nTp1qFi9ebLZv3262b99uXnvtNRMZGWnGjBnj3ScjI8P7GKtRo4YZNWqUSUpKMikpKd59iurjsDDOmXzpFudMKKkK4/neP+X1PMWWcyV/+HtfHT161NSuXdtcf/31ZsuWLebzzz83DRo0MLfffrt3H1/OK4tqz23CUKqIe+mll0xMTIwpU6aMadmypfn888+9l/Xv399ccskl3u8vueQSIynXV//+/b37jBkzxtSvX9+ULVvWnHPOOaZt27ZmwYIFua73v//9r2natKkJCwszcXFxZsaMGYV5MwudP/ejMcYcPnzYhIeH+3y784r9jTfeaGrUqGFKly5tatasaa677rocT66BksDfx15eDYuJifFevmrVKhMfH2/CwsJM5cqVTb9+/cyePXtyHCM9Pd0MGzbMnHfeeaZs2bImNjbWjBkzxmRkZBTmTc2TP7f/6aefNvXq1fP2uX379ubjjz/Odcy1a9eaNm3amLCwMBMbG2uefPJJk5mZ6b185cqVZ/3/AqAk8edxOGXKFNOkSRNTrlw5ExkZaVq0aGGmTZtmsrKyvPvs3Lkzz8fY349TlB+HhXHOdLZucc6EkizQz/f+Ka/nKTadK/nD3z4lJyebLl26mPDwcFO7dm0zcuRIc+LECe/lvpxXFuWe28JjjDGF+EIsAAAAAAAAIBc+UwoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCrlceumlGj58uM/7p6amyuPxaNOmTYW2JklatWqVPB6PDh8+XKjXA8Ae9AhAUUO37FDSbi9QEHTLLh6PRx988IGk4N3XbmIoVUgGDBggj8cjj8ej0qVLKzY2VqNGjdLx48cL5bp69uwZsOMtXLhQjz/+uM/7R0dHKy0tTU2bNg3YGlBwK1asULt27RQREaEaNWrogQceUGZmptvLggvoEXwR6N/daRMmTNCFF16oiIgIVatWTT179tS2bdty7LNw4UJdccUVqlKlSrE/8YJv6BYKql27dkpLS1NUVJTbS0EJQbcAZxhKFaIrr7xSaWlp2rFjh5544glNmzZNo0aNynPfP//8s9DX4+t1VKpUSRERET4ft1SpUqpevbpCQ0OdLg0BtnnzZl111VW68sorlZSUpAULFmjx4sV68MEH3V4aXEKP4JbPP/9cQ4YM0bp167R8+XJlZmbq8ssvz3GSfvz4cSUkJOipp55ycaWwDd1CQZQpU0bVq1eXx+NxeykoQegW4IBBoejfv7/p0aNHjm233367qV69ujHGmEceecQ0b97czJo1y9StW9d4PB6TnZ1tDh8+bO644w5TtWpVExERYTp16mQ2bdqU7/U88sgjRlKOr5UrV5qdO3caSebtt982l1xyiQkLCzOvvfaaOXDggOnTp4+pVauWCQ8PN02bNjXz5s3LccxLLrnEDBs2zPt9TEyMefLJJ83AgQNNhQoVTHR0tHnllVe8l5++rqSkJGOMMStXrjSSzP/+9z/TqlUrEx4ebtq2bWu2bt2a43oef/xxU7VqVVOhQgUzaNAg88ADD5jmzZvne1tPH/f333/3bnvvvfdM48aNTZkyZUxMTIx57rnncvzMSy+9ZOrXr2/CwsJMtWrVzL/+9S/vZe+++65p2rSpKVu2rKlUqZLp3LmzOXbsWL7Xv2rVKnPhhReaMmXKmOrVq5sHHnjA/Pnnnznut3vuucfcd9995pxzzjHnnnuueeSRR/I9njHGZGZmmhEjRpioqChTqVIlc99995lbb701x787l1xyibn77rvNsGHDTMWKFU21atXMK6+8Yo4dO2YGDBhgKlSoYGJjY80nn3zi/ZnRo0eb1q1b57iuRYsWmbJly5r09PQzrgnFDz2iR2frUX6/O2OM2bx5s+nUqZN3bXfccYc5evSo934oXbq0Wb16tfdYzz33nKlcubL59ddf87yuffv2GUnm888/z3XZP39/KLnoFt3y5Tzqn787SSYmJibP2zt79mwTFRVlli5dauLi4kz58uXNFVdckatVs2bN8t4n1atXN0OGDDnjGoDT6Bbd8qVb69evN126dDGVK1c2kZGRpmPHjiYxMTHHPpLMokWLjDEl49yIoVQhyStK99xzj6lcubIx5q+YnP4/w40bN5pvv/3WZGdnm4SEBNO9e3ezYcMGs337dnPvvfeaypUrm4MHD+Z5PUePHjW9e/c2V155pUlLSzNpaWkmIyPD+y9vnTp1zPvvv2927Nhh9uzZY3755Rfz7LPPmqSkJPPTTz+ZKVOmmFKlSpl169Z5j5lXlCpVqmReeuklk5KSYiZMmGBCQkJMcnKyMSb/KLVp08asWrXKbNmyxXTo0MG0a9fOe8y5c+easmXLmtdee81s27bNPProoyYyMtKvKH3zzTcmJCTEPPbYY2bbtm1m9uzZJjw83MyePdsYY8yGDRtMqVKlzLx580xqaqrZuHGjeeGFF4wxxvz6668mNDTUPP/882bnzp1m8+bN5qWXXvI+0fqnX375xZQrV87cddddJjk52SxatMhUqVIlR3QuueQSExkZacaNG2e2b99u5syZYzwej/n000/zvU1PP/20iYqKMu+995754YcfzKBBg0xERESuoVRERIR5/PHHzfbt283jjz9uQkJCTLdu3cyMGTPM9u3bzZ133mkqV65sjh8/bowxZuTIkaZ9+/Y5rmvp0qU5nmii5KBH9OhsPcrvd3f8+HFTs2ZNc91115nvvvvOrFixwtStW9f079/f+7P33XefiYmJMYcPHzabNm0yYWFhZuHChfnedykpKUaS+e6773JdVhJOvOAbukW3fDmPOv07S0tLMz/++KOpX7++6devX563d/bs2aZ06dKmS5cuZsOGDSYxMdHEx8ebm2++2Xu8adOmmbJly5rJkyebbdu2mfXr15tJkyble/3A39EtuuVLt1asWGHefPNN88MPP3if/5177rk5XjjAUAoB8c8off3116Zy5cqmd+/expi/olS6dGmzb98+7z4rVqwwkZGR5o8//shxrHr16uWYTJ/tuoz5v395J0+efNa1XnXVVebee+/1fp9XlPr27ev9Pjs721SrVs1Mnz49x3XlNSk/7eOPPzaSzMmTJ40xxrRp0ybXf3lKSEjwK0o333yz6dq1a4597rvvPtO4cWNjjDHvv/++iYyMzPOVQYmJiUaSSU1Nzff6/u6hhx4yjRo1MtnZ2d5tL730kqlQoYLJysoyxvx1v/1zEHThhReaBx54IN/j1qhRwzz11FPe7//8809Tu3btXEOpvx83MzPTlC9f3nvSZcxfJ2WSzFdffWWMMWbZsmUmJCTEzJs3z2RmZppffvnFtG/f3kjK9V9GUPzRI3pkzNl7lNfvbsaMGeacc87J8V8RP/74YxMSEmL27t1rjDEmIyPDtGjRwvTu3ds0adLE3H777fleR3Z2tunevXuutZ1WEk684Bu6RbeMOXu3TsvOzja9evUyrVq1MidOnMjz9s6ePdtIMj/++GOONZx77rne72vWrGnGjBnj020C/olu0S1jfO/WaZmZmSYiIsL897//9W4raUMpPlOqEH300UeqUKGCypYtq7Zt26pjx4568cUXvZfHxMSoatWq3u8TExN17NgxVa5cWRUqVPB+7dy5Uz/99JN27dqVY/v48ePPuobWrVvn+D4rK0tPPvmkzj//fO/1fPrpp9q1a9cZj3P++ed7/9nj8ah69erat2+fzz9To0YNSfL+zLZt23TRRRfl2P+f359NcnKyEhIScmxLSEhQSkqKsrKy1LVrV8XExCg2Nlb9+vXTW2+9pRMnTkiSmjdvrs6dO6tZs2a64YYbNHPmTP3+++9nvK62bdvm+FyChIQEHTt2TL/88kuet/n07c7vfjpy5IjS0tLUtm1b77bQ0NBcv7N/HrdUqVKqXLmymjVr5t127rnnSvq/+/fyyy/Xs88+q8GDByssLEwNGzbU1Vdf7f15lDz0iB6dqUdnuq7mzZurfPnyOa4rOzvb+2HlZcqU0dy5c/X+++/r5MmTmjx5cr7Hu/vuu7V582bNnz/fr3WgZKJbdMvXbj300EP66quv9MEHHyg8PDzf/cqVK6d69erlefx9+/bp119/VefOnc96fUB+6BbdOlu39u3bp8GDB6thw4aKiopSVFSUjh07dtbfR3HGJ5MVok6dOmn69OkqXbq0atasqdKlS+e4/O8n+ZKUnZ2tGjVqaNWqVbmOVbFiRVWsWDHHXySqVKnSWdfwz+uYOHGiJk2apMmTJ6tZs2YqX768hg8frlOnTp3xOP9cu8fjUXZ2ts8/c/rB/Pef+ecHTxpjzni8fzLGnPEYERER2rhxo1atWqVPP/1UDz/8sMaNG6cNGzaoYsWKWr58udauXatPP/1UL774osaMGaOvv/5adevW9eu6/r7dyf3ki7yOe7b7d+TIkRoxYoTS0tJ0zjnnKDU1VaNHj87z9qH4o0f0yEmP8rquvx/vtLVr10qSDh06pEOHDuX6XUvSPffco8WLF2v16tWqXbu2X+tAyUS36JYv99PcuXM1adIkrVq16qxtyev4p9dxpmEW4Cu6RbfOdj8NGDBA+/fv1+TJkxUTE6OwsDC1bdv2rL+P4oxXShWi8uXLq379+oqJicn1L2teWrZsqb179yo0NFT169fP8VWlSpVc209HqUyZMsrKyvJpTV988YV69Oihvn37qnnz5oqNjVVKSkqBbqcTjRo10vr163Ns++abb/w6RuPGjbVmzZoc29auXauGDRt6Xw0UGhqqLl266JlnntHmzZuVmpqqzz77TNJfwUhISNCjjz6qpKQklSlTRosWLcr3utauXZsjemvXrlVERIRq1arl17pPi4qKUo0aNbRu3TrvtszMTCUmJjo6Xl48Ho9q1qyp8PBwzZ8/X9HR0WrZsmXAjo+igx7ljx79Ja/fXePGjbVp06Ycfynvyy+/VEhIiBo2bChJ+umnnzRixAjNnDlTF198sW699dYcJ2PGGN19991auHChPvvsMwbj8Bndyh/d+stXX32l22+/Xa+88oouvvhix8eR/noyW6dOHa1YsaJAx0HJRrfyR7f+8sUXX2jo0KG66qqr1KRJE4WFhenAgQOOj1ccMJSySJcuXdS2bVv17NlTy5YtU2pqqtauXauxY8ee8QFbp04dbd68Wdu2bdOBAwfO+Kc/69ev750QJycn69///rf27t1bGDfnjO655x7NmjVLc+bMUUpKip544glt3rzZrz/be++992rFihV6/PHHtX37ds2ZM0dTp071/tnVjz76SFOmTNGmTZv0888/64033lB2drYaNWqkr7/+WuPHj9c333yjXbt2aeHChdq/f7/i4+PzvK677rpLu3fv1j333KOtW7fqww8/1COPPKKRI0cqJMT5w2jYsGF66qmntGjRIm3dulV33XWXDh8+7Ph4f/fss8/qu+++05YtW/T444/rqaee0pQpU3j7HnxCj0pej/L63d1yyy0qW7as+vfvr++//14rV67UPffco379+uncc89VVlaW+vXrp8svv1wDBw7U7Nmz9f3332vixIne4w4ZMkRz587VvHnzFBERob1792rv3r06efKkd59Dhw5p06ZN+uGHHyT99RL/TZs2ufLvA4ouulWyurV371716tVLffr00RVXXOFty/79+x0dT5LGjRuniRMnasqUKUpJSdHGjRtzvPUKCDS6VbK6Jf31+3jzzTeVnJysr7/+WrfcckuJf6UmQymLeDweffLJJ+rYsaNuu+02NWzYUH369FFqaqr3M4Pycscdd6hRo0Zq3bq1qlatqi+//DLfff/zn/+oZcuWuuKKK3TppZeqevXq6tmzZyHcmjO75ZZbNHr0aI0aNUotW7bUzp07NWDAAJUtW9bnY7Rs2VLvvPOOFixYoKZNm+rhhx/WY489pgEDBkj66yWvCxcu1GWXXab4+Hi9/PLLmj9/vpo0aaLIyEitXr1aV111lRo2bKixY8dq4sSJ6tatW57XVatWLX3yySdav369mjdvrsGDB2vQoEEaO3Zsge6He++9V7feeqsGDBigtm3bKiIiQr169SrQMU9bsmSJOnTooNatW+vjjz/Whx9+6MrvGkUTPSp5Pcrrd1euXDktW7ZMhw4d0oUXXqjrr79enTt31tSpUyVJTz75pFJTUzVjxgxJUvXq1fXqq69q7Nix3rcbTJ8+XUeOHNGll16qGjVqeL/efvtt73UvXrxYLVq08H72XZ8+fdSiRQu9/PLLBbpNKFnoVsnq1tatW/Xbb79pzpw5Odpy4YUXOj5m//79NXnyZE2bNk1NmjTRNddc48orSlBy0K2S1S1Jeu211/T777+rRYsW6tevn4YOHapq1aoV6JhFncf4+0ZOoJB07dpV1atX15tvvun2Ulw1YMAAHT58WB988IHbSwFKLHoEoKihWwCKGroFiQ86h0tOnDihl19+WVdccYVKlSql+fPn63//+5+WL1/u9tIAlDD0CEBRQ7cAFDV0C/lhKAVXnH6p6hNPPKGMjAw1atRI77//vrp06eL20gCUMPQIQFFDtwAUNXQL+eHtewAAAAAAAAg6PugcAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABB9/8BTFHE/p4+VYsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, axs = plt.subplots(nrows=1, ncols=4, figsize=(12, 4))\n", + "for i, loss_label in enumerate(PT_TASKS + [\"all\"]):\n", + " draw_boxplot(results, \"caco2\", loss_label=loss_label, ax=axs[i])\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAGGCAYAAACqvTJ0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABjMklEQVR4nO3deZyN9f//8ecxmBmMdSxjacbaEFlbpBKh1CfRIhXhg+/HR6XSxkeLRJRKScmWFqUNrSp7KokGH6mxhMmeJDO2Rmbevz/6zfk0zeCca65zrvfMedxvt7llrnOd67zO6Dy65t1ZfMYYIwAAAAAAACCMink9AAAAAAAAACIPi1IAAAAAAAAIOxalAAAAAAAAEHYsSgEAAAAAACDsWJQCAAAAAABA2LEoBQAAAAAAgLBjUQoAAAAAAABhx6IUAAAAAAAAwq641wOEU3Z2tnbv3q24uDj5fD6vxwHw/xljdOjQIVWvXl3FikXmWjl9AuwV6Y2iT4C9Ir1PEo0CbBVonyJqUWr37t2qVauW12MAOIkdO3aoZs2aXo/hCfoE2C9SG0WfAPtFap8kGgXY7nR9iqhFqbi4OEl//lDKli3r8TQAcmRkZKhWrVr+x2gkok+AvSK9UfQJsFek90miUYCtAu1TRC1K5Tyds2zZsgQLsFAkP+WaPgH2i9RG0SfAfpHaJ4lGAbY7XZ8i84XHAAAAAAAA8BSLUgAAAAAAAAg7FqUAAAAAAAAQdixKAQAAAAAAIOxYlAIAAAAAAEDYOVqUqlOnjn799dc82w8ePKg6deoUeCgAcIo+AbAVfQJgMxoFwAuOFqXS0tKUlZWVZ3tmZqZ27dpV4KEAwCn6BMBW9AmAzWgUAC8UD2bnDz74wP/nzz77TOXKlfN/n5WVpUWLFikpKcm14QAgUPQJgK3oEwCb0SgAXgpqUapr166SJJ/Pp969e+e6rESJEkpKStJTTz3l2nAAECj6BMBW9AmAzWgUAC8FtSiVnZ0tSapdu7ZWrVql+Pj4kAwFAMGiTwBsRZ8A2IxGAfBSUItSObZt25Zn28GDB1W+fPmCzgMABUKfANiKPgGwGY0C4AVHi1KPP/64kpKSdMMNN0iSrr/+es2ePVsJCQmaN2+emjZt6uqQCL2jR49qw4YNebYfO3ZMaWlpSkpKUmxsbK7LkpOTVapUqXCNCASEPkUeJ/2SaBjCjz4VbbQIhR2NKtpoFGzlaFFq8uTJmjlzpiRpwYIFWrhwoT799FO9/fbbuvfeezV//nxXh0TobdiwQS1btgzqOikpKWrRokWIJgKcoU+Rx0m/JBqG8KNPRRstQmFHo4o2GgVbOVqU2rNnj2rVqiVJ+uijj9S9e3d16tRJSUlJOu+881wdEOGRnJyslJSUPNtTU1PVs2dPzZw5Uw0bNsxzHcA29CnyOOlXzvWAcKJPRRstQmFHo4o2GgVbOVqUqlChgnbs2KFatWrp008/1ahRoyRJxhhlZWW5OiDCo1SpUqdcAW/YsCEr5CgU6FPkoV8oLOhT0UaLUNjRqKKNRsFWjhalrrnmGt10002qX7++fv31V3Xu3FmStHbtWtWrV8/VAQEgGPQJgK3oEwCb0SgAXnC0KDV+/HglJSVpx44deuKJJ1SmTBlJfz7lc9CgQa4OCADBoE8AbEWfANiMRgHwgqNFqRIlSuiee+7Js/3OO+8s6DwAUCD0CYCt6BMAm9EoAF4o5vSKr732mi688EJVr15dP/30kyTpmWee0fvvv+/acADgBH0CYCv6BMBmNApAuDlalJo0aZKGDBmizp076+DBg/43vitfvryeeeYZN+cDgKDQJwC2ok8AbEajAHjB0aLUc889p6lTp2r48OGKioryb2/VqpW+++4714YDgGDRJwC2ok8AbEajAHjB0aLUtm3b1Lx58zzbo6OjdeTIkQIPBQBO0ScAtqJPAGxGowB4wdGiVO3atbV27do82z/55BM1atSooDMBgGP0CYCt6BMAm9EoAF5w9Ol79957r2699Vb9/vvvMsZo5cqVmjVrlsaMGaNp06a5PSMABIw+AbAVfQJgMxoFwAuOFqX69u2rEydO6L777tPRo0d10003qUaNGnr22WfVo0cPt2cEgIDRJwC2ok8AbEajAHjB0aKUJA0YMEADBgzQ/v37lZ2drSpVqrg5FwA4Rp8A2Io+AbAZjQIQbo7eU6p9+/Y6ePCgJCk+Pt4fq4yMDLVv39614QAgWPQJgK3oEwCb0SgAXnC0KLV06VIdP348z/bff/9dX3zxRYGHAgCn6BMAW9EnADajUQC8ENTL99atW+f/8w8//KC9e/f6v8/KytKnn36qGjVquDcdQmLz5s06dOhQQPumpqbm+ufpxMXFqX79+o5nA5yiT0VfMO2Sgu9XDjoGt9GnoieU51ISHUJ40aiih0ahMAlqUapZs2by+Xzy+Xz5PoUzNjZWzz33nGvDwX2bN29WgwYNgr5ez549A95306ZNRAphR5+KNqftkoLrVw46BjfRp6IlHOdSEh1C+NCoooVGobAJalFq27ZtMsaoTp06WrlypSpXruy/rGTJkqpSpYqioqJcHxLuyVkxnzlzpho2bHja/Y8dO6a0tDQlJSUpNjb2lPumpqaqZ8+eQT2TAXALfSragm2XFFy/ctAxhAJ9KlpCeS4l0SGEH40qWmgUCpugFqUmT56srl27Kjs7O1TzIEwaNmyoFi1aBLRvmzZtQjwNUHD0KTIE0y6JfsEO9Klo4lwKRQWNKppoFAqLoN7ofM+ePfrHP/6hhIQE/d///Z8+/vhjZWZmhmo2AAgYfQJgK/oEwGY0CoCXglqUmjFjhn7++We9/fbbKl++vO6++27Fx8frmmuu0csvv6z9+/eHak6/F154QbVr11ZMTIxatmzJJ0EAkESfANjLhj5JNApA/mxoFH0CIldQL9+TJJ/Pp4suukgXXXSRnnjiCaWmpurDDz/U1KlT9a9//UvnnXeeunTpohtvvNH1T2l46623dOedd+qFF15QmzZtNHnyZHXu3Fk//PCDzjjjDFdvCwjE0aNHtWHDhjzbT/Xa7OTkZJUqVSpcI0YU+oRIcbL2SKd/bwga5A0v+yTRKHjPyTmTRLPChXMoRDL65DHjop9//tlMmzbNdOnSxYwbN87NQxtjjDn33HPNwIEDc21LTk42Q4cODej66enpRpJJT093fbbCIiUlxUgyKSkpherYtsq5z8F8RdLPJ1DheGzSp8ItXH0pLB1z0h4a5FyoH5+h7pMxBWsUfcot1J0oLB0KltNuFbWfg9s4h6JRf0ejgkefQiPQx2bQz5T6qx9//FFbtmzRxRdfrNjYWFWuXFn9+vVTv379CnLYfB0/flwpKSkaOnRoru2dOnXS8uXL871OZmZmrtdDZ2RkuD4XIltycrJSUlLybM/5VIr8PvUiOTk5XONFNPqEouxk7ZFO3Z+c68Jb4eyTFHyj6BNCwck5U871EF6cQyHS0CdvOVqU+vXXX3XDDTdo8eLF8vl82rx5s+rUqaP+/furQoUKevLJJ92eU/v371dWVpaqVq2aa3vVqlW1d+/efK8zZswYPfLII67PAuQoVarUKT/VIthPCkPB0SdEgtO1R6I/NvKiT1LwjaJPCAXOmezHORQiFX3yVlBvdJ7jrrvuUvHixbV9+/Zcr6G84YYb9Mknn7g2XH58Pl+u740xebblGDZsmNLT0/1fO3bsCOlsALxHnwDYyss+SYE3ij4BkYlzKABecPRMqfnz5+uzzz5TzZo1c22vX7++fvrpJ1cG+7v4+HhFRUXlWTHft29fnpX1HNHR0YqOjg7JPIWV78Tval6tmGIPbpJ2O1qTPKnYg5vUvFox+U787upxgWDQp6IplO36KzqGUPKiT1LwjaJPpxbqHtEheIVzqKKBRqGwcbQodeTIkXzfZX7//v0hC0TJkiXVsmVLLViwQN26dfNvX7Bgga6++uqQ3GZRFHN4u1b/q4y07F/SMneP3VDS6n+VUerh7ZIucPfgQIDoU9EUynb9FR1DKHnRJ4lGuS3UPaJD8ArnUEUDjUJh42hR6uKLL9arr76qRx99VNKfT7fMzs7WuHHj1K5dO1cH/KshQ4aoV69eatWqlVq3bq0pU6Zo+/btGjhwYMhus6j5vcwZajH5sF5//XU1dPmN2VI3bNDNN9+s6VcUzY9u3bx5sw4dOhTQvqmpqbn+eTpxcXGqX7++49nwP/SpaAplu/7Kxo4F0x4p+P5INChcvOqTRKPcFOoe2dihYIXynEmiWaHCOVTRQKNOjT7Zx9Gi1Lhx43TJJZfo22+/1fHjx3Xffffp+++/14EDB/TVV1+5PaPfDTfcoF9//VUjR47Unj171LhxY82bN0+JiYkhu82ixhSP0Zq92TpWvoFUvZmrxz62N1tr9mbLFI9x9bg22Lx5sxo0aBD09Xr27Bnwvps2bSJgLqBPRVMo2/VXtnXMaXuk4Poj0aBw8KpPEo1yU6h7ZFuHghWOcyaJZoUC51BFA406OfpkJ0eLUo0aNdK6des0adIkRUVF6ciRI7rmmmt06623KiEhwe0Zcxk0aJAGDRoU0tsA/i5nNf1kHwf6d8eOHVNaWpqSkpIUGxt7yn1zPmo0mGdC4OToE4qSYNsjBdcfiQaFk5d9kmgUwiOU50wSzQolzqFQ1NEnOwW9KPXHH3+oU6dOmjx5Mh/FiYgTzMeBtmnTJsTT4O/oE4qqYD+KmP7Yhz4h0nDOVLjQKEQS+mSXoN+Ov0SJElq/fv1JP6ITALxCnwDYij4BsBmNAuAVR58Recstt2j69OluzwIABUafANiKPgGwGY0C4AVH7yl1/PhxTZs2TQsWLFCrVq1UunTpXJc//fTTrgwHAMGiTwBsRZ8A2IxGAfCCo0Wp9evX+1+DuWnTplyX8ZRPAF6iTwBsRZ8A2IxGAfCCo0WpJUuWuD0HwuTo0aOSpNWrVwe0f7CfIgd4jT4VTcG2Swr+E1MkOobQok9FQyjPpSQ6BO/QqKKBRqGwcbQo9Vc7d+6Uz+dTjRo13JgHIbZhwwZJ0oABA0J2G3FxcSE7NhAM+lR0hKNdf0XHEGr0qfAKV4/oELxEowovGoXCxtGiVHZ2tkaNGqWnnnpKhw8flvTnv5R33323hg8frmLFHL1/OsKga9eukqTk5GSVKlXqtPunpqaqZ8+emjlzpho2bHja/ePi4lS/fv2Cjgk4Rp+KpmDbJQXfrxx0DKFCn4qGUJ9LSXQI3qBRRQONQmHjaFFq+PDhmj59usaOHas2bdrIGKOvvvpKI0aM0O+//67Ro0e7PSdcEh8fr/79+wd9vYYNG/pfYw7YjD4VTU7bJdEv2IM+FQ2cS6GoolFFA41CYeNoUeqVV17RtGnT1KVLF/+2pk2bqkaNGho0aBDBAuAZ+gTAVvQJgM1oFAAvOHoO5oEDB5ScnJxne3Jysg4cOFDgoQDAKfoEwFb0CYDNaBQALzhalGratKkmTpyYZ/vEiRPVtGnTAg8FAE7RJwC2ok8AbEajAHjB0cv3nnjiCV155ZVauHChWrduLZ/Pp+XLl2vHjh2aN2+e2zMCQMDoEwBb0ScANqNRALzg6JlSbdu21caNG9WtWzcdPHhQBw4c0DXXXKONGzfqoosucntGAAgYfQJgK/oEwGY0CoAXHD1TSpJq1KjBm90BsBJ9AmAr+gTAZjQKQLg5eqbUjBkz9M477+TZ/s477+iVV14p8FAA4BR9AmAr+gTAZjQKgBccLUqNHTtW8fHxebZXqVJFjz32WIGHAgCn6BMAW9EnADajUQC84GhR6qefflLt2rXzbE9MTNT27dsLPBQAOEWfANiKPgGwGY0C4AVHi1JVqlTRunXr8mz/73//q0qVKhV4KABwij4BsBV9AmAzGgXAC44WpXr06KHBgwdryZIlysrKUlZWlhYvXqw77rhDPXr0cHtGAAgYfQJgK/oEwGY0CoAXHH363qhRo/TTTz/p0ksvVfHifx4iOztbt9xyC683BuAp+gTAVvQJgM1oFAAvOFqUKlmypN566y2NGjVKa9euVWxsrJo0aaLExES35wOAoNAnALaiTwBsRqMAeMHRolSO+vXrq379+srKytJ3332nsmXLqkKFCm7NBgCO0ScAtqJPAGxGowCEk6P3lLrzzjs1ffp0SVJWVpbatm2rFi1aqFatWlq6dKmb8wFAUOgTAFvRJwA2o1EAvODomVLvvvuuevbsKUn68MMPtXXrVm3YsEGvvvqqhg8frq+++srVIRF6R48e1YYNG/JsT01NzfXPv0pOTlapUqVCPhsQDPoUeZz0S6JhCD/6VLTRIhR2NKpoo1GwlaNFqf3796tatWqSpHnz5ql79+5q0KCB+vXrpwkTJrg6IMJjw4YNatmy5Ukvz/kP1F+lpKSoRYsWoRwLCBp9ijxO+iXRMIQffSraaBEKOxpVtNEo2MrRolTVqlX1ww8/KCEhQZ9++qleeOEFSX+uvkZFRbk6IMIjOTlZKSkpebYfO3ZMaWlpSkpKUmxsbJ7rALahT5HHSb9yrgeEE30q2mgRCjsaVbTRKNjK0aJU37591b17dyUkJMjn86ljx46SpG+++YZ/aQupUqVKnXQFvE2bNmGeBnCOPkUe+oXCgj4VbbQIhR2NKtpoFGzlaFFqxIgRaty4sXbs2KHrr79e0dHRkqSoqCgNHTrU1QEBIBj0CYCt6BMAm9EoAF5wtCglSdddd12ebb179871fZMmTTRv3jzVqlXL6c0AQNDoEwBb0ScANqNRAMKtWCgPnpaWpj/++COUNwEAjtAnALaiTwBsRqMAuCmki1IAAAAAAABAfliUAgAAAAAAQNixKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMIupItSkydPVtWqVUN5EwDgCH0CYCv6BMBmNAqAmxwtSu3cuVOHDx/Os/2PP/7QsmXL/N/fdNNNKl26tPPpACBI9AmAregTAJvRKABeCGpRas+ePTr33HOVmJio8uXLq3fv3rnCdeDAAbVr1871IQHgdOgTAFvRJwA2o1EAvBTUotTQoUMVFRWlb775Rp9++ql++OEHXXLJJfrtt9/8+xhjXB8SAE6HPgGwFX0CYDMaBcBLQS1KLVy4UM8++6xatWqlDh066Msvv1TNmjXVvn17HThwQJLk8/lCMigAnAp9AmAr+gTAZjQKgJeCWpRKT09XhQoV/N9HR0fr3XffVVJSktq1a6d9+/a5PiAABII+AbAVfQJgMxoFwEtBLUrVqVNH69aty7WtePHieuedd1SnTh394x//cHW4vxo9erQuuOAClSpVSuXLlw/Z7QAonOgTAFt52SeJRgE4Nc6hAHgpqEWpzp07a8qUKXm250SrWbNmbs2Vx/Hjx3X99dfr3//+d8huA0DhRZ8A2MrLPkk0CsCpcQ4FwEvFg9l59OjROnr0aP4HKl5cc+bM0c6dO10Z7O8eeeQRSdLLL78ckuMDKNzoEwBbedkniUYBODXOoQB4KahnShUvXlyxsbGqU6eOfvjhhzyXR0VFKTEx0bXhACBQ9AmAregTAJvRKABeCuqZUpJUokQJZWZmFopPYMjMzFRmZqb/+4yMDA+nARBq9AmAregTAJvRKABeCeqZUjluv/12Pf744zpx4kSBbnzEiBHy+Xyn/Pr2228dH3/MmDEqV66c/6tWrVoFmheA/egTAFu51ScptI2iT0Bk4hwKgBeCfqaUJH3zzTdatGiR5s+fryZNmqh06dK5Lp8zZ05Ax7ntttvUo0ePU+6TlJTkZERJ0rBhwzRkyBD/9xkZGUQLKOLoEwBbudUnKbSNok9AZOIcCoAXHC1KlS9fXtdee22Bbzw+Pl7x8fEFPs7JREdHKzo6OmTHB2Af+gTAVm71SQpto+gTEJk4hwLghaAWpQ4fPqwyZcpoxowZoZrnpLZv364DBw5o+/btysrK0tq1ayVJ9erVU5kyZcI+DwC70CcAtvKyTxKNAnBqnEMB8FJQi1Lx8fFq166dunTpoquvvlrVq1cP1Vx5PPTQQ3rllVf83zdv3lyStGTJEl1yySVhmwOAnegTAFt52SeJRgE4Nc6hAHgpqDc637hxo6644grNnj1btWvX1jnnnKNHH31U69atC9V8fi+//LKMMXm+iBUAiT4BsJeXfZJoFIBT4xwKgJeCWpRKTEzU7bffroULF2rfvn0aMmSIvv/+e1188cWqXbu27rjjDi1evFhZWVmhmhcA8kWfANiKPgGwGY0C4KWgFqX+qly5crrxxhv15ptvav/+/Zo8ebKys7PVt29fVa5cWa+//rqbcwJAwOgTAFvRJwA2o1EAws3Rp+/lOUjx4urUqZM6deqk5557TmvWrNGJEyfcODQAFAh9AmAr+gTAZjQKQDgUaFHq6NGj2r59u44fP55re84b1AFFhe/E72perZhiD26Sdjt+gmG+Yg9uUvNqxeQ78burx4109AlFQSjbk4MGhR99QlEW6m7RrNCjUSiq6JOdHC1K/fLLL+rbt68++eSTfC/n9cYoamIOb9fqf5WRlv1LWubusRtKWv2vMko9vF3SBe4ePALRJxQloWxPDhoUPvQJkSDU3aJZoUOjUNTRJzs5WpS688479dtvv2nFihVq166d5s6dq59//lmjRo3SU0895faMgOd+L3OGWkw+rNdff10Nk5NdPXbqhg26+eabNf2KM1w9bqSiTyhKQtmeHDQofOgTIkGou0WzQodGoaijT3ZytCi1ePFivf/++zrnnHNUrFgxJSYmqmPHjipbtqzGjBmjK6+80u05AU+Z4jFaszdbx8o3kKo3c/XYx/Zma83ebJniMa4eN1LRJxQloWxPDhoUPvQJkSDU3aJZoUOjUNTRJzs5eiHlkSNHVKVKFUlSxYoV9csvv0iSmjRpotWrV7s3HQAEiT4BsBV9AmAzGgXAC44Wpc4880xt3LhRktSsWTNNnjxZu3bt0osvvqiEhARXBwSAYNAnALaiTwBsRqMAeMHxe0rt2bNHkvTwww/rsssu0+uvv66SJUvq5ZdfdnM+AAgKfQJgK/oEwGY0CoAXHC1K3Xzzzf4/N2/eXGlpadqwYYPOOOMMxcfHuzYcAASLPgGwFX0CYDMaBcALjhal/q5UqVJq0aKFG4cCAFfRJwC2ok8AbEajAISDo/eUuu666zR27Ng828eNG6frr7++wEMBgFP0CYCt6BMAm9EoAF5wtCj1+eef5/uRoJdffrmWLVtW4KEAwCn6BMBW9AmAzWgUAC84WpQ6fPiwSpYsmWd7iRIllJGRUeChAMAp+gTAVvQJgM1oFAAvOFqUaty4sd56660829988001atSowEMBgFP0CYCt6BMAm9EoAF5w9EbnDz74oK699lpt2bJF7du3lyQtWrRIs2bN0jvvvOPqgAAQDPoEwFb0CYDNaBQALzhalOrSpYvee+89PfbYY3r33XcVGxurs88+WwsXLlTbtm3dnhEAAkafANiKPgGwGY0C4AVHi1KSdOWVV+b7RngA4DX6BMBW9AmAzWgUgHBz9J5SknTw4EFNmzZN//nPf3TgwAFJ0urVq7Vr1y7XhgMAJ+gTAFvRJwA2o1EAws3RM6XWrVunDh06qFy5ckpLS1P//v1VsWJFzZ07Vz/99JNeffVVt+cEgIDQJwC2ok8AbEajAHjB0TOlhgwZoj59+mjz5s2KiYnxb+/cubOWLVvm2nAAECz6BMBW9AmAzWgUAC84WpRatWqV/vWvf+XZXqNGDe3du7fAQwGAU/QJgK3oEwCb0SgAXnC0KBUTE6OMjIw82zdu3KjKlSsXeCgAcIo+AbAVfQJgMxoFwAuOFqWuvvpqjRw5Un/88Yckyefzafv27Ro6dKiuvfZaVwcEgGDQJwC2ok8AbEajAHjB0aLUk08+qV9++UVVqlTRsWPH1LZtW9WrV09xcXEaPXq02zMCQMDoEwBb0ScANqNRALzg6NP3ypYtqy+//FKLFy/W6tWrlZ2drRYtWqhDhw5uzwcAQaFPAGxFnwDYjEYB8IKjRakc7du3V/v27d2aBQBcQ58A2Io+AbAZjQIQTo4XpRYtWqRFixZp3759ys7OznXZSy+9VODBAMAp+gTAVvQJgM1oFIBwc7Qo9cgjj2jkyJFq1aqVEhIS5PP53J4LAByhTwBsRZ8A2IxGAfCCo0WpF198US+//LJ69erl9jwAUCD0CYCt6BMAm9EoAF5w9Ol7x48f1wUXXOD2LABQYPQJgK3oEwCb0SgAXnC0KNW/f3+98cYbbs8CAAVGnwDYij4BsBmNAuAFRy/f+/333zVlyhQtXLhQZ599tkqUKJHr8qefftqV4QAgWPQJgK3oEwCb0SgAXnC0KLVu3To1a9ZMkrR+/fpcl/GGeAC8RJ8A2Io+AbAZjQLgBUeLUkuWLHF7DgBwBX0CYCv6BMBmNAqAFxy9p1SOH3/8UZ999pmOHTsmSTLGuDIUABQUfQJgK/oEwGY0CkA4OVqU+vXXX3XppZeqQYMGuuKKK7Rnzx5Jf7453t133+3qgAAQDPoEwFb0CYDNaBQALzhalLrrrrtUokQJbd++XaVKlfJvv+GGG/Tpp5+6NhwABIs+AbAVfQJgMxoFwAuO3lNq/vz5+uyzz1SzZs1c2+vXr6+ffvrJlcEAwAn6BMBW9AmAzWgUAC84eqbUkSNHcq2e59i/f7+io6MLPBQAOEWfANiKPgGwGY0C4AVHi1IXX3yxXn31Vf/3Pp9P2dnZGjdunNq1a+facAAQLPoEwFb0CYDNaBQALzh6+d64ceN0ySWX6Ntvv9Xx48d133336fvvv9eBAwf01VdfuT2j0tLS9Oijj2rx4sXau3evqlevrp49e2r48OEqWbKk67cHoPCiTwBsFe4+STQKQOA4hwLgBUeLUo0aNdJ///tfvfjii4qKitKRI0d0zTXX6NZbb1VCQoLbM2rDhg3Kzs7W5MmTVa9ePa1fv14DBgzQkSNH9OSTT7p+ewAKL/oEwFbh7pNEowAEjnMoAF5wtCglSQkJCXrkkUfcnOWkLr/8cl1++eX+7+vUqaONGzdq0qRJBAtAHvQJgK3C2SeJRgEIDudQAMLN0XtK1alTR3379lVmZmau7fv371edOnVcGex00tPTVbFixbDcFoDCgz4BsJUNfZJoFID82dAo+gREHkeLUmlpafrqq6900UUXac+ePf7tWVlZYfm40C1btui5557TwIEDT7lfZmamMjIycn0BKNroEwBbed0nKbBG0ScgMnndKM6hgMjkaFHK5/Pp008/Vc2aNdWqVSutWrXK0Y2PGDFCPp/vlF/ffvttruvs3r1bl19+ua6//nr179//lMcfM2aMypUr5/+qVauWozkBFB70CYCt3OqTFNpG0ScgMnEOBcALjt5TyhijMmXKaM6cORo2bJjatm2rKVOmqGPHjkEd57bbblOPHj1OuU9SUpL/z7t371a7du3UunVrTZky5bTHHzZsmIYMGeL/PiMjg2gBRRx9AmArt/okhbZR9AmITJxDAfCCo0Upn8/n//OYMWN01llnacCAAbrxxhuDOk58fLzi4+MD2nfXrl1q166dWrZsqRkzZqhYsdM/ySs6OlrR0dFBzQSgcKNPAGzlVp+k0DaKPgGRiXMoAF5w/Eypv+rZs6fq1q2rbt26uTLU3+3evVuXXHKJzjjjDD355JP65Zdf/JdVq1YtJLcJoHCiTwBsFe4+STQKQOA4hwLgBUeLUtnZ2Xm2tW7dWv/973+1YcOGAg/1d/Pnz9ePP/6oH3/8UTVr1sx12d/jCSCy0ScAtgp3nyQaBSBwnEMB8IKjNzo/mapVq6pt27ZuHlKS1KdPHxlj8v0CgEDQJwC2ClWfJBoFoOA4hwIQSo6eKSVJ7777rt5++21t375dx48fz3XZ6tWrCzwYADhFnwDYij4BsBmNAhBujp4pNWHCBPXt21dVqlTRmjVrdO6556pSpUraunWrOnfu7PaMABAw+gTAVvQJgM1oFAAvOFqUeuGFFzRlyhRNnDhRJUuW1H333acFCxZo8ODBSk9Pd3tGAAgYfQJgK/oEwGY0CoAXHC1Kbd++XRdccIEkKTY2VocOHZIk9erVS7NmzXJvOgAIEn0CYCv6BMBmNAqAFxwtSlWrVk2//vqrJCkxMVErVqyQJG3bto03pgPgKfoEwFb0CYDNaBQALzhalGrfvr0+/PBDSVK/fv101113qWPHjrrhhhvUrVs3VwcEgGDQJwC2ok8AbEajAHjB0afvTZkyRdnZ2ZKkgQMHqmLFivryyy911VVXaeDAga4OCADBoE8AbEWfANiMRgHwQtCLUidOnNDo0aP1z3/+U7Vq1ZIkde/eXd27d3d9OAAIBn0CYCv6BMBmNAqAV4J++V7x4sU1btw4ZWVlhWIeAHCMPgGwFX0CYDMaBcArjt5TqkOHDlq6dKnLowBAwdEnALaiTwBsRqMAeMHRe0p17txZw4YN0/r169WyZUuVLl061+VdunRxZTgACBZ9AmAr+gTAZjQKgBccLUr9+9//liQ9/fTTeS7z+Xw87ROAZ+gTAFvRJwA2o1EAvOBoUSrnUxkAwDb0CYCt6BMAm9EoAF5w9J5Sr776qjIzM/NsP378uF599dUCDwUATtEnALaiTwBsRqMAeMHRolTfvn2Vnp6eZ/uhQ4fUt2/fAg8FAE7RJwC2ok8AbEajAHjB0aKUMUY+ny/P9p07d6pcuXIFHgoAnKJPAGxFnwDYjEYB8EJQ7ynVvHlz+Xw++Xw+XXrppSpe/H9Xz8rK0rZt23T55Ze7PiQAnA59AmAr+gTAZjQKgJeCWpTq2rWrJGnt2rW67LLLVKZMGf9lJUuWVFJSkq699lpXBwSAQNAnALaiTwBsRqMAeCmoRamHH35YkpSUlKQePXooOjo6JEMBQLDoEwBb0ScANqNRALzk6D2l2rdvr19++cX//cqVK3XnnXdqypQprg0GAE7QJwC2ok8AbEajAHjB0aLUTTfdpCVLlkiS9u7dqw4dOmjlypX6z3/+o5EjR7o6IAAEgz4BsBV9AmAzGgXAC44WpdavX69zzz1XkvT222+rSZMmWr58ud544w29/PLLbs4HAEGhTwBsRZ8A2IxGAfCCo0WpP/74w/9a44ULF6pLly6SpOTkZO3Zs8e96QAgSPQJgK3oEwCb0SgAXnC0KHXWWWfpxRdf1BdffKEFCxb4PyJ09+7dqlSpkqsDAkAw6BMAW9EnADajUQC84GhR6vHHH9fkyZN1ySWX6MYbb1TTpk0lSR988IH/KZ8A4AX6BMBW9AmAzWgUAC8Ud3KlSy65RPv371dGRoYqVKjg3/5///d/KlWqlGvDAUCw6BMAW9EnADajUQC84GhRSpKioqJyxUqSkpKSCjoPABQYfQJgK/oEwGY0CkC4OV6Uevfdd/X2229r+/btOn78eK7LVq9eXeDBAMAp+gTAVvQJgM1oFIBwc/SeUhMmTFDfvn1VpUoVrVmzRueee64qVaqkrVu3qnPnzm7PCAABo08AbEWfANiMRgHwgqNFqRdeeEFTpkzRxIkTVbJkSd13331asGCBBg8erPT0dLdnBICA0ScAtqJPAGxGowB4wdGi1Pbt23XBBRdIkmJjY3Xo0CFJUq9evTRr1iz3pgOAINEnALaiTwBsRqMAeMHRolS1atX066+/SpISExO1YsUKSdK2bdtkjHFvOgAIEn0CYCv6BMBmNAqAFxwtSrVv314ffvihJKlfv36666671LFjR91www3q1q2bqwMCQDDoEwBb0ScANqNRALzg6NP3pkyZouzsbEnSwIEDVbFiRX355Ze66qqrNHDgQFcHBIBg0CcAtqJPAGxGowB4wdGiVLFixVSs2P+eZNW9e3d17949z36DBg3SyJEjFR8f73xCAAgCfQJgK/oEwGY0CoAXHL18L1AzZ85URkZGKG8CAByhTwBsRZ8A2IxGAXBTSBeleEM8ALaiTwBsRZ8A2IxGAXBTSBelAAAAAAAAgPywKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMIupItSPXv2VNmyZUN5EwDgCH0CYCv6BMBmNAqAmxwtSiUlJWnkyJHavn37KfebNGmS4uPjHQ0GAE7QJwC2ok8AbEajAHjB0aLU3Xffrffff1916tRRx44d9eabbyozM9Pt2XLp0qWLzjjjDMXExCghIUG9evXS7t27Q3qbAAof+gTAVl70SaJRAALDORQALzhalLr99tuVkpKilJQUNWrUSIMHD1ZCQoJuu+02rV692u0ZJUnt2rXT22+/rY0bN2r27NnasmWLrrvuupDcFoDCiz4BsJUXfZJoFIDAcA4FwBPGBcePHzfPPPOMiY6ONsWKFTNnn322mT59usnOznbj8Pl6//33jc/nM8ePHw/4Ounp6UaSSU9PD9lcKJpSUlKMJJOSklKojl1YhPKxSZ9QmIWjDzTo9EL1+PSiT8YE3yj6hGCEuik0KzfOoWgUAkefwivQx2bxgixo/fHHH5o7d65mzJihBQsW6Pzzz1e/fv20e/duDR8+XAsXLtQbb7xRkJvI14EDB/T666/rggsuUIkSJU66X2ZmZq6nnGZkZLg+CwA70ScAtvKqT1JgjaJPQGTjHApAODlalFq9erVmzJihWbNmKSoqSr169dL48eOVnJzs36dTp066+OKLXRtUku6//35NnDhRR48e1fnnn6+PPvrolPuPGTNGjzzyiKszALAbfQJgK6/6JAXXKPoERCbOoQB4wdF7Sp1zzjnavHmzJk2apJ07d+rJJ5/MFStJatSokXr06HHK44wYMUI+n++UX99++61//3vvvVdr1qzR/PnzFRUVpVtuuUXGmJMef9iwYUpPT/d/7dixw8ndBVCI0CcAtnKrT1JoG0WfgMjEORQALzh6ptTWrVuVmJh4yn1Kly6tGTNmnHKf22677bRRS0pK8v85Pj5e8fHxatCggRo2bKhatWppxYoVat26db7XjY6OVnR09CmPD6BooU8AbOVWn6TQNoo+AZGJcygAXnC0KNWuXTutWrVKlSpVyrX94MGDatGihbZu3RrQcXIC5ETO6nk4PkoZQOFBnwDYyq0+STQKgPs4hwLgBUeLUmlpacrKysqzPTMzU7t27SrwUH+3cuVKrVy5UhdeeKEqVKigrVu36qGHHlLdunVPuoIOIDLRJwC2CnefJBoFIHCcQwHwQlCLUh988IH/z5999pnKlSvn/z4rK0uLFi3K9VRMt8TGxmrOnDl6+OGHdeTIESUkJOjyyy/Xm2++yVM3AUiiTwDs5VWfJBoF4PQ4hwLgpaAWpbp27SpJ8vl86t27d67LSpQooaSkJD311FOuDZejSZMmWrx4sevHBVB00CcAtvKqTxKNAnB6nEMB8FJQi1LZ2dmSpNq1a2vVqlWOXysMAG6jTwBsRZ8A2IxGAfCSo/eU2rZtm9tzAIAr6BMAW9EnADajUQC8EPCi1IQJE/R///d/iomJ0YQJE0657+DBgws8GAAEij4BsBV9AmAzGgXAawEvSo0fP14333yzYmJiNH78+JPu5/P5CBaAsKJPAGxFnwDYjEYB8FrAi1J/fTonT+0EYBP6BMBW9AmAzWgUAK8V83oAAAAAAAAARB5Hi1LXXXedxo4dm2f7uHHjdP311xd4KABwij4BsBV9AmAzGgXAC44WpT7//HNdeeWVebZffvnlWrZsWYGHAgCn6BMAW9EnADajUQC84GhR6vDhwypZsmSe7SVKlFBGRkaBhwIAp+gTAFvRJwA2o1EAvOBoUapx48Z666238mx/88031ahRowIPBQBO0ScAtqJPAGxGowB4IeBP3/urBx98UNdee622bNmi9u3bS5IWLVqkWbNm6Z133nF1QAAIBn0CYCv6BMBmNAqAFxwtSnXp0kXvvfeeHnvsMb377ruKjY3V2WefrYULF6pt27ZuzwgAAaNPAGxFnwDYjEYB8IKjRSlJuvLKK/N9IzwA8Bp9AmAr+gTAZjQKQLg5ek8pAAAAAAAAoCACfqZUxYoVtWnTJsXHx6tChQry+Xwn3ffAgQOuDAcAgaBPAGxFnwDYjEYB8FrAi1Ljx49XXFycJOmZZ54J1TwAEDT6BMBW9AmAzWgUAK8FvCjVu3fvfP8MAF6jTwBsRZ8A2IxGAfBawItSGRkZAR+0bNmyjoYBACfoEwBb0ScANqNRALwW8KJU+fLlT/kaY0kyxsjn8ykrK6vAgwFAoOgTAFvRJwA2o1EAvBbwotSSJUtCOQcAOEafANiKPgGwGY0C4LWAF6Xatm0byjkAwDH6BMBW9AmAzWgUAK8FvCi1bt06NW7cWMWKFdO6detOue/ZZ59d4MEAmxw9elSStHr16oD2P3bsmNLS0pSUlKTY2NhT7puamlrg+SIdfUJRFWx7pOD6I9GgUKNPiDShPGeSaJbbaBQiCX2yU8CLUs2aNdPevXtVpUoVNWvWTD6fT8aYPPvxemMURRs2bJAkDRgwIGS3kfNxvAgefUJRFY725KBBoUGfEGnC1S2a5Q4ahUhCn+wU8KLUtm3bVLlyZf+fgUjStWtXSVJycrJKlSp12v1TU1PVs2dPzZw5Uw0bNjzt/nFxcapfv35Bx4xY9AlFVbDtkYLvj0SDQok+IdKE+pxJolluolGIJPTJTgEvSiUmJub7ZyASxMfHq3///kFfr2HDhmrRokUIJsJf0ScUVU7bI9EfW9AnRBrOmQoXGoVIQp/sFPCi1N9t2rRJS5cu1b59+5SdnZ3rsoceeqjAgwGAU/QJgK3oEwCb0SgA4eZoUWrq1Kn697//rfj4eFWrVk0+n89/mc/nI1gAPEOfANiKPgGwGY0C4AVHi1KjRo3S6NGjdf/997s9DwAUCH0CYCv6BMBmNAqAF4o5udJvv/2m66+/3u1ZAKDA6BMAW9EnADajUQC84GhR6vrrr9f8+fPdngUACow+AbAVfQJgMxoFwAsBv3xvwoQJ/j/Xq1dPDz74oFasWKEmTZqoRIkSufYdPHiwexMCwGnQJwC2ok8AbEajAHgt4EWp8ePH5/q+TJky+vzzz/X555/n2u7z+QgWgLCiTwBsRZ8A2IxGAfBawItS27ZtC+UcAOAYfQJgK/oEwGY0CoDXHL2n1F8ZY2SMcWMWAHAVfQJgK/oEwGY0CkC4OF6Umj59uho3bqyYmBjFxMSocePGmjZtmpuzAYAj9AmAregTAJvRKADhFvDL9/7qwQcf1Pjx43X77berdevWkqSvv/5ad911l9LS0jRq1ChXhwSAQNEnALaiTwBsRqMAeMHRotSkSZM0depU3Xjjjf5tXbp00dlnn63bb7+dYAHwDH0CYCv6BMBmNAqAFxy9fC8rK0utWrXKs71ly5Y6ceJEgYcCAKfoEwBb0ScANqNRALzgaFGqZ8+emjRpUp7tU6ZM0c0331zgoQDAKfoEwFb0CYDNaBQALzh6+Z7055vgzZ8/X+eff74kacWKFdqxY4duueUWDRkyxL/f008/XfApASAI9AmAregTAJvRKADh5mhRav369WrRooUkacuWLZKkypUrq3Llylq/fr1/P5/P58KIABA4+gTAVvQJgM1oFAAvOFqUWrJkidtzAIAr6BMAW9EnADajUQC84Og9pf5q586d2rVrlxuzBCQzM1PNmjWTz+fT2rVrw3a7AAof+gTAVuHuk0SjAASOcygA4eJoUSo7O1sjR45UuXLllJiYqDPOOEPly5fXo48+quzsbLdnzOW+++5T9erVQ3obAAov+gTAVl72SaJRAE6NcygAXnD08r3hw4dr+vTpGjt2rNq0aSNjjL766iuNGDFCv//+u0aPHu32nJKkTz75RPPnz9fs2bP1ySefhOQ2ABRu9AmArbzqk0SjAJwe51AAvOBoUeqVV17RtGnT1KVLF/+2pk2bqkaNGho0aFBIgvXzzz9rwIABeu+991SqVKmArpOZmanMzEz/9xkZGa7PBcAu9AmArbzokxR8o+gTEJk4hwLgBUcv3ztw4ICSk5PzbE9OTtaBAwcKPNTfGWPUp08fDRw4UK1atQr4emPGjFG5cuX8X7Vq1XJ9NgB2oU8AbBXuPknOGkWfgMjEORQALzhalGratKkmTpyYZ/vEiRPVtGnTgI8zYsQI+Xy+U359++23eu6555SRkaFhw4YFNeewYcOUnp7u/9qxY0dQ1wdQ+NAnALZyq09SaBtFn4DIxDkUAC84evneE088oSuvvFILFy5U69at5fP5tHz5cu3YsUPz5s0L+Di33XabevToccp9kpKSNGrUKK1YsULR0dG5LmvVqpVuvvlmvfLKK/leNzo6Os91ABRt9AmArdzqkxTaRtEnIDJxDgXACz5jjHFyxd27d+v555/Xhg0bZIxRo0aNNGjQoJB8asL27dtzvVZ49+7duuyyy/Tuu+/qvPPOU82aNQM6TkZGhsqVK6f09HSVLVvW9TmBHKtXr1bLli2VkpKiFi1aeD2O9dx+bNInRDL64z43H5/h7JPkTqPoE0KJZhUM51A0CqFDnwom0Memo2dKSVL16tVD+ikxf3XGGWfk+r5MmTKSpLp16wYcKwCRgz4BsFU4+yTRKADB4RwKQLg5XpT67bffNH36dKWmpsrn86lhw4bq27evKlas6OZ8ABA0+gTAVvQJgM1oFIBwc/RG559//rlq166tCRMm6LffftOBAwc0YcIE1a5dW59//rnbM+aRlJQkY4yaNWsW8tsCULjQJwC28rpPEo0CcHJeN4o+AZHJ0TOlbr31VnXv3l2TJk1SVFSUJCkrK0uDBg3SrbfeqvXr17s6JAAEij4BsBV9AmAzGgXAC46eKbVlyxbdfffd/lhJUlRUlIYMGaItW7a4NhwABIs+AbAVfQJgMxoFwAuOFqVatGih1NTUPNtTU1N5uiUAT9EnALaiTwBsRqMAeMHRy/cGDx6sO+64Qz/++KPOP/98SdKKFSv0/PPPa+zYsVq3bp1/37PPPtudSQEgAPQJgK3oEwCb0SgAXnC0KHXjjTdKku677758L/P5fDLGyOfzKSsrq2ATAkAQ6BMAW9EnADajUQC84GhRatu2bW7PAQCuoE8AbEWfANiMRgHwgqNFqcTERLfnAABX0CcAtqJPAGxGowB4IeBFqQ8++ECdO3dWiRIl9MEHH5xy3y5duhR4MAAIFH0CYCv6BMBmNAqA1wJelOratav27t2rKlWqqGvXrifdj9cYAwg3+gTAVvQJgM1oFACvBbwolZ2dne+fAcBr9AmAregTAJvRKABeK+b1AAAAAAAAAIg8AT9TasKECQEfdPDgwY6GAQAn6BMAW9EnADajUQC8FvCi1Pjx4wPaz+fzESwAYUWfANiKPgGwGY0C4LWAF6W2bdsWyjkAwDH6BMBW9AmAzWgUAK+F9D2lypYtq61bt4byJgDAEfoEwFb0CYDNaBQAN4V0UcoYE8rDA4Bj9AmAregTAJvRKABu4tP3AAAAAAAAEHYsSgEAAAAAACDsWJQCAAAAAABA2IV0Ucrn84Xy8ADgGH0CYCv6BMBmNAqAm3ijcwARiT4BsBV9AmAzGgXATSFdlPrkk09Uo0aNUN4EADhCnwDYij4BsBmNAuCm4k6ulJWVpZdfflmLFi3Svn37lJ2dnevyxYsXS5IuvPDCgk8IAEGgTwBsRZ8A2IxGAfCCo0WpO+64Qy+//LKuvPJKNW7cmNcVA7AGfQJgK/oEwGY0CoAXHC1Kvfnmm3r77bd1xRVXuD0PABQIfQJgK/oEwGY0CoAXHL2nVMmSJVWvXj23ZwGAAqNPAGxFnwDYjEYB8IKjRam7775bzz77LJ+8AMA69AmAregTAJvRKABeCPjle9dcc02u7xcvXqxPPvlEZ511lkqUKJHrsjlz5rgzHQAEgD4BsBV9AmAzGgXAawEvSpUrVy7X9926dXN9GABwgj4BsBV9AmAzGgXAawEvSs2YMSOUcwCAY/QJgK3oEwCb0SgAXnP06Xs59u3bp40bN8rn86lBgwaqUqWKW3MBQIHQJwC2ok8AbEajAISTozc6z8jIUK9evVSjRg21bdtWF198sWrUqKGePXsqPT3d7RkBIGD0CYCt6BMAm9EoAF5wtCjVv39/ffPNN/roo4908OBBpaen66OPPtK3336rAQMGuD0jAASMPgGwFX0CYDMaBcALjl6+9/HHH+uzzz7ThRde6N922WWXaerUqbr88stdGw4AgkWfANiKPgGwGY0C4AVHz5SqVKlSnk9qkP789IYKFSoUeCgAcIo+AbAVfQJgMxoFwAuOFqUeeOABDRkyRHv27PFv27t3r+699149+OCDrg0HAMGiTwBsRZ8A2IxGAfCCo5fvTZo0ST/++KMSExN1xhlnSJK2b9+u6Oho/fLLL5o8ebJ/39WrV7szKQAEgD4BsBV9AmAzGgXAC44Wpbp27eryGADgDvoEwFb0CYDNaBQALzhalHr44YfdngMAXEGfANiKPgGwGY0C4AVH7ykFAAAAAAAAFISjZ0plZWVp/Pjxevvtt7V9+3YdP3481+UHDhxwZTgACBZ9AmAr+gTAZjQKgBccPVPqkUce0dNPP63u3bsrPT1dQ4YM0TXXXKNixYppxIgRLo8IAIGjTwBsRZ8A2IxGAfCCo0Wp119/XVOnTtU999yj4sWL68Ybb9S0adP00EMPacWKFW7PKElKSkqSz+fL9TV06NCQ3BaAwos+AbCVF32SaBSAwHAOBcALjl6+t3fvXjVp0kSSVKZMGaWnp0uS/vGPf+jBBx90b7q/GTlypAYMGOD/vkyZMiG7LQCFE30CYCuv+iTRKACnxzkUAC84eqZUzZo1tWfPHklSvXr1NH/+fEnSqlWrFB0d7d50fxMXF6dq1ar5vwgWgL+jTwBs5VWfJBoF4PQ4hwLgBUeLUt26ddOiRYskSXfccYcefPBB1a9fX7fccov++c9/ujrgXz3++OOqVKmSmjVrptGjR+d58z0AoE8AbOVVnyQaBeD0OIcC4AVHL98bO3as/8/XXXedatWqpa+++kr16tVTly5dXBvur+644w61aNFCFSpU0MqVKzVs2DBt27ZN06ZNO+l1MjMzlZmZ6f8+IyMjJLMBsAd9AmArL/okBd8o+gREJs6hAHjCBOn48eOmT58+ZsuWLcFeNY+HH37YSDrl16pVq/K97rvvvmskmf379wd9/PT09ALPDpxKSkqKkWRSUlK8HqVQSE9Pd+WxSZ8A+hMKbjTKzT4ZE9pG0SeEE80qGM6haBRChz4VTKB98hljTLALWeXLl9fq1atVp06dYK+ay/79+7V///5T7pOUlKSYmJg823ft2qWaNWtqxYoVOu+88/K9bn6r6LVq1VJ6errKli1boNmBU1m9erVatmyplJQUtWjRwutxrJeRkaFy5cq58tikT4h09Md9bjXKrT5JoW0UfUI40ayC4RyKRiF06FPBBNonRy/f69atm9577z0NGTLE8YCSFB8fr/j4eEfXXbNmjSQpISHhpPtER0eH/I1DAdiFPgGwlVt9kkLbKPoERCbOoQB4wdGiVL169fToo49q+fLlatmypUqXLp3r8sGDB7syXI6vv/5aK1asULt27VSuXDmtWrVKd911l7p06aIzzjjD1dsCULjRJwC2CnefJBoFIHCcQwHwgqNFqWnTpql8+fJKSUlRSkpKrst8Pp/rwYqOjtZbb72lRx55RJmZmUpMTNSAAQN03333uXo7QLCOHj2qDRs25Nmempqa659/lZycrFKlSoV8tkhFnxAJTtYe6dT9kWiQl8LdJ4lGwR5OzpkkmhVOnEMhUtEnbzl6T6nCys3XXAPS/15nHAxek5wXj01+BgiOk/bkoEHBi/THZ6Tff7jDabdo1qnx+ORngIKjT6ER0veUOtnrjH0+n2JiYlSvXj1dffXVqlixopPDA4VGcnJynv+TJEnHjh1TWlqakpKSFBsbm+c6CB36hEhwsvZIp+5PznXhDfqESObknCnneggPGoVIRZ+85eiZUu3atdPq1auVlZWlM888U8YYbd68WVFRUUpOTtbGjRvl8/n05ZdfqlGjRqGY2xFW0QE7ufnYpE8A3ObW45M+AXAb51A0CrBVoI/NYk4OfvXVV6tDhw7avXu3UlJStHr1au3atUsdO3bUjTfeqF27duniiy/WXXfd5fgOAIAT9AmAregTAJvRKABecPRMqRo1amjBggV5Vsi///57derUSbt27dLq1avVqVMn7d+/37VhC4pVdMBObj426RMAt7n1+KRPANzGORSNAmwV0mdKpaena9++fXm2//LLL8rIyJAklS9fXsePH3dyeABwjD4BsBV9AmAzGgXAC45fvvfPf/5Tc+fO1c6dO7Vr1y7NnTtX/fr1U9euXSVJK1euVIMGDdycFQBOiz4BsBV9AmAzGgXAC45evnf48GHdddddevXVV3XixAlJUvHixdW7d2+NHz9epUuX1tq1ayVJzZo1c3PeAuGpnYCd3Hxs0icAbnPr8UmfALiNcygaBdgq0Memo0WpHIcPH9bWrVtljFHdunVVpkwZp4cKC4IF2CkUj036BMAtbj8+6RMAt3AORaMAWwX62CxekBspU6aMzj777IIcAgBCgj4BsBV9AmAzGgUgnBy9pxQAAAAAAABQECxKAQAAAAAAIOxYlAIAAAAAAEDYsSgFAAAAAACAsCvQG50XNjkfNJiRkeHxJAD+KucxWYAPAy306BNgr0hvFH0C7BXpfZJoFGCrQPsUUYtShw4dkiTVqlXL40kA5OfQoUMqV66c12N4gj4B9ovURtEnwH6R2ieJRgG2O12ffCaCltWzs7O1e/duxcXFyefzeT1OoZCRkaFatWppx44dKlu2rNfjFBr83IJjjNGhQ4dUvXp1FSsWma8qpk/ui9THYaTe71CK9EbRp4LhMXlq/HwKJtL7JNGoguIxeHL8bAom0D5F1DOlihUrppo1a3o9RqFUtmxZHogO8HMLXKT+370c9Cl0IvVxGKn3O1QiuVH0yR08Jk+Nn49zkdwniUa5hcfgyfGzcS6QPkXmcjoAAAAAAAA8xaIUAAAAAAAAwo5FKZxSdHS0Hn74YUVHR3s9SqHCzw3wXqQ+DiP1fgO24jF5avx8AG/xGDw5fjbhEVFvdA4AAAAAAAA78EwpAAAAAAAAhB2LUgAAAAAAAAg7FqUAAAAAAAAQdixKFXEvvPCCateurZiYGLVs2VJffPHFSfedM2eOOnbsqMqVK6ts2bJq3bq1Pvvsszz7tGrVSuXLl1fp0qXVrFkzvfbaa3mOtWvXLvXs2VOVKlVSqVKl1KxZM6WkpLh+/0IlmJ9bnz595PP58nydddZZ+e7/5ptvyufzqWvXrrm2jxgxIs8xqlWr5ubdAgqVYB6He/bs0U033aQzzzxTxYoV05133plnnz/++EMjR45U3bp1FRMTo6ZNm+rTTz/Ntc+JEyf0wAMPqHbt2oqNjVWdOnU0cuRIZWdnu333TiqY+7106dJ8+7Nhw4Zc+x08eFC33nqrEhISFBMTo4YNG2revHn+y5ctW6arrrpK1atXl8/n03vvvRequwcUOsE8Jr/88ku1adNGlSpVUmxsrJKTkzV+/Phc+3z//fe69tprlZSUJJ/Pp2eeeSbPcQrTYzIU50ynaxbnTMD/uP373l+d7PcWG86XAhHMz0aSMjMzNXz4cCUmJio6Olp169bVSy+95L88kHPJwtRvW7AoVYS99dZbuvPOOzV8+HCtWbNGF110kTp37qzt27fnu/+yZcvUsWNHzZs3TykpKWrXrp2uuuoqrVmzxr9PxYoVNXz4cH399ddat26d+vbtq759++aK2W+//aY2bdqoRIkS+uSTT/TDDz/oqaeeUvny5UN9l10R7M/t2Wef1Z49e/xfO3bsUMWKFXX99dfn2fenn37SPffco4suuijfY5111lm5jvXdd9+5et+AwiLYx2FmZqYqV66s4cOHq2nTpvnu88ADD2jy5Ml67rnn9MMPP2jgwIHq1q1brsY9/vjjevHFFzVx4kSlpqbqiSee0Lhx4/Tcc8+F5H7+XbD3O8fGjRtztaN+/fr+y44fP66OHTsqLS1N7777rjZu3KipU6eqRo0a/n2OHDmipk2bauLEiSG7b0BhFOxjsnTp0rrtttu0bNkypaam6oEHHtADDzygKVOm+Pc5evSo6tSpo7Fjx550IaWwPCZDcc4USLMkzpkAKTS/7+U41e8tXp8vBcLJOVX37t21aNEiTZ8+XRs3btSsWbOUnJzsvzyQc8nC0m+rGBRZ5557rhk4cGCubcnJyWbo0KEBH6NRo0bmkUceOeU+zZs3Nw888ID/+/vvv99ceOGFwQ1rkYL+3ObOnWt8Pp9JS0vLtf3EiROmTZs2Ztq0aaZ3797m6quvznX5ww8/bJo2bVqQ0YEioyCPw7Zt25o77rgjz/aEhAQzceLEXNuuvvpqc/PNN/u/v/LKK80///nPXPtcc801pmfPnkFM71yw93vJkiVGkvntt99OesxJkyaZOnXqmOPHjwc0gyQzd+7cQEcGijQ3zqW6det20oYkJiaa8ePHn/L6Nj8mQ3HOFEizOGcC/hSq3/dO93uL1+dLgQj2Z/PJJ5+YcuXKmV9//fWkxwzkXPKvbO63TXimVBF1/PhxpaSkqFOnTrm2d+rUScuXLw/oGNnZ2Tp06JAqVqyY7+XGGC1atEgbN27UxRdf7N/+wQcfqFWrVrr++utVpUoVNW/eXFOnTnV+Z8LIjZ/b9OnT1aFDByUmJubaPnLkSFWuXFn9+vU76XU3b96s6tWrq3bt2urRo4e2bt0a/J0ACjk3Hof5yczMVExMTK5tsbGx+vLLL/3fX3jhhVq0aJE2bdokSfrvf/+rL7/8UldccYXj2w1UQe538+bNlZCQoEsvvVRLlizJddkHH3yg1q1b69Zbb1XVqlXVuHFjPfbYY8rKynL9PgBFiRstWrNmjZYvX662bduGYkRPheqcKdBmcc6ESBfK3/dO93uLl+dLgXDys8n5HfaJJ55QjRo11KBBA91zzz06duyYf59AziURvOJeD4DQ2L9/v7KyslS1atVc26tWraq9e/cGdIynnnpKR44cUffu3XNtT09PV40aNZSZmamoqCi98MIL6tixo//yrVu3atKkSRoyZIj+85//aOXKlRo8eLCio6N1yy23FPzOhVBBf2579uzRJ598ojfeeCPX9q+++krTp0/X2rVrT3rd8847T6+++qoaNGign3/+WaNGjdIFF1yg77//XpUqVXJ0f4DCyI1+5eeyyy7T008/rYsvvlh169bVokWL9P777+f6Ref+++9Xenq6kpOTFRUVpaysLI0ePVo33nij49sNlJP7nZCQoClTpqhly5bKzMzUa6+9pksvvVRLly71/8+CrVu3avHixbr55ps1b948bd68WbfeeqtOnDihhx56KOT3CyisCtKimjVr6pdfftGJEyc0YsQI9e/fP5SjeiJU50yBNItzJiB0v+8F8nuLl+dLgXDys9m6dau+/PJLxcTEaO7cudq/f78GDRqkAwcO+N9XKpBzSQSPRakizufz5freGJNnW35mzZqlESNG6P3331eVKlVyXRYXF6e1a9fq8OHDWrRokYYMGaI6derokksukfTninurVq302GOPSfrz/+B///33mjRpkvWLUjmc/txefvlllS9fPtebAR46dEg9e/bU1KlTFR8ff9Lrdu7c2f/nJk2aqHXr1qpbt65eeeUVDRkyJPg7ARRyTh+HJ/Pss89qwIABSk5Ols/nU926ddW3b1/NmDHDv89bb72lmTNn6o033tBZZ52ltWvX6s4771T16tXVu3dvx7cdjGDu95lnnqkzzzzT/33r1q21Y8cOPfnkk/5FqezsbFWpUkVTpkxRVFSUWrZsqd27d2vcuHEsSgEBcNKiL774QocPH9aKFSs0dOhQ1atXz5pf1tzm5jmTFFizOGcC/sfN3/cC/b3FhvOlQATzs8nOzpbP59Prr7+ucuXKSZKefvppXXfddXr++ecVGxsb0LkkgseiVBEVHx+vqKioPCvB+/bty7Ni/HdvvfWW+vXrp3feeUcdOnTIc3mxYsVUr149SVKzZs2UmpqqMWPG+BelEhIS1KhRo1zXadiwoWbPnl2AexQeBfm5GWP00ksvqVevXipZsqR/+5YtW5SWlqarrrrKvy3nkymKFy+ujRs3qm7dunmOV7p0aTVp0kSbN28uyF0CCp2CPA5PpXLlynrvvff0+++/69dff1X16tU1dOhQ1a5d27/Pvffeq6FDh6pHjx6S/vxl56efftKYMWNCfpLl1v0+//zzNXPmTP/3CQkJKlGihKKiovzbGjZsqL179+r48eO5egXgfwrymMzpSpMmTfTzzz9rxIgRRW5RKhTnTJKzZnHOhEgUit/3Av29xcvzpUA4+dkkJCSoRo0a/gUp6c/2GGO0c+dO1a9fP6BzSQSP95QqokqWLKmWLVtqwYIFubYvWLBAF1xwwUmvN2vWLPXp00dvvPGGrrzyyoBuyxijzMxM//dt2rTRxo0bc+2zadOmPO+xZCOnPzdJ+vzzz/Xjjz/mee11cnKyvvvuO61du9b/1aVLF7Vr105r165VrVq18j1eZmamUlNTlZCQULA7BRQyBXkcBiImJkY1atTQiRMnNHv2bF199dX+y44ePapixXL/pzEqKiosH3Hs1v1es2ZNrm60adNGP/74Y677sGnTJiUkJLAgBZyCW4/Jv58nFRWhOGeSnDWLcyZEolD8vhfo7y1eni8FwsnPpk2bNtq9e7cOHz7s37Zp0yYVK1ZMNWvWzLXvqc4l4YAX766O8HjzzTdNiRIlzPTp080PP/xg7rzzTlO6dGn/J5wMHTrU9OrVy7//G2+8YYoXL26ef/55s2fPHv/XwYMH/fs89thjZv78+WbLli0mNTXVPPXUU6Z48eJm6tSp/n1WrlxpihcvbkaPHm02b95sXn/9dVOqVCkzc+bM8N35Agj255ajZ8+e5rzzzgvoNvL7FIu7777bLF261GzdutWsWLHC/OMf/zBxcXF5PsUPiAROHodr1qwxa9asMS1btjQ33XSTWbNmjfn+++/9l69YscLMnj3bbNmyxSxbtsy0b9/e1K5dO9cn1/Xu3dvUqFHDfPTRR2bbtm1mzpw5Jj4+3tx3331W3u/x48ebuXPnmk2bNpn169eboUOHGklm9uzZ/n22b99uypQpY2677TazceNG89FHH5kqVaqYUaNG+fc5dOiQ/+cnyTz99NNmzZo15qeffgrL/QZsFexjcuLEieaDDz4wmzZtMps2bTIvvfSSKVu2rBk+fLh/n8zMTP/jLSEhwdxzzz1mzZo1ZvPmzf59CstjMhTnTIE0i3Mm4E+h+H3v7/L7vcXr86VABPuzOXTokKlZs6a57rrrzPfff28+//xzU79+fdO/f3//PoGcSxaWftuERaki7vnnnzeJiYmmZMmSpkWLFubzzz/3X9a7d2/Ttm1b//dt27Y1kvJ89e7d27/P8OHDTb169UxMTIypUKGCad26tXnzzTfz3O6HH35oGjdubKKjo01ycrKZMmVKKO+m64L5uRljzMGDB01sbGzA9zO/uN9www0mISHBlChRwlSvXt1cc801uX6hBiJNsI/D/PqVmJjov3zp0qWmYcOGJjo62lSqVMn06tXL7Nq1K9cxMjIyzB133GHOOOMMExMTY+rUqWOGDx9uMjMzQ3lXcwnmfj/++OOmbt26/iZfeOGF5uOPP85zzOXLl5vzzjvPREdHmzp16pjRo0ebEydO+C9fsmTJafsPRKpgHpMTJkwwZ511lilVqpQpW7asad68uXnhhRdMVlaWf59t27bl+3j763EK02MyFOdMp2sW50zA/7j9+97f5fd7iw3nS4EItk+pqammQ4cOJjY21tSsWdMMGTLEHD161H95IOeShanftvAZY0wIn4gFAAAAAAAA5MF7SgEAAAAAACDsWJQCAAAAAABA2LEoBQAAAAAAgLBjUQoAAAAAAABhx6IUAAAAAAAAwo5FKQAAAAAAAIQdi1IAAAAAAAAIOxalAAAAAAAAEHYsSiFgl1xyie68886A909LS5PP59PatWtDNpMkLV26VD6fTwcPHgzp7QCwBz0CUNjQLTtE2v0FCoJu2cXn8+m9996TFL6fdTiwKBVmffr0kc/nk8/nU4kSJVSnTh3dc889OnLkSEhuq2vXrq4db86cOXr00UcD3r9WrVras2ePGjdu7NoMKLhFixbpggsuUFxcnBISEnT//ffrxIkTXo8FD9AjBMLtv7scY8aM0TnnnKO4uDhVqVJFXbt21caNG3PtM2fOHF122WWKj48vMideKBi6hYK64IILtGfPHpUrV87rURAh6BZwaixKeeDyyy/Xnj17tHXrVo0aNUovvPCC7rnnnnz3/eOPP0I+T6C3UbFiRcXFxQV83KioKFWrVk3Fixd3Ohpctm7dOl1xxRW6/PLLtWbNGr355pv64IMPNHToUK9Hg0foEbzy+eef69Zbb9WKFSu0YMECnThxQp06dcp1kn7kyBG1adNGY8eO9XBS2IZuoSBKliypatWqyefzeT0KIgjdAk7BIKx69+5trr766lzb+vfvb6pVq2aMMebhhx82TZs2NdOnTze1a9c2Pp/PZGdnm4MHD5oBAwaYypUrm7i4ONOuXTuzdu3ak97Oww8/bCTl+lqyZInZtm2bkWTeeust07ZtWxMdHW1eeukls3//ftOjRw9To0YNExsbaxo3bmzeeOONXMds27atueOOO/zfJyYmmtGjR5u+ffuaMmXKmFq1apnJkyf7L8+5rTVr1hhjjFmyZImRZBYuXGhatmxpYmNjTevWrc2GDRty3c6jjz5qKleubMqUKWP69etn7r//ftO0adOT3tec4/7222/+be+++65p1KiRKVmypElMTDRPPvlkrus8//zzpl69eiY6OtpUqVLFXHvttf7L3nnnHdO4cWMTExNjKlasaC699FJz+PDhk97+0qVLzTnnnGNKlixpqlWrZu6//37zxx9/5Pq53X777ebee+81FSpUMFWrVjUPP/zwSY9njDEnTpwwd911lylXrpypWLGiuffee80tt9yS69+dtm3bmttuu83ccccdpnz58qZKlSpm8uTJ5vDhw6ZPnz6mTJkypk6dOmbevHn+6wwbNsy0atUq123NnTvXxMTEmIyMjFPOhKKHHtGj0/XoZH93xhizbt06065dO/9sAwYMMIcOHfL/HEqUKGGWLVvmP9aTTz5pKlWqZHbv3p3vbe3bt89IMp9//nmey/7+94fIRbfoViDnUX//u5NkEhMT872/M2bMMOXKlTOffvqpSU5ONqVLlzaXXXZZnlZNnz7d/zOpVq2aufXWW085A5CDbtGtQLq1cuVK06FDB1OpUiVTtmxZc/HFF5uUlJRc+0gyc+fONcYUrXMjFqXCLL8o3X777aZSpUrGmD9jkvMfw9WrV5v//ve/Jjs727Rp08ZcddVVZtWqVWbTpk3m7rvvNpUqVTK//vprvrdz6NAh0717d3P55ZebPXv2mD179pjMzEz/v7xJSUlm9uzZZuvWrWbXrl1m586dZty4cWbNmjVmy5YtZsKECSYqKsqsWLHCf8z8olSxYkXz/PPPm82bN5sxY8aYYsWKmdTUVGPMyaN03nnnmaVLl5rvv//eXHTRReaCCy7wH3PmzJkmJibGvPTSS2bjxo3mkUceMWXLlg0qSt9++60pVqyYGTlypNm4caOZMWOGiY2NNTNmzDDGGLNq1SoTFRVl3njjDZOWlmZWr15tnn32WWOMMbt37zbFixc3Tz/9tNm2bZtZt26def755/2/aP3dzp07TalSpcygQYNMamqqmTt3romPj88VnbZt25qyZcuaESNGmE2bNplXXnnF+Hw+M3/+/JPep8cff9yUK1fOvPvuu+aHH34w/fr1M3FxcXkWpeLi4syjjz5qNm3aZB599FFTrFgx07lzZzNlyhSzadMm8+9//9tUqlTJHDlyxBhjzJAhQ8yFF16Y67Y+/fTTXL9oInLQI3p0uh6d7O/uyJEjpnr16uaaa64x3333nVm0aJGpXbu26d27t/+69957r0lMTDQHDx40a9euNdHR0WbOnDkn/dlt3rzZSDLfffddnsuK0okXCoZu0a1AzqNy/s727NljfvzxR1OvXj3Tq1evfO/vjBkzTIkSJUyHDh3MqlWrTEpKimnYsKG56aab/Md74YUXTExMjHnmmWfMxo0bzcqVK8348eNPevvAX9EtuhVItxYtWmRee+0188MPP/h//6tatWquJw6wKAVX/D1K33zzjalUqZLp3r27MebPKJUoUcLs27fPv8+iRYtM2bJlze+//57rWHXr1s21Mn262zLmf//yPvPMM6ed9YorrjB33323//v8otSzZ0//99nZ2aZKlSpm0qRJuW4rv5XyHB9//LGRZI4dO2aMMea8887L83+e2rRpE1SUbrrpJtOxY8dc+9x7772mUaNGxhhjZs+ebcqWLZvvM4NSUlKMJJOWlnbS2/ur//znP+bMM8802dnZ/m3PP/+8KVOmjMnKyjLG/Plz+/tC0DnnnGPuv//+kx43ISHBjB071v/9H3/8YWrWrJlnUeqvxz1x4oQpXbq0/6TLmD9PyiSZr7/+2hhjzGeffWaKFStm3njjDXPixAmzc+dOc+GFFxpJef7PCIo+ekSPjDl9j/L7u5syZYqpUKFCrv+L+PHHH5tixYqZvXv3GmOMyczMNM2bNzfdu3c3Z511lunfv/9JbyM7O9tcddVVeWbLUZROvFAwdItuGXP6buXIzs423bp1My1btjRHjx7N9/7OmDHDSDI//vhjrhmqVq3q/7569epm+PDhAd0n4O/oFt0yJvBu5Thx4oSJi4szH374oX9bUV2U4j2lPPDRRx+pTJkyiomJUevWrXXxxRfrueee81+emJioypUr+79PSUnR4cOHValSJZUpU8b/tW3bNm3ZskXbt2/Ptf2xxx477QytWrXK9X1WVpZGjx6ts88+23878+fP1/bt2095nLPPPtv/Z5/Pp2rVqmnfvn0BXychIUGS/NfZuHGjzj333Fz7//3700lNTVWbNm1ybWvTpo02b96srKwsdezYUYmJiapTp4569eql119/XUePHpUkNW3aVJdeeqmaNGmi66+/XlOnTtVvv/12yttq3bp1rvclaNOmjQ4fPqydO3fme59z7vfJfk7p6enas2ePWrdu7d9WvHjxPH9nfz9uVFSUKlWqpCZNmvi3Va1aVdL/fr6dOnXSuHHjNHDgQEVHR6tBgwa68sor/ddH5KFH9OhUPTrVbTVt2lSlS5fOdVvZ2dn+NysvWbKkZs6cqdmzZ+vYsWN65plnTnq82267TevWrdOsWbOCmgORiW7RrUC79Z///Edff/213nvvPcXGxp50v1KlSqlu3br5Hn/fvn3avXu3Lr300tPeHnAydItuna5b+/bt08CBA9WgQQOVK1dO5cqV0+HDh0/791EU8A5kHmjXrp0mTZqkEiVKqHr16ipRokSuy/96ki9J2dnZSkhI0NKlS/Mcq3z58ipfvnyuTySqWLHiaWf4+2089dRTGj9+vJ555hk1adJEpUuX1p133qnjx4+f8jh/n93n8yk7Ozvg6+Q8mP96nb+/8aQx5pTH+ztjzCmPERcXp9WrV2vp0qWaP3++HnroIY0YMUKrVq1S+fLltWDBAi1fvlzz58/Xc889p+HDh+ubb75R7dq1g7qtv2538nMKRH7HPd3Pd8iQIbrrrru0Z88eVahQQWlpaRo2bFi+9w9FHz2iR056lN9t/fV4OZYvXy5JOnDggA4cOJDn71qSbr/9dn3wwQdatmyZatasGdQciEx0i24F8nOaOXOmxo8fr6VLl562LfkdP2eOUy1mAYGiW3TrdD+nPn366JdfftEzzzyjxMRERUdHq3Xr1qf9+ygKeKaUB0qXLq169eopMTExz7+s+WnRooX27t2r4sWLq169erm+4uPj82zPiVLJkiWVlZUV0ExffPGFrr76avXs2VNNmzZVnTp1tHnz5gLdTyfOPPNMrVy5Mte2b7/9NqhjNGrUSF9++WWubcuXL1eDBg38zwYqXry4OnTooCeeeELr1q1TWlqaFi9eLOnPYLRp00aPPPKI1qxZo5IlS2ru3Lknva3ly5fnit7y5csVFxenGjVqBDV3jnLlyikhIUErVqzwbztx4oRSUlIcHS8/Pp9P1atXV2xsrGbNmqVatWqpRYsWrh0fhQc9Ojl69Kf8/u4aNWqktWvX5vqkvK+++krFihVTgwYNJElbtmzRXXfdpalTp+r888/XLbfckutkzBij2267TXPmzNHixYtZGEfA6NbJ0a0/ff311+rfv78mT56s888/3/FxpD9/mU1KStKiRYsKdBxENrp1cnTrT1988YUGDx6sK664QmeddZaio6O1f/9+x8crTFiUKgQ6dOig1q1bq2vXrvrss8+Ulpam5cuX64EHHjjlAzYpKUnr1q3Txo0btX///lN+9Ge9evX8K8Spqan617/+pb1794bi7pzS7bffrunTp+uVV17R5s2bNWrUKK1bty6oj+29++67tWjRIj366KPatGmTXnnlFU2cONH/sasfffSRJkyYoLVr1+qnn37Sq6++quzsbJ155pn65ptv9Nhjj+nbb7/V9u3bNWfOHP3yyy9q2LBhvrc1aNAg7dixQ7fffrs2bNig999/Xw8//LCGDBmiYsWcP7zuuOMOjR07VnPnztWGDRs0aNAgHTx40PHx/mrcuHH67rvv9P333+vRRx/V2LFjNWHCBF6+h4DQo8jrUX5/dzfffLNiYmLUu3dvrV+/XkuWLNHtt9+uXr16qWrVqsrKylKvXr3UqVMn9e3bVzNmzND69ev11FNP+Y976623aubMmXrjjTcUFxenvXv3au/evTp27Jh/nwMHDmjt2rX64YcfJP35FP+1a9d68u8DCi+6FVnd2rt3r7p166YePXrosssu87fll19+cXQ8SRoxYoSeeuopTZgwQZs3b9bq1atzvfQKcBvdiqxuSX/+fbz22mtKTU3VN998o5tvvjlinqnJolQh4PP5NG/ePF188cX65z//qQYNGqhHjx5KS0vzv2dQfgYMGKAzzzxTrVq1UuXKlfXVV1+ddN8HH3xQLVq00GWXXaZLLrlE1apVU9euXUNwb07t5ptv1rBhw3TPPfeoRYsW2rZtm/r06aOYmJiAj9GiRQu9/fbbevPNN9W4cWM99NBDGjlypPr06SPpz6e8zpkzR+3bt1fDhg314osvatasWTrrrLNUtmxZLVu2TFdccYUaNGigBx54QE899ZQ6d+6c723VqFFD8+bN08qVK9W0aVMNHDhQ/fr10wMPPFCgn8Pdd9+tW265RX369FHr1q0VFxenbt26FeiYOT755BNddNFFatWqlT7++GO9//77nvxdo3CiR5HXo/z+7kqVKqXPPvtMBw4c0DnnnKPrrrtOl156qSZOnChJGj16tNLS0jRlyhRJUrVq1TRt2jQ98MAD/pcbTJo0Senp6brkkkuUkJDg/3rrrbf8t/3BBx+oefPm/ve+69Gjh5o3b64XX3yxQPcJkYVuRVa3NmzYoJ9//lmvvPJKrracc845jo/Zu3dvPfPMM3rhhRd01lln6R//+IcnzyhB5KBbkdUtSXrppZf022+/qXnz5urVq5cGDx6sKlWqFOiYhYXPBPuCTSDMOnbsqGrVqum1117zehRP9enTRwcPHtR7773n9ShAxKJHAAobugWgsKFbkYU3OodVjh49qhdffFGXXXaZoqKiNGvWLC1cuFALFizwejQAEYYeAShs6BaAwoZugUUpWCXnqaqjRo1SZmamzjzzTM2ePVsdOnTwejQAEYYeAShs6BaAwoZugZfvAQAAAAAAIOx4o3MAAAAAAACEHYtSAAAAAAAACDsWpQAAAAAAABB2LEoBAAAAAAAg7FiUAgAAAAAAQNixKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMKORSkAAAAAAACE3f8DHGTvV2Jked0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, axs = plt.subplots(nrows=1, ncols=4, figsize=(12, 4))\n", + "for i, loss_label in enumerate(PT_TASKS + [\"all\"]):\n", + " draw_boxplot(results, \"lipophilicity\", loss_label=loss_label, ax=axs[i])\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
toymix_gcn_1toymix_gcn_2
caco20.5025650.523741
lipophilicity0.4789700.041120
\n", + "
" + ], + "text/plain": [ + " toymix_gcn_1 toymix_gcn_2\n", + "caco2 0.502565 0.523741\n", + "lipophilicity 0.478970 0.041120" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols = sorted(list(results.keys()))\n", + "\n", + "rows = set()\n", + "for k in cols: \n", + " rows.update(results[k].ft_results.keys())\n", + "rows = sorted(list(rows))\n", + "\n", + "data = pd.DataFrame(columns=cols, index=rows, dtype=float)\n", + "\n", + "for pt_label, pt_results in results.items(): \n", + " for ft_label, ft_results in pt_results.ft_results.items(): \n", + " data.loc[ft_label, pt_label] = ft_results.best(FT_METRICS[ft_label], minimize=\"mae\" in FT_METRICS[ft_label])\n", + "\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGdCAYAAACPX3D5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAz3UlEQVR4nO3deVhV5f7//xcgk6iIIoiKgFrOI1qhOXQ0yzLTsmzSyilLLaO+Fulx6hRm5tD55NjRstScU0tTMqdS00PY6JBNKIIKOGQm4/r94a992htQ2K7NBtfz0bWuy33vte71Xl4tfPO+73UvD8MwDAEAAMvydHcAAADAvUgGAACwOJIBAAAsjmQAAACLIxkAAMDiSAYAALA4kgEAACyOZAAAAIsjGQAAwOIquDuAv2T9uMvdIQBljmdQTXeHAJRJ3sH1XNp/TvrPpvXl6ljNUGaSAQAAyoz8PHdHUKpIBgAAcGTkuzuCUsWcAQAALI7KAAAAjvKtVRkgGQAAwIHBMAEAALASKgMAADhimAAAAItjmAAAAFgJlQEAAByx6BAAABbHMAEAALASKgMAADiy2NMEVAYAAHBgGPmmbSU1a9YsRUVFyc/PT9HR0dq5c2eR+27btk0eHh4FtoMHD5bonFQGAABw5KbKwLJlyzRq1CjNmjVLHTp00Ny5c9WjRw/98MMPqlu3bpHHHTp0SFWqVLF9rlGjRonOS2UAAIAyYtq0aRo0aJAGDx6sxo0ba8aMGQoPD9fs2bMve1xISIhq1qxp27y8vEp0XpIBAAAcGfmmbVlZWTp37pzdlpWVVeCU2dnZSkxMVPfu3e3au3fvrl27dl023NatWyssLExdu3bV1q1bS3y5JAMAADjKzzNti4+PV2BgoN0WHx9f4JTp6enKy8tTaGioXXtoaKjS0tIKDTMsLEzz5s3TqlWrtHr1ajVs2FBdu3bVjh07SnS5zBkAAMCF4uLiFBsba9fm6+tb5P4eHh52nw3DKND2l4YNG6phw4a2zzExMTp69KimTp2qTp06FTtGkgEAAByZuOiQr6/vZf/x/0twcLC8vLwKVAFOnjxZoFpwOTfddJPef//9EsXIMAEAAI7y883bisnHx0fR0dFKSEiwa09ISFD79u2L3U9SUpLCwsKKvb9EZQAAgDIjNjZW/fv3V9u2bRUTE6N58+YpOTlZw4YNk3RpyCElJUWLFi2SJM2YMUORkZFq2rSpsrOz9f7772vVqlVatWpVic5LMgAAgCM3vZugX79+ysjI0KRJk5SamqpmzZppw4YNioiIkCSlpqYqOTnZtn92draef/55paSkyN/fX02bNtXHH3+sO+64o0Tn9TAMwzD1SpyU9ePlH5sArMgzqKa7QwDKJO/gei7tP+ubTab15dviNtP6chXmDAAAYHEMEwAA4MAw8twdQqkiGQAAwJGb5gy4C8kAAACOeIUxAACwEioDAAA4YpgAAACLy7fWBEKGCQAAsDgqAwAAOGKYAAAAi+NpAgAAYCVUBgAAcMQwAQAAFscwAQAAsBIqAwAAOLJYZYBkAAAAB7y1EAAAq7NYZYA5AwAAWByVAQAAHPFoIQAAFscwAQAAsBIqAwAAOGKYAAAAi2OYAAAAWAmVAQAAHDFMAACAxTFMAAAArITKAAAAjixWGSAZAADAEXMGAACwOItVBpgzAACAxVEZAADAEcMEAABYHMMEAADASqgMAADgiGECAAAsjmECAABgJVQGAABwZLHKAMkAAACODMPdEZQqhgkAALA4KgMAADhimAAAAIsjGQAAwOIsts4AcwYAALA4KgMAADhimAAAAIvj0UIAAGAlVAYAAHDEMAEAABZnsWSAYQIAACyuxMnAxx9/rMGDB2v06NE6ePCg3XenT5/WP/7xD9OCAwDALYx887ZyoETJwJIlS3T33XcrLS1Nu3fvVuvWrbV48WLb99nZ2dq+fbvpQQIAUJqMfMO0rTwo0ZyBqVOnavr06Ro5cqQkaeXKlXr88cd18eJFDRo0yCUBAgBQ6iw2Z6BEycDhw4fVs2dP2+e+ffsqODhYvXr1Uk5Ojvr06WN6gAAAwLVKlAxUqVJFJ06cUFRUlK2tS5cuWr9+vXr27Kljx46ZHiAAAKWunIz1m6VEcwZuuOEGbdy4sUB7586dtX79es2YMcOsuAAAcJ98w7ytHChRMvDss8/Kz8+v0O+6dOmijz76SAMGDDAlMAAAUDo8DKNsLMCc9eMud4cAlDmeQTXdHQJQJnkH13Np/xf+/ZRpfVUcOcu0vlzF6RUI8/Ly9OGHH+rAgQPy8PBQ48aNdffdd8vLy8vM+AAAKH08TXBlR44c0Z133qljx46pYcOGMgxDhw8fVnh4uD7++GPVr1/f7DgBAICLOLUc8dNPP6169erp6NGj+uqrr5SUlKTk5GRFRUXp6aefNjtGAABKl2GYt5UDTlUGtm/frj179qhatWq2turVq2vy5Mnq0KGDacHBNT74+DO9s3qj0jPPqH7d2ho95CFFN7u+0H33fXNQg156rUD72tmvKio8zNWhAi7zweqPtHDJSp3KyFSDqAi98PQTim7VrNB99371jQaOfKFA+7ol81QvIlyStHLdRq3buEVHfvlNktSkYQM988Rjat6koesuAq7DMMGV+fr66vfffy/Qfv78efn4+Fx1UHCdT3Z8qSnzl2jMk/3Vusl1WrFxm56aME0fznpFYSHVizxu3dx4Varob/scVKVyaYQLuMTGT7dr8sy5GvvccLVu0UQrPtygYc//U+ven6uwmiFFHvfR0vmqFFDR9jmoaqDtz/u++kZ33NpFrZo1lo+vjxYsXqGhz47Rh+/PUWiNYJdeD3C1nBom6Nmzp4YOHaovv/xShmHIMAzt2bNHw4YNU69evcyOESZa9OFm9bm1k+69rbPqhdfSC0MfUs3galq+4bPLHlctsIqCgwJtm5cXL7xE+bVo2Rrd07O7+va6XfUj6+rFUcNUM6SGPljz8WWPqxZUVcHVq9m2v0+Yfm3CC3rgnp5qdH191YsI18QXnlF+fr72/He/i68GLmGxdQacqgy8+eabevTRRxUTEyNvb29JUm5urnr16qWZM2eaGiDMk5OTqwNHftWgvnfYtce0bqr9B3+67LH3PzNe2dk5qhdeS0MfuEs3tGjsylABl8nJydEPh37UoEfus2tvf0Mbff3dD5c99r7HRygrO1v1I+vqiUcf1A3RLYvc9+LFLOXm5imQKlr5xAqEV1a1alWtXbtWhw8f1sqVK7VixQodOnRIa9asUWBg4JU7gFucPve78vLzVT2oil179aBApZ8+W+gxwdUCNW7EY5oWN1zTXxqhyDo1NWTM6/rvd4dKI2TAdKfPnFNeXr6qVwuya68eVFXpGacLPaZG9Wqa8MLTmv7KWM149Z+KrFtHg56J03/3f1vkeabPWaiQGtUV07a1qfGjlLixMjBr1ixFRUXJz89P0dHR2rlzZ7GO++KLL1ShQgW1atWqxOd0ep0BSWrQoIEaNGhQ4uOysrKUlZVl35idLV/mG5QKD3nYfTYMQx4ehe8bVSdMUXX+N1GwZeMGSjuVqXdXf6K2zZgYhfLLw+F/ekNGgba/REXUUVREHdvnVs0aK+3kKb2zZJXatmpeYP8Fi1doQ8I2Lfy/KfL15ecaim/ZsmUaNWqUZs2apQ4dOmju3Lnq0aOHfvjhB9WtW7fI486ePasBAwaoa9euOnHiRInP61RloG/fvpo8eXKB9tdff1333XdfIUfYi4+PV2BgoN02Zc57zoSCEgiqUllenp4FqgCZZ86petXiV3RaNKqv5OMl/58NKAuCqlaRl5en0jMy7dozT59V9WpVi91Pi6aN9Nux4wXaFy5ZqfmLlmne9FfUsEFUIUeiPDDy803bSmLatGkaNGiQBg8erMaNG2vGjBkKDw/X7NmzL3vcE088oYceekgxMTFOXa9TycD27dt15513Fmi//fbbtWPHjiseHxcXp7Nnz9pto4f1dyYUlIC3dwU1bhCp3fu/t2vfs/8HtWpU/IWiDv6UrOBqDAehfPL29laThtdp974ku/bd+75Sy2ZNit3PwcM/qUb1anZtCxav1Nx3lmrOGy+rWePCH9dFOWHiMEFWVpbOnTtntxWojkvKzs5WYmKiunfvbtfevXt37dpV9JL9Cxcu1E8//aTx48c7fblOJQNFPULo7e2tc+fOXfF4X19fValSxW5jiKB0DOjdXas379CazTv089HjmjJ/qVJPZei+O26RJM18Z4VeemO+bf/31m7WZ7u/0m8paTryW4pmvrNCn+76rx7s2dVdlwBctQH9+mjV+k1a/dEm/fRrsl6bOVepJ06pX59Lk2unz16ouJen2vZ/b9kabdmxS78dTdGRn3/T9NkLlbDtCz147122fRYsXqF/z39XL8c9q9phoUrPyFR6RqYuXPiz1K8PZUth1fD4+PgC+6WnpysvL0+hoaF27aGhoUpLSyu07x9//FEvvviiFi9erAoVnB/5d+rIZs2aadmyZRo3bpxd+wcffKAmTYqfWaP03d7pRp35/Q/N/WCdTmWeVYOI2nprwrOqFXLpOehTp88q7VSGbf+cnFy9sWCZTmaclq+Pj+rXraW3xo9Sx3ZFz6IGyroe3Trr7LnfNWfhEp3KyNR19SI1e+ok1ap56YdwekamUk+ctO2fk5urqf/3tk6eypCvr48aREVo1usT1an9DbZ9Plj9kXJycvXs2FfszvXkwIc1fNAjpXNhMI+JTxPExcUpNjbWrs3X17fI/QvMZzEKn8+Sl5enhx56SBMnTtT1119dJcqptxauW7dO9957rx566CH94x//kCRt2bJFS5cu1YoVK9S7d+8SB8JbC4GCeGshUDhXv7Xwj0kPm9ZXwLjFxdovOztbFStW1IoVK9SnTx9b+zPPPKP9+/dr+/btdvufOXNGQUFBdutd5OfnyzAMeXl5afPmzbZ/o6/EqcpAr1699OGHH+rVV1/VypUr5e/vrxYtWujTTz9V586dnekSAABL8/HxUXR0tBISEuySgYSEBN19990F9q9SpYq+/db+8dZZs2bps88+08qVKxUVVfwJrE4PMNx5552FTiIEAKDcc9O7CWJjY9W/f3+1bdtWMTExmjdvnpKTkzVs2DBJl4YcUlJStGjRInl6eqpZM/v3aYSEhMjPz69A+5Vc1ToDAABck9y0jHC/fv2UkZGhSZMmKTU1Vc2aNdOGDRsUEREhSUpNTVVycrLp53VqzkBeXp6mT5+u5cuXKzk5WdnZ2XbfZ2ZmFnFk0ZgzABTEnAGgcC6fMzDuAdP6Cpj0gWl9uYpTjxZOnDhR06ZN0/3336+zZ88qNjZW99xzjzw9PTVhwgSTQwQAoJQZ+eZt5YBTycDixYs1f/58Pf/886pQoYIefPBBvf322xo3bpz27NljdowAAJQui7210KlkIC0tTc2bX1qPu1KlSjp79tLytj179tTHH1/+FaAAAJR17lqO2F2cSgbq1Kmj1NRUSZdeVrR582ZJ0r59+y67kAIAACh7nEoG+vTpoy1btki6tBjCP//5T1133XUaMGCABg4caGqAAACUOosNEzj1aOHf31jYt29fhYeH64svvlCDBg3Uq1cv04IDAMAtysk/4mZxqjIQHx+vBQsW2D7feOONio2NVXp6ul577TXTggMAAK7nVDIwd+5cNWrUqEB706ZNNWfOnKsOCgAAt7LYo4VODROkpaUpLCysQHuNGjVsEwsBACi3GCa4sr/mCDj64osvVKtWrasOCgAAlB6nKgODBw/WqFGjlJOTY/cK49GjR+u5554zNUAAAEqbYbHKgFPJwOjRo5WZmamnnnrK9l4CPz8/vfDCC4qLizM1QAAASp3FkgGnXlT0l/Pnz+vAgQPy9/fXddddd1ULDvGiIqAgXlQEFM7VLyr6/emepvVV+c2PTOvLVa7qFcaVKlVSu3btzIoFAICyoZwsI2yWq0oGAAC4JllsmIBkAAAARxZLBpx6tBAAAFw7qAwAAODgKubWl0skAwAAOGKYAAAAWAmVAQAAHFmsMkAyAACAA6stR8wwAQAAFkdlAAAARxarDJAMAADgyFqrETNMAACA1VEZAADAgdUmEJIMAADgiGQAAACLY84AAACwEioDAAA4YM4AAABWxzABAACwEioDAAA4YJgAAACrY5gAAABYCZUBAAAcGBarDJAMAADgyGLJAMMEAABYHJUBAAAcMEwAAIDVkQwAAGBtVqsMMGcAAACLozIAAIADq1UGSAYAAHBgtWSAYQIAACyOygAAAI4MD3dHUKpIBgAAcMAwAQAAsBQqAwAAODDyGSYAAMDSGCYAAACWQmUAAAAHBk8TAABgbVYbJiAZAADAgdUmEDJnAAAAi6MyAACAA8NwdwSli2QAAAAHDBMAAABLoTIAAIADq1UGSAYAAHBgtTkDDBMAAGBxVAYAAHDAMAEAABZnteWIGSYAAMDiqAwAAODAau8moDIAAICDfMPDtK2kZs2apaioKPn5+Sk6Olo7d+4sct/PP/9cHTp0UPXq1eXv769GjRpp+vTpJT4nlQEAABy4a87AsmXLNGrUKM2aNUsdOnTQ3Llz1aNHD/3www+qW7dugf0DAgI0YsQItWjRQgEBAfr888/1xBNPKCAgQEOHDi32eT0Mo2w8TZn14y53hwCUOZ5BNd0dAlAmeQfXc2n/hxr1MK2vhgc3FnvfG2+8UW3atNHs2bNtbY0bN1bv3r0VHx9frD7uueceBQQE6L333iv2eRkmAADAgZHvYdqWlZWlc+fO2W1ZWVkFzpmdna3ExER1797drr179+7atat4vzAnJSVp165d6ty5c4mul2QAAAAHhmHeFh8fr8DAQLutsN/y09PTlZeXp9DQULv20NBQpaWlXTbeOnXqyNfXV23bttXw4cM1ePDgEl0vcwYAAHChuLg4xcbG2rX5+voWub+Hh/18BcMwCrQ52rlzp86fP689e/boxRdfVIMGDfTggw8WO0aSAQAAHJi5AqGvr+9l//H/S3BwsLy8vApUAU6ePFmgWuAoKipKktS8eXOdOHFCEyZMKFEywDABAAAO3PFooY+Pj6Kjo5WQkGDXnpCQoPbt2xe7H8MwCp2TcDlUBgAAKCNiY2PVv39/tW3bVjExMZo3b56Sk5M1bNgwSZeGHFJSUrRo0SJJ0ltvvaW6deuqUaNGki6tOzB16lSNHDmyROclGQAAwIG71hno16+fMjIyNGnSJKWmpqpZs2basGGDIiIiJEmpqalKTk627Z+fn6+4uDj98ssvqlChgurXr6/JkyfriSeeKNF5WWcAKMNYZwAonKvXGfgm8i7T+mrx63rT+nIV5gwAAGBxDBMAAODAmXcKlGckAwAAOHDXnAF3IRkAAMBB2ZhNV3qYMwAAgMVRGQAAwAFzBtzEOJ3i7hCAMse/6X3uDgEok3KzXftvhtXmDDBMAACAxZWZygAAAGUFwwQAAFicxR4mYJgAAACrozIAAIADhgkAALA4niYAAACWQmUAAAAH+e4OoJSRDAAA4MCQtYYJSAYAAHCQb7FnC5kzAACAxVEZAADAQT7DBAAAWJvV5gwwTAAAgMVRGQAAwAGPFgIAYHEMEwAAAEuhMgAAgAOGCQAAsDirJQMMEwAAYHFUBgAAcGC1CYQkAwAAOMi3Vi5AMgAAgCOrLUfMnAEAACyOygAAAA4s9gZjkgEAABzxaCEAALAUKgMAADjI97DWBEKSAQAAHFhtzgDDBAAAWByVAQAAHFhtAiHJAAAADqy2AiHDBAAAWByVAQAAHFhtOWKSAQAAHFjtaQKSAQAAHDBnAAAAWAqVAQAAHPBoIQAAFme1OQMMEwAAYHFUBgAAcGC1CYQkAwAAOLDanAGGCQAAsDgqAwAAOLBaZYBkAAAAB4bF5gwwTAAAgMVRGQAAwAHDBAAAWBzJAAAAFscKhAAAwFKoDAAA4IAVCAEAsDirzRlgmAAAAIujMgAAgAOrVQZIBgAAcMDTBMXwxx9/mB0HAABwE6eSgdDQUA0cOFCff/652fEAAOB2+R7mbeWBU8nA0qVLdfbsWXXt2lXXX3+9Jk+erOPHj5sdGwAAbpFv4lYeOJUM3HXXXVq1apWOHz+uJ598UkuXLlVERIR69uyp1atXKzc31+w4AQCwhFmzZikqKkp+fn6Kjo7Wzp07i9x39erVuvXWW1WjRg1VqVJFMTEx2rRpU4nPeVWPFlavXl3PPvusvv76a02bNk2ffvqp+vbtq1q1amncuHG6cOHC1XQPAIBbGCZuJbFs2TKNGjVKY8aMUVJSkjp27KgePXooOTm50P137NihW2+9VRs2bFBiYqJuueUW3XXXXUpKSirReT0Mw3B60mRaWpoWLVqkhQsXKjk5WX369NGgQYN0/PhxTZ48WWFhYdq8eXOx+rq4d4WzYQDXrEo3j3J3CECZlJud4tL+X4l42LS+xvy2uNj73njjjWrTpo1mz55ta2vcuLF69+6t+Pj4YvXRtGlT9evXT+PGjSv2eZ16tHD16tVauHChNm3apCZNmmj48OF65JFHVLVqVds+rVq1UuvWrZ3pHgAAtzJzrD8rK0tZWVl2bb6+vvL19bVry87OVmJiol588UW79u7du2vXrl3FOld+fr5+//13VatWrUQxOjVM8Pjjj6tWrVr64osvtH//fo0YMcIuEZCkevXqacyYMc50DwDANSM+Pl6BgYF2W2G/5aenpysvL0+hoaF27aGhoUpLSyvWud544w398ccfuv/++0sUo1OVgdTUVFWsWPGy+/j7+2v8+PHOdA8AgFuZuehQXFycYmNj7docqwJ/5+Fh/zyiYRgF2gqzdOlSTZgwQWvXrlVISEiJYnQqGahcubJSU1MLnCwjI0MhISHKy8tzplsAAMoEM4cJChsSKExwcLC8vLwKVAFOnjxZoFrgaNmyZRo0aJBWrFihbt26lThGp4YJippzmJWVJR8fH2e6BADA0nx8fBQdHa2EhAS79oSEBLVv377I45YuXarHHntMS5Ys0Z133unUuUtUGXjzzTclXSphvP3226pUqZLtu7y8PO3YsUONGjVyKhAAAMoKd60cGBsbq/79+6tt27aKiYnRvHnzlJycrGHDhkm6NOSQkpKiRYsWSbqUCAwYMEAzZ87UTTfdZKsq+Pv7KzAwsNjnLVEyMH36dEmXKgNz5syRl5eX7TsfHx9FRkZqzpw5JekSAIAyJ99Nryrq16+fMjIyNGnSJKWmpqpZs2basGGDIiIiJF2as/f3NQfmzp2r3NxcDR8+XMOHD7e1P/roo3rnnXeKfV6n1hm45ZZbtHr1agUFBZX00CKxzgBQEOsMAIVz9ToDYyMfMq2vf/26xLS+XMWpCYRbt241Ow4AAMoMq73CuNjJQGxsrF5++WUFBAQUeETC0bRp0646MAAA3KW8vGDILMVOBpKSkpSTk2P7c1GK8ywkAAAoO4qdDPx9aIBhAgDAtcxdEwjdxal1Bs6ePavMzMwC7ZmZmTp37txVBwUAgDu5662F7uJUMvDAAw/ogw8+KNC+fPlyPfDAA1cdFAAA7pRv4lYeOJUMfPnll7rlllsKtHfp0kVffvnlVQcFAABKj1OPFmZlZSk3N7dAe05Ojv7888+rDgoAAHdizkAxtGvXTvPmzSvQPmfOHEVHR191UAAAuJPV5gw4VRl45ZVX1K1bN3399dfq2rWrJGnLli3at2+fNm/ebGqAAADAtZyqDHTo0EG7d+9WeHi4li9frvXr16tBgwb65ptv1LFjR7NjBACgVFltAqFTlQFJatWqlRYvXmxmLAAAlAlGuSnwm6PYycC5c+dUpUoV258v56/9AABA2VfsZCAoKEipqakKCQlR1apVC1122DAMeXh4KC8vz9QgAQAoTeWlvG+WYicDn332mapVqyaJ5YgBANc2qz1aWOxkoHPnzoX+GQAAlG/FTga++eabYnfaokULp4IBAKAssFZdoATJQKtWreTh4SHDuPxfEXMGypZln36pdz7eqfSz51W/dohGP3KH2jSMvOJxSYd/06BX/qMGdUK0/JURdt+9/8kuLd+yV2kZZ1S1ckXd2q6Znr7/Vvn6eLvoKoCrM+yJR/Vc7DCFhYXo+x8O67nnxuvzL/YWuX+njjfp9dfHq2mT63X8+AlNfWO25s1/r9B977+/l5a8P1tr132ie/sOsrV3vPlGPffck2rTurlq1aqpe/oO1Lp1m0y/NrgGwwRF+OWXX1wZB1zgkz3fasr7GzTmsbvU6rq6Wrl1n556fZHWTH5aYcFVizzu9wsXNXbuSt3QtJ4yz563++7jL/Zr5vLNmji4j1peV1e/paVr3LzVkqT/98gdrrwcwCn33ddL096YoBEjX9Ku3fs0ZHB/fbT+fTVv2UVHjx4vsH9kZLjWr3tPb/9niR59bKTax7TT//37VZ1Kz9CaNRvs9q1bt7amTB6nnTv3FOgnIKCivvnmB73z7jKtXP62y64PrsEEwiJERES4Mg64wHsbv1CfztG6p0tbSdLoR+7Urm+PaPmWvXqmX/cij3t5wVr1iGkpL08PbU08YPfd10eOqtV1dXVH+5aSpNo1gnR7TAt999Mx110IcBWefWaIFiz8QAsWLpUkPff8eHXv3lnDnhigMWMnF9j/iaH9lXw0Rc89P16SdPDgEUVHt9Rzzw6zSwY8PT313rv/p4mTpurmm29U1ar2j1R/smmrPtnEZGuUD8VOBtatW6cePXrI29tb69atu+y+vXr1uurAcHVycnN14NfjGnhXJ7v2mGYN9PWPyUUe9+GORB07malXn+yr+Wu3Ffi+9fUR2rDra3370zE1r19Hx05m6vOvD+uum1uZfAXA1fP29labNi302utv2bUnJGxXzE1tCz3mphujlZCw3a5tc8I2DXz8AVWoUMH2krZ/jn1Wp9IztPCdD3TzzTe65gLgNiw6VITevXsrLS1NISEh6t27d5H7MWegbDj9+wXl5eerepVKdu3VAwOU7lD6/8tvaemauWyzFo4dogpeXoXu0yOmhU7//ocee3m+JEO5efm6v+sNGnQXT5ig7AkOrqYKFSro5Il0u/aTJ9MVWjOk0GNCa4bo5EmH/U+ky9vbW8HB1ZSWdlLtY9rq8cceVHS7W10WO9yLYYIi5OfnF/pnZ2RlZSkrK8uuzcjOYQKaCziuDWUYBdskKS8/X3GzVujJe7oqMiy4yP72HfhZb6/brjGP3aXm9eso+USmprz/sYI/3Konet9icvSAORwnPl9pMnTB/f/XXqlSgN59598a9uT/U0bGadNjBdzB6XcTXI34+HhNnDjRrm3M4L4aO+R+d4RzTQqqXFFenp4FqgCZ5/4oUC2QpD/+zNL3v6To4G+pmrzoI0lSvmHIMAy1eXScZo9+VDc2ra+3Vm5Rzw6tbPMQrguvqT+zsvXygrUa0quzPD2devcV4BLp6ZnKzc1VaM0adu01alTXyROnCj3mRNpJhYY67B8SrJycHGVknFbTpg0VFVVXH655x/b9X//fX7zwm5o066Sff/7N3AtBqWOYoJi2bNmiLVu26OTJkwUqBQsWLLjssXFxcYqNjbVrM775yNlQUAjvChXUOLKW9nx3RF3bNrG17/nuiLq0aVxg/0r+vlr56ki7tuVbvtTeH37W1JEPqnaNIEnSxeycAktRe3le+i3LWrcOyoOcnBx99dU36ta1k9au/cTW3q1bJ61fX/hjfnu+TNSdd9qX/2/t1lmJid8oNzdXBw8eUcvW/7D7ftLE0apcqZKefW5coU8ooPxhmKAYJk6cqEmTJqlt27YKCwsr9D0Fl+Pr6ytfX1+7tosMEZiuf48OGjNnpZpE1VbLBuFatfW/Ss04q/u6tpMkzVy2WSdPn9Mrw/rK09NT14WH2h1frUqAfL0r2LV3bt1Q723cpUYRYWpev46OnsjUWyu3qHObRvKiKoAyaPrM+Xp34UwlJn6tPV8masigR1Q3vLbmzru0bsAr/3pRtWqF6fGBz0iS5s57T089+bimThmvtxcs1k03Rmvg4w/o4f7DJV0a5vz++0N25zhz5tLL2/7eHhBQUQ0aRNk+R0XWVcuWTZWZeZqEAWWOU8nAnDlz9M4776h///5mxwMT3X5Tc509f0HzPtyqU2d+V4M6oXrr+f6qFXzpt/z0M78rLeNMifoccncXechDb638VCdPn1NQlQB1btVII+7r5oIrAK7eihXrVL1akMaOeVZhYSH67vtDuqtXfyUnp0iSatYMVd3wWrb9f/31qO7q1V9Tp07Qk08+quPHT2jUs+MKrDFwJW2jW2rLpyttn9+YOkGS9O6i5Ro0+NmrvzC4VP4VFti71ngYV1pSsBDVq1fX3r17Vb9+fdMCubh3hWl9AdeKSjePcncIQJmUm53i0v4fibjHtL7e/221aX25ilN13cGDB2vJkiVmxwIAANyg2MMEf5/wl5+fr3nz5unTTz9VixYt5O1tP94/bdo08yIEAKCU8W6CIiQlJdl9btWqlSTpu+++s2sv6WRCAADKGqs9H1XsZGDrVtbYBgBYg9UeLbzqZ8GOHj2qY8d4SQ0AAOWVU8lAbm6u/vnPfyowMFCRkZGKiIhQYGCgxo4dq5ycHLNjBACgVOXLMG0rD5xaZ2DEiBFas2aNpkyZopiYGEnS7t27NWHCBKWnp2vOnDmmBgkAQGlizkAxLF26VB988IF69Ohha2vRooXq1q2rBx54gGQAAIByxKlkwM/PT5GRkQXaIyMj5ePjc7UxAQDgVkwgLIbhw4fr5ZdftnsNcVZWll555RWNGDHCtOAAAHAH4/9/a6sZW3ngVGUgKSlJW7ZsUZ06ddSyZUtJ0tdff63s7Gx17dpV99zzv2UcV68u+8swAgBgZU4lA1WrVtW9995r1xYeHm5KQAAAuFt5eQrALE4lAwsXLjQ7DgAAygyrzRlwKhn4y6lTp3To0CF5eHjo+uuvV40aNcyKCwAAlBKnJhD+8ccfGjhwoMLCwtSpUyd17NhRtWrV0qBBg3ThwgWzYwQAoFQZJv5XHjiVDMTGxmr79u1av369zpw5ozNnzmjt2rXavn27nnvuObNjBACgVLECYTGsWrVKK1euVJcuXWxtd9xxh/z9/XX//fdr9uzZZsUHAECpKy+PBJrFqcrAhQsXFBoaWqA9JCSEYQIAAMoZp5KBmJgYjR8/XhcvXrS1/fnnn5o4caLtXQUAAJRX+SZu5YFTwwQzZ87U7bffblt0yMPDQ/v375efn582bdpkdowAAJSq8jLxzyxOJQPNmjXTjz/+qPfff18HDx6UYRh64IEH9PDDD8vf39/sGAEAgAs5vc6Av7+/hgwZYmYsAACUCeXlKQCzOJ0MHDp0SP/+97914MABeXh4qFGjRhoxYoQaNWpkZnwAAJQ6niYohpUrV6pZs2ZKTExUy5Yt1aJFC3311Vdq3ry5VqxYYXaMAADAhZyqDIwePVpxcXGaNGmSXfv48eP1wgsv6L777jMlOAAA3MFqwwROVQbS0tI0YMCAAu2PPPKI0tLSrjooAADcieWIi6FLly7auXNngfbPP/9cHTt2vOqgAABwp3zDMG0rD5waJujVq5deeOEFJSYm6qabbpIk7dmzRytWrNDEiRO1bt06u30BAEDZ5WE4MWXS07N4BQUPDw/l5eUVa9+Le5l4CDiqdPMod4cAlEm52Sku7b9j7a6m9bUzZYtpfbmKU5WB/PzyssAiAAAlxwRCAABgKcWuDLz55psaOnSo/Pz89Oabb15236effvqqAwMAwF2sVhko9pyBqKgo/fe//1X16tUVFRVVdIceHvr5559LHAhzBoCCmDMAFM7VcwZuqtXFtL72HN9mWl+uUuzKwC+//FLonwEAQPlW7GQgNja2WPt5eHjojTfecDogAADczWrDBMVOBpKSkoq1n4eHh9PBAABQFpSXlQPNUuxkYOvWra6MAwAAuInTrzAGAOBaxSuMAQCwuHwZpm0lNWvWLEVFRcnPz0/R0dGFvgvoL6mpqXrooYfUsGFDeXp6atSoUU5dL8kAAAAODMMwbSuJZcuWadSoURozZoySkpLUsWNH9ejRQ8nJyYXun5WVpRo1amjMmDFq2bKl09dLMgAAQBkxbdo0DRo0SIMHD1bjxo01Y8YMhYeHa/bs2YXuHxkZqZkzZ2rAgAEKDAx0+rzMGQAAwIGZjxZmZWUpKyvLrs3X11e+vr52bdnZ2UpMTNSLL75o1969e3ft2rXLtHgKQ2UAAAAHhon/xcfHKzAw0G6Lj48vcM709HTl5eUpNDTUrj00NFRpaWkuvV4qAwAAuFBcXFyBhfscqwJ/57hej2EYLl/Dh2QAAAAH+SY+WljYkEBhgoOD5eXlVaAKcPLkyQLVArMxTAAAgAMzhwmKy8fHR9HR0UpISLBrT0hIUPv27c2+RDtUBgAAKCNiY2PVv39/tW3bVjExMZo3b56Sk5M1bNgwSZeGHFJSUrRo0SLbMfv375cknT9/XqdOndL+/fvl4+OjJk2aFPu8JAMAADgwc5igJPr166eMjAxNmjRJqampatasmTZs2KCIiAhJlxYZclxzoHXr1rY/JyYmasmSJYqIiNCvv/5a7PN6GGVkzcWLe1e4OwSgzKl08yh3hwCUSbnZKS7tv1FIO9P6Onhyn2l9uQpzBgAAsDiGCQAAcOCuYQJ3IRkAAMBBSZ4CuBaQDAAA4MBqlQHmDAAAYHFUBgAAcMAwAQAAFmcY+e4OoVQxTAAAgMVRGQAAwEE+wwQAAFhbGVmct9QwTAAAgMVRGQAAwAHDBAAAWBzDBAAAwFKoDAAA4MBqyxGTDAAA4IAVCAEAsDjmDAAAAEuhMgAAgAMeLQQAwOIYJgAAAJZCZQAAAAc8WggAgMUxTAAAACyFygAAAA54mgAAAItjmAAAAFgKlQEAABzwNAEAABbHi4oAALA4q1UGmDMAAIDFURkAAMCB1Z4mIBkAAMCB1eYMMEwAAIDFURkAAMABwwQAAFic1ZIBhgkAALA4KgMAADiwVl1A8jCsVgvBZWVlZSk+Pl5xcXHy9fV1dzhAmcB9gWsdyQDsnDt3ToGBgTp79qyqVKni7nCAMoH7Atc65gwAAGBxJAMAAFgcyQAAABZHMgA7vr6+Gj9+PJOkgL/hvsC1jgmEAABYHJUBAAAsjmQAAACLIxkAAMDiSAYswMPDQx9++KG7wwDKHO4N4BKSARfp0qWLRo0a5e4wJEmpqanq0aOHu8Mwxffff697771XkZGR8vDw0IwZM9wdEkqIe8M15s+fr44dOyooKEhBQUHq1q2b9u7d6+6wUE6QDFhAzZo1r5lHoi5cuKB69epp8uTJqlmzprvDQTl3Ld0b27Zt04MPPqitW7dq9+7dqlu3rrp3766UlBR3h4bywIDpHn30UUOXXnpl23755Rdj27ZtRrt27QwfHx+jZs2axgsvvGDk5OQYhmEY7777rlGtWjXj4sWLdn3dc889Rv/+/Q3DMIzx48cbLVu2NP7zn/8Y4eHhRkBAgDFs2DAjNzfXeO2114zQ0FCjRo0axr/+9S+7PiQZa9assZ0nICDAOHz4sO37ESNGGNddd51x/vz5K17b8ePHjTvuuMPw8/MzIiMjjcWLFxsRERHG9OnTbfucPn3aGDJkiBESEmL4+voaTZs2NdavX28YhmEsXLjQCAwMND755BOjUaNGRkBAgHHbbbcZx48fL/Hfs+N5UfZxb5TOvWEYhpGbm2tUrlzZePfdd506HtZCMuACZ86cMWJiYowhQ4YYqampRmpqqnHs2DGjYsWKxlNPPWUcOHDAWLNmjREcHGyMHz/eMAzDuHDhghEYGGgsX77c1s+pU6cMHx8f47PPPjMM49IPvEqVKhl9+/Y1vv/+e2PdunWGj4+PcdtttxkjR440Dh48aCxYsMCQZOzevdvWz99/4BmGYdx3331Gu3btjJycHGPjxo2Gt7e3sXfv3mJdW7du3YxWrVoZe/bsMRITE43OnTsb/v7+th94eXl5xk033WQ0bdrU2Lx5s/HTTz8Z69evNzZs2GAYxqUfeN7e3ka3bt2Mffv2GYmJiUbjxo2Nhx56qMR/zyQD5Q/3RuncG4ZhGOfOnTP8/PxsyQZwOSQDLtK5c2fjmWeesX1+6aWXjIYNGxr5+fm2trfeesuoVKmSkZeXZxiGYTz55JNGjx49bN/PmDHDqFevnu2Y8ePHGxUrVjTOnTtn2+e2224zIiMjbX0YhmE0bNjQiI+Pt312/IGXmZlp1KlTx3jyySeN0NDQAr8tFeXAgQOGJGPfvn22th9//NGQZPuBt2nTJsPT09M4dOhQoX0sXLjQkGQcOXLE7u8hNDS0WDH8HclA+cS94fp7wzAM46mnnjLq169v/Pnnn04dD2upUJpDElZ24MABxcTEyMPDw9bWoUMHnT9/XseOHVPdunU1ZMgQtWvXTikpKapdu7YWLlyoxx57zO6YyMhIVa5c2fY5NDRUXl5e8vT0tGs7efJkkbEEBQXpP//5j2677Ta1b99eL774YrGu4dChQ6pQoYLatGlja2vQoIGCgoJsn/fv3686dero+uuvL7KfihUrqn79+rbPYWFhl40X1zbujf8x696YMmWKli5dqm3btsnPz6/Ex8N6mEBYSgzDsPvB9VebJFt769at1bJlSy1atEhfffWVvv32Wz322GN2x3h7e9t99vDwKLQtPz//svHs2LFDXl5eOn78uP74449iX8OV2v39/a/YT2HxFtU3rn3cG/9jxr0xdepUvfrqq9q8ebNatGhRomNhXSQDLuLj46O8vDzb5yZNmmjXrl12N/auXbtUuXJl1a5d29Y2ePBgLVy4UAsWLFC3bt0UHh5uemy7du3SlClTtH79elWpUkUjR44s1nGNGjVSbm6ukpKSbG1HjhzRmTNnbJ9btGihY8eO6fDhw2aHjWsE94br7o3XX39dL7/8sj755BO1bdvWZefBtYdkwEUiIyP15Zdf6tdff1V6erqeeuopHT16VCNHjtTBgwe1du1ajR8/XrGxsXZlzIcfflgpKSmaP3++Bg4caHpcv//+u/r376+RI0eqR48eWrJkiZYvX64VK1Zc8dhGjRqpW7duGjp0qPbu3aukpCQNHTpU/v7+tt/gOnfurE6dOunee+9VQkKCfvnlF23cuFGffPKJKfFnZ2dr//792r9/v7Kzs5WSkqL9+/fryJEjpvQP1+PecM29MWXKFI0dO1YLFixQZGSk0tLSlJaWpvPnz5vSP65tJAMu8vzzz8vLy0tNmjRRjRo1lJOTow0bNmjv3r1q2bKlhg0bpkGDBmns2LF2x1WpUkX33nuvKlWqpN69e5se1zPPPKOAgAC9+uqrkqSmTZvqtdde07Bhw4r1PPKiRYsUGhqqTp06qU+fPhoyZIgqV65sNy65atUqtWvXTg8++KCaNGmi0aNH2/0meDWOHz+u1q1bq3Xr1kpNTdXUqVPVunVrDR482JT+4XrcG665N2bNmqXs7Gz17dtXYWFhtm3q1Kmm9I9rG68wLoNuvfVWNW7cWG+++aa7Q7miY8eOKTw8XJ9++qm6du3q7nBwjePeAFyDZKAMyczM1ObNm/Xwww/rhx9+UMOGDd0dUgGfffaZzp8/r+bNmys1NVWjR49WSkqKDh8+XGDyE2AW7g3AtXi0sAxp06aNTp8+rddee80tP+x27tx52XXaz58/r5ycHL300kv6+eefVblyZbVv316LFy827YddpUqVivxu48aN6tixoynnQfnCvcG9AdeiMgCbP//887Jjow0aNHB5DJebCFi7du1iPZ4FmI17A9c6kgEAACyOpwkAALA4kgEAACyOZAAAAIsjGQAAwOJIBgAAsDiSAQAALI5kAAAAiyMZAADA4v4/YDgYj30wX4gAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sns.heatmap(data, annot=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The End." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From f74bf20c984805e919a70608dd3c110cde3fe78f Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Tue, 22 Aug 2023 17:34:28 +0200 Subject: [PATCH 04/14] Removed CLI reference from docs --- docs/cli_references.md | 9 --------- mkdocs.yml | 1 - 2 files changed, 10 deletions(-) delete mode 100644 docs/cli_references.md diff --git a/docs/cli_references.md b/docs/cli_references.md deleted file mode 100644 index b65bb2fba..000000000 --- a/docs/cli_references.md +++ /dev/null @@ -1,9 +0,0 @@ -# CLI Reference - -This page provides documentation for our command line tools. - -::: mkdocs-click - :module: graphium.cli - :command: main_cli - :style: table - :prog_name: graphium diff --git a/mkdocs.yml b/mkdocs.yml index e0759cebe..e1ae2f7e1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,7 +42,6 @@ nav: - Pretrained Models: pretrained_models.md - Contribute: contribute.md - License: license.md - - CLI: cli_references.md theme: name: material From 86d451911927f51420ef21ea4b55160170477ee0 Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Tue, 22 Aug 2023 17:44:56 +0200 Subject: [PATCH 05/14] Removed the click dependency --- env.yml | 1 - mkdocs.yml | 1 - pyproject.toml | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/env.yml b/env.yml index dade0638b..fa4e89136 100644 --- a/env.yml +++ b/env.yml @@ -66,7 +66,6 @@ dependencies: - mkdocstrings - mkdocstrings-python - mkdocs-jupyter - - mkdocs-click - markdown-include - mike >=1.0.0 diff --git a/mkdocs.yml b/mkdocs.yml index e1ae2f7e1..ce2715b61 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,7 +71,6 @@ markdown_extensions: - pymdownx.tabbed - pymdownx.tasklist - pymdownx.details - - mkdocs-click - pymdownx.arithmatex: generic: true - toc: diff --git a/pyproject.toml b/pyproject.toml index 102303072..1e1ccc8b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "click", + "typer", "loguru", "omegaconf >=2.0.0", "tqdm", From e39e19917c8481d49c4edfbdd0955a0f4934bb70 Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Wed, 23 Aug 2023 09:11:43 +0200 Subject: [PATCH 06/14] Fixed typing bug in py38 --- graphium/cli/finetune_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphium/cli/finetune_utils.py b/graphium/cli/finetune_utils.py index 1bf993ea1..1e9f575dc 100644 --- a/graphium/cli/finetune_utils.py +++ b/graphium/cli/finetune_utils.py @@ -17,7 +17,7 @@ @finetune_app.command(name="admet") def benchmark_tdc_admet_cli( - overrides: list[str], + overrides: List[str], name: Optional[List[str]] = None, inclusive_filter: bool = True, ): From ec695347027af3986a6261166a52fbacbb6a425d Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Wed, 23 Aug 2023 09:39:14 +0200 Subject: [PATCH 07/14] Replaced click by typer in IPU requirements --- requirements_ipu.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements_ipu.txt b/requirements_ipu.txt index 59981ff38..813471ea4 100644 --- a/requirements_ipu.txt +++ b/requirements_ipu.txt @@ -1,7 +1,7 @@ --find-links https://data.pyg.org/whl/torch-2.0.1+cpu.html pip -click +typer loguru tqdm numpy @@ -34,7 +34,6 @@ mkdocs-material-extensions mkdocstrings mkdocstrings-python mkdocs-jupyter -mkdocs-click markdown-include rever >==0.4.5 omegaconf From 351e309f9b7e9341b445a4b300414804752ff401 Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Wed, 23 Aug 2023 16:44:45 +0200 Subject: [PATCH 08/14] Added `LargeMix` baselines to the website --- README.md | 2 +- docs/baseline.md | 38 ++++++++++++++++++++++++++++++-- graphium/cli/data.py | 30 ++++++++++++++++++++++++++ graphium/cli/fingerprints.py | 6 ++++++ graphium/cli/prepare_data.py | 42 ------------------------------------ pyproject.toml | 3 +-- 6 files changed, 74 insertions(+), 47 deletions(-) create mode 100644 graphium/cli/fingerprints.py delete mode 100644 graphium/cli/prepare_data.py diff --git a/README.md b/README.md index cba6691fe..11b707bba 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ However, when working with larger datasets, it is recommended to perform data pr The following command-line will prepare the data and cache it, then use it to train a model. ```bash # First prepare the data and cache it in `path_to_cached_data` -graphium-prepare-data datamodule.args.processed_graph_data_path=[path_to_cached_data] +graphium data prepare ++datamodule.args.processed_graph_data_path=[path_to_cached_data] # Then train the model on the prepared data graphium-train [...] datamodule.args.processed_graph_data_path=[path_to_cached_data] diff --git a/docs/baseline.md b/docs/baseline.md index 73039a197..c843671e8 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -1,6 +1,6 @@ # ToyMix Baseline -From the paper to be released soon. Below, you can see the baselines for the `ToyMix` dataset, a multitasking dataset comprising of `QM9`, `Zinc12k` and `Tox21`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). +From the paper to be released soon. Below, you can see the baselines for the `ToyMix` dataset, a multitasking dataset comprising of `QM9`, `Zinc12k` and `Tox21`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with ~150k parameters. One can observe that the smaller datasets (`Zinc12k` and `Tox21`) beneficiate from adding another unrelated task (`QM9`), where the labels are computed from DFT simulations. @@ -25,7 +25,41 @@ One can observe that the smaller datasets (`Zinc12k` and `Tox21`) beneficiate fr | | GINE | 0.201 ± 0.007 | 0.783 ± 0.007 | 0.345 ± 0.02 | 0.177 ± 0.0008 | 0.836 ± 0.004 | **0.455 ± 0.008** | # LargeMix Baseline -Coming soon! + +From the paper to be released soon. Below, you can see the baselines for the `LargeMix` dataset, a multitasking dataset comprising of `PCQM4M_N4`, `PCQM4M_G25`, `PCBA_1328`, `L1000_VCAP`, and `L1000_MCF7`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with 4-6M parameters. + +One can observe that the smaller datasets (`L1000_VCAP` and `L1000_MCF7`) beneficiate tremendously from the multitasking. Indeed, the lack of molecular samples means that it is very easy for a model to overfit. + +While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4` and assay predictions of `PCBA_1328` take a hit, but it is most likely due to underfitting since the training loss is also increased. It seems that 4-6M parameters is far from sufficient to capturing all of the tasks simultaneously, which motivates the need for a larger model. + +| Dataset | Model | MAE ↓ | Pearson ↑ | R² ↑ | MAE ↓ | Pearson ↑ | R² ↑ | +|-----------|-------|-----------|-----------|-----------|---------|-----------|---------| +| | Single-Task Model Multi-Task Model | +| | | | | | | | | +| Pcqm4m_g25 | GCN | 0.2362 ± 0.0003 | 0.8781 ± 0.0005 | 0.7803 ± 0.0006 | 0.2458 ± 0.0007 | 0.8701 ± 0.0002 | **0.8189 ± 0.0726** | +| | GIN | 0.2270 ± 0.0003 | 0.8854 ± 0.0004 | 0.7912 ± 0.0006 | 0.2352 ± 0.0006 | 0.8802 ± 0.0007 | 0.7827 ± 0.0005 | +| | GINE| **0.2223 ± 0.0007** | **0.8874 ± 0.0003** | 0.7949 ± 0.0001 | 0.2315 ± 0.0002 | 0.8823 ± 0.0002 | 0.7864 ± 0.0008 | +| Pcqm4m_n4 | GCN | 0.2080 ± 0.0003 | 0.5497 ± 0.0010 | 0.2942 ± 0.0007 | 0.2040 ± 0.0001 | 0.4796 ± 0.0006 | 0.2185 ± 0.0002 | +| | GIN | 0.1912 ± 0.0027 | **0.6138 ± 0.0088** | **0.3688 ± 0.0116** | 0.1966 ± 0.0003 | 0.5198 ± 0.0008 | 0.2602 ± 0.0012 | +| | GINE| **0.1910 ± 0.0001** | 0.6127 ± 0.0003 | 0.3666 ± 0.0008 | 0.1941 ± 0.0003 | 0.5303 ± 0.0023 | 0.2701 ± 0.0034 | +| | | | CE ↓ | AUROC ↑ | AP ↑ | CE ↓ | AUROC ↑ | AP ↑ | +| | | | | | | | | | + + +| | | BCE ↓ | AUROC ↑ | AP ↑ | BCE ↓ | AUROC ↑ | AP ↑ | +|-----------|-------|-----------|-----------|-----------|---------|-----------|---------| +| | Single-Task Model Multi-Task Model | +| | | | | | | | | +| Pcba\_1328 | GCN | **0.0316 ± 0.0000** | **0.7960 ± 0.0020** | **0.3368 ± 0.0027** | 0.0349 ± 0.0002 | 0.7661 ± 0.0031 | 0.2527 ± 0.0041 | +| | GIN | 0.0324 ± 0.0000 | 0.7941 ± 0.0018 | 0.3328 ± 0.0019 | 0.0342 ± 0.0001 | 0.7747 ± 0.0025 | 0.2650 ± 0.0020 | +| | GINE | 0.0320 ± 0.0001 | 0.7944 ± 0.0023 | 0.3337 ± 0.0027 | 0.0341 ± 0.0001 | 0.7737 ± 0.0007 | 0.2611 ± 0.0043 | +| L1000\_vcap | GCN | 0.1900 ± 0.0002 | 0.5788 ± 0.0034 | 0.3708 ± 0.0007 | 0.1872 ± 0.0020 | 0.6362 ± 0.0012 | 0.4022 ± 0.0008 | +| | GIN | 0.1909 ± 0.0005 | 0.5734 ± 0.0029 | 0.3731 ± 0.0014 | 0.1870 ± 0.0010 | 0.6351 ± 0.0014 | 0.4062 ± 0.0001 | +| | GINE | 0.1907 ± 0.0006 | 0.5708 ± 0.0079 | 0.3705 ± 0.0015 | **0.1862 ± 0.0007** | **0.6398 ± 0.0043** | **0.4068 ± 0.0023** | +| L1000\_mcf7 | GCN | 0.1869 ± 0.0003 | 0.6123 ± 0.0051 | 0.3866 ± 0.0010 | 0.1863 ± 0.0011 | **0.6401 ± 0.0021** | 0.4194 ± 0.0004 | +| | GIN | 0.1862 ± 0.0003 | 0.6202 ± 0.0091 | 0.3876 ± 0.0017 | 0.1874 ± 0.0013 | 0.6367 ± 0.0066 | **0.4198 ± 0.0036** | +| | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 | + # UltraLarge Baseline Coming soon! diff --git a/graphium/cli/data.py b/graphium/cli/data.py index ccfcaf427..6884247cf 100644 --- a/graphium/cli/data.py +++ b/graphium/cli/data.py @@ -1,9 +1,14 @@ +import timeit +from typing import List +from omegaconf import OmegaConf import typer import graphium from loguru import logger +from hydra import initialize, compose from .main import app +from graphium.config._loader import load_datamodule data_app = typer.Typer(help="Graphium datasets.") @@ -29,3 +34,28 @@ def download(name: str, output: str, progress: bool = True): def list(): logger.info("Graphium datasets:") logger.info(graphium.data.utils.list_graphium_datasets()) + + +@data_app.command(name="prepare", help="Prepare a Graphium dataset.") +def prepare_data(overrides: List[str]) -> None: + with initialize(version_base=None, config_path="../../expts/hydra-configs"): + cfg = compose( + config_name="main", + overrides=overrides, + ) + cfg = OmegaConf.to_container(cfg, resolve=True) + st = timeit.default_timer() + + # Checking that `processed_graph_data_path` is provided + path = cfg["datamodule"]["args"].get("processed_graph_data_path", None) + if path is None: + raise ValueError( + "Please provide `datamodule.args.processed_graph_data_path` to specify the caching dir." + ) + logger.info(f"The caching dir is set to '{path}'") + + # Data-module + datamodule = load_datamodule(cfg, "cpu") + datamodule.prepare_data() + + logger.info(f"Data preparation took {timeit.default_timer() - st:.2f} seconds.") diff --git a/graphium/cli/fingerprints.py b/graphium/cli/fingerprints.py new file mode 100644 index 000000000..62b078eb9 --- /dev/null +++ b/graphium/cli/fingerprints.py @@ -0,0 +1,6 @@ +from .main import app + + +@app.command(name="fp") +def get_fingerprints_from_model(): + ... diff --git a/graphium/cli/prepare_data.py b/graphium/cli/prepare_data.py deleted file mode 100644 index 7a8c6eceb..000000000 --- a/graphium/cli/prepare_data.py +++ /dev/null @@ -1,42 +0,0 @@ -import hydra -import timeit - -from omegaconf import DictConfig, OmegaConf -from loguru import logger - -from graphium.config._loader import load_datamodule, load_accelerator - - -@hydra.main(version_base=None, config_path="../../expts/hydra-configs", config_name="main") -def cli(cfg: DictConfig) -> None: - """ - CLI endpoint for preparing the data in advance. - """ - run_prepare_data(cfg) - - -def run_prepare_data(cfg: DictConfig) -> None: - """ - The main (pre-)training and fine-tuning loop. - """ - - cfg = OmegaConf.to_container(cfg, resolve=True) - st = timeit.default_timer() - - # Checking that `processed_graph_data_path` is provided - path = cfg["datamodule"]["args"].get("processed_graph_data_path", None) - if path is None: - raise ValueError( - "Please provide `datamodule.args.processed_graph_data_path` to specify the caching dir." - ) - logger.info(f"The caching dir is set to '{path}'") - - # Data-module - datamodule = load_datamodule(cfg, "cpu") - datamodule.prepare_data() - - logger.info(f"Data preparation took {timeit.default_timer() - st:.2f} seconds.") - - -if __name__ == "__main__": - cli() diff --git a/pyproject.toml b/pyproject.toml index 1e1ccc8b8..f24f61c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,7 @@ dependencies = [ [project.scripts] graphium = "graphium.cli.main:app" - graphium-train = "graphium.cli.train_finetune:cli" - graphium-prepare-data = "graphium.cli.prepare_data:cli" +graphium-train = "graphium.cli.train_finetune:cli" [project.urls] Website = "https://graphium.datamol.io/" From 3b79735307d8d72ddd372553c1db8fd7e64c786c Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 17:44:22 -0400 Subject: [PATCH 09/14] Added placeholder for training performance --- docs/baseline.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/baseline.md b/docs/baseline.md index c843671e8..1726d5321 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -60,6 +60,18 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | | GIN | 0.1862 ± 0.0003 | 0.6202 ± 0.0091 | 0.3876 ± 0.0017 | 0.1874 ± 0.0013 | 0.6367 ± 0.0066 | **0.4198 ± 0.0036** | | | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 | +| | | Training loss (CE or MSE) in single-task ↓ | Training loss (CE or MSE) in multi-task ↓ +|-----------|-------|-----------|-----------| +| Pcba\_1328 | GCN | TODO | TODO | +| | GIN | TODO | TODO | +| | GINE | TODO | TODO | +| L1000\_vcap | GCN | TODO | TODO | +| | GIN | TODO | TODO | +| | GINE | TODO | TODO | +| L1000\_mcf7 | GCN | TODO | TODO | +| | GIN | TODO | TODO | +| | GINE | TODO | TODO | + # UltraLarge Baseline Coming soon! From 531d8968c95ff0b7bceec64c9f29dee9780b208c Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 17:59:23 -0400 Subject: [PATCH 10/14] Added some baseline data --- docs/baseline.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/baseline.md b/docs/baseline.md index 1726d5321..724102cad 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -62,15 +62,21 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | | | Training loss (CE or MSE) in single-task ↓ | Training loss (CE or MSE) in multi-task ↓ |-----------|-------|-----------|-----------| -| Pcba\_1328 | GCN | TODO | TODO | -| | GIN | TODO | TODO | -| | GINE | TODO | TODO | -| L1000\_vcap | GCN | TODO | TODO | -| | GIN | TODO | TODO | -| | GINE | TODO | TODO | -| L1000\_mcf7 | GCN | TODO | TODO | -| | GIN | TODO | TODO | -| | GINE | TODO | TODO | +| Pcqm4m\_g25 | GCN | TO±DO | TO±DO | +| | GIN | TO±DO | TO±DO | +| | GINE | TO±DO | TO±DO | +| Pcqm4m\_g4 | GCN | TO±DO | TO±DO | +| | GIN | TO±DO | TO±DO | +| | GINE | TO±DO | TO±DO | +| Pcba\_1328 | GCN | .02836±.0010 | TO±DO | +| | GIN | .02487±.0017 | TO±DO | +| | GINE | .02581±.00017 | TO±DO | +| L1000\_vcap | GCN | TO±DO | TO±DO | +| | GIN | TO±DO | TO±DO | +| | GINE | TO±DO | TO±DO | +| L1000\_mcf7 | GCN | TO±DO | TO±DO | +| | GIN | TO±DO | TO±DO | +| | GINE | TO±DO | TO±DO | # UltraLarge Baseline From e6e79a121b412c1d40bce4e25ca77c690def701a Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 22:18:09 -0400 Subject: [PATCH 11/14] Added the training set baselines for `LargeMix` --- docs/baseline.md | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/docs/baseline.md b/docs/baseline.md index 724102cad..16eb608d6 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -42,8 +42,6 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | Pcqm4m_n4 | GCN | 0.2080 ± 0.0003 | 0.5497 ± 0.0010 | 0.2942 ± 0.0007 | 0.2040 ± 0.0001 | 0.4796 ± 0.0006 | 0.2185 ± 0.0002 | | | GIN | 0.1912 ± 0.0027 | **0.6138 ± 0.0088** | **0.3688 ± 0.0116** | 0.1966 ± 0.0003 | 0.5198 ± 0.0008 | 0.2602 ± 0.0012 | | | GINE| **0.1910 ± 0.0001** | 0.6127 ± 0.0003 | 0.3666 ± 0.0008 | 0.1941 ± 0.0003 | 0.5303 ± 0.0023 | 0.2701 ± 0.0034 | -| | | | CE ↓ | AUROC ↑ | AP ↑ | CE ↓ | AUROC ↑ | AP ↑ | -| | | | | | | | | | | | | BCE ↓ | AUROC ↑ | AP ↑ | BCE ↓ | AUROC ↑ | AP ↑ | @@ -60,24 +58,31 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | | GIN | 0.1862 ± 0.0003 | 0.6202 ± 0.0091 | 0.3876 ± 0.0017 | 0.1874 ± 0.0013 | 0.6367 ± 0.0066 | **0.4198 ± 0.0036** | | | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 | -| | | Training loss (CE or MSE) in single-task ↓ | Training loss (CE or MSE) in multi-task ↓ -|-----------|-------|-----------|-----------| -| Pcqm4m\_g25 | GCN | TO±DO | TO±DO | -| | GIN | TO±DO | TO±DO | -| | GINE | TO±DO | TO±DO | -| Pcqm4m\_g4 | GCN | TO±DO | TO±DO | -| | GIN | TO±DO | TO±DO | -| | GINE | TO±DO | TO±DO | -| Pcba\_1328 | GCN | .02836±.0010 | TO±DO | -| | GIN | .02487±.0017 | TO±DO | -| | GINE | .02581±.00017 | TO±DO | -| L1000\_vcap | GCN | TO±DO | TO±DO | -| | GIN | TO±DO | TO±DO | -| | GINE | TO±DO | TO±DO | -| L1000\_mcf7 | GCN | TO±DO | TO±DO | -| | GIN | TO±DO | TO±DO | -| | GINE | TO±DO | TO±DO | +Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets. This suggests that the gap on the test set is likely to shrink with increasing the number of parameters. + +| | | CE or BCE loss in single-task $\downarrow$ | CE or BCE loss in multi-task $\downarrow$ | +|------------|-------|-----------------------------------------|-----------------------------------------| +| | | | | +| **Pcqm4m\_g25** | GCN | **0.2660 ± 0.0005** | 0.2767 ± 0.0015 | +| | GIN | **0.2439 ± 0.0004** | 0.2595 ± 0.0016 | +| | GINE | **0.2424 ± 0.0007** | 0.2568 ± 0.0012 | +| | | | | +| **Pcqm4m\_n4** | GCN | **0.2515 ± 0.0002** | 0.2613 ± 0.0008 | +| | GIN | **0.2317 ± 0.0003** | 0.2512 ± 0.0008 | +| | GINE | **0.2272 ± 0.0001** | 0.2483 ± 0.0004 | +| | | | | +| **Pcba\_1328** | GCN | **0.0284 ± 0.0010** | 0.0382 ± 0.0005 | +| | GIN | **0.0249 ± 0.0017** | 0.0359 ± 0.0011 | +| | GINE | **0.0258 ± 0.0017** | 0.0361 ± 0.0008 | +| | | | | +| **L1000\_vcap** | GCN | 0.1906 ± 0.0036 | **0.1854 ± 0.0148** | +| | GIN | 0.1854 ± 0.0030 | **0.1833 ± 0.0185** | +| | GINE | **0.1860 ± 0.0025** | 0.1887 ± 0.0200 | +| | | | | +| **L1000\_mcf7** | GCN | 0.1902 ± 0.0038 | **0.1829 ± 0.0095** | +| | GIN | 0.1873 ± 0.0033 | **0.1701 ± 0.0142** | +| | GINE | 0.1883 ± 0.0039 | **0.1771 ± 0.0010** | # UltraLarge Baseline Coming soon! From 2cba5a6eed56af292bea860feeda477aea970e75 Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 23:26:29 -0400 Subject: [PATCH 12/14] Minor --- docs/baseline.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/baseline.md b/docs/baseline.md index 16eb608d6..3792948b0 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -59,7 +59,9 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 | -Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets. This suggests that the gap on the test set is likely to shrink with increasing the number of parameters. +Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets. + +This is not surprising as they contain two orders of magnitude more datapoints and pose a significant challenge for the relatively small models used in this analysis. This favors the \emph{Single dataset} setup (which uses a model of the same size) and we conjecture larger models to bridge this gap moving forward. | | | CE or BCE loss in single-task $\downarrow$ | CE or BCE loss in multi-task $\downarrow$ | |------------|-------|-----------------------------------------|-----------------------------------------| From 6289de553f4c8df48be642c6b8721817260aa18b Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 23:37:24 -0400 Subject: [PATCH 13/14] Minor --- docs/baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/baseline.md b/docs/baseline.md index 3792948b0..2c69a27eb 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -61,7 +61,7 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets. -This is not surprising as they contain two orders of magnitude more datapoints and pose a significant challenge for the relatively small models used in this analysis. This favors the \emph{Single dataset} setup (which uses a model of the same size) and we conjecture larger models to bridge this gap moving forward. +This is not surprising as they contain two orders of magnitude more datapoints and pose a significant challenge for the relatively small models used in this analysis. This favors the Single dataset setup (which uses a model of the same size) and we conjecture larger models to bridge this gap moving forward. | | | CE or BCE loss in single-task $\downarrow$ | CE or BCE loss in multi-task $\downarrow$ | |------------|-------|-----------------------------------------|-----------------------------------------| From d1e8cb9abe3edd1fd983c92c4eff58476426478c Mon Sep 17 00:00:00 2001 From: DomInvivo Date: Fri, 25 Aug 2023 23:40:01 -0400 Subject: [PATCH 14/14] Minor --- docs/baseline.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/baseline.md b/docs/baseline.md index 2c69a27eb..d9ff12bc3 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -1,4 +1,4 @@ -# ToyMix Baseline +# ToyMix Baseline - Test set metrics From the paper to be released soon. Below, you can see the baselines for the `ToyMix` dataset, a multitasking dataset comprising of `QM9`, `Zinc12k` and `Tox21`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with ~150k parameters. @@ -25,6 +25,7 @@ One can observe that the smaller datasets (`Zinc12k` and `Tox21`) beneficiate fr | | GINE | 0.201 ± 0.007 | 0.783 ± 0.007 | 0.345 ± 0.02 | 0.177 ± 0.0008 | 0.836 ± 0.004 | **0.455 ± 0.008** | # LargeMix Baseline +## LargeMix test set metrics From the paper to be released soon. Below, you can see the baselines for the `LargeMix` dataset, a multitasking dataset comprising of `PCQM4M_N4`, `PCQM4M_G25`, `PCBA_1328`, `L1000_VCAP`, and `L1000_MCF7`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with 4-6M parameters. @@ -58,6 +59,7 @@ While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4 | | GIN | 0.1862 ± 0.0003 | 0.6202 ± 0.0091 | 0.3876 ± 0.0017 | 0.1874 ± 0.0013 | 0.6367 ± 0.0066 | **0.4198 ± 0.0036** | | | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 | +## LargeMix training set loss Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets.