diff --git a/.gitignore b/.gitignore
index 77cd466fc..e2d3cd745 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,8 @@
*.datacache.gz
lightning_logs/
logs/
+multirun/
+hparam-search-results/
models_checkpoints/
outputs/
out/
@@ -39,7 +41,8 @@ graphium/data/neurips2023/dummy-dataset/
graphium/data/make_data_splits/*.csv*
graphium/data/make_data_splits/*.pt*
graphium/data/make_data_splits/*.parquet*
-
+*.csv.gz
+*.pt
# Others
expts_untracked/
diff --git a/README.md b/README.md
index cba6691fe..11b707bba 100644
--- a/README.md
+++ b/README.md
@@ -105,7 +105,7 @@ However, when working with larger datasets, it is recommended to perform data pr
The following command-line will prepare the data and cache it, then use it to train a model.
```bash
# First prepare the data and cache it in `path_to_cached_data`
-graphium-prepare-data datamodule.args.processed_graph_data_path=[path_to_cached_data]
+graphium data prepare ++datamodule.args.processed_graph_data_path=[path_to_cached_data]
# Then train the model on the prepared data
graphium-train [...] datamodule.args.processed_graph_data_path=[path_to_cached_data]
diff --git a/docs/baseline.md b/docs/baseline.md
index 73039a197..d9ff12bc3 100644
--- a/docs/baseline.md
+++ b/docs/baseline.md
@@ -1,6 +1,6 @@
-# ToyMix Baseline
+# ToyMix Baseline - Test set metrics
-From the paper to be released soon. Below, you can see the baselines for the `ToyMix` dataset, a multitasking dataset comprising of `QM9`, `Zinc12k` and `Tox21`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401).
+From the paper to be released soon. Below, you can see the baselines for the `ToyMix` dataset, a multitasking dataset comprising of `QM9`, `Zinc12k` and `Tox21`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with ~150k parameters.
One can observe that the smaller datasets (`Zinc12k` and `Tox21`) beneficiate from adding another unrelated task (`QM9`), where the labels are computed from DFT simulations.
@@ -25,7 +25,68 @@ One can observe that the smaller datasets (`Zinc12k` and `Tox21`) beneficiate fr
| | GINE | 0.201 ± 0.007 | 0.783 ± 0.007 | 0.345 ± 0.02 | 0.177 ± 0.0008 | 0.836 ± 0.004 | **0.455 ± 0.008** |
# LargeMix Baseline
-Coming soon!
+## LargeMix test set metrics
+
+From the paper to be released soon. Below, you can see the baselines for the `LargeMix` dataset, a multitasking dataset comprising of `PCQM4M_N4`, `PCQM4M_G25`, `PCBA_1328`, `L1000_VCAP`, and `L1000_MCF7`. The datasets and their splits are available on [this link](https://zenodo.org/record/7998401). The following baselines are all for models with 4-6M parameters.
+
+One can observe that the smaller datasets (`L1000_VCAP` and `L1000_MCF7`) beneficiate tremendously from the multitasking. Indeed, the lack of molecular samples means that it is very easy for a model to overfit.
+
+While `PCQM4M_G25` has no noticeable changes, the node predictions of `PCQM4M_N4` and assay predictions of `PCBA_1328` take a hit, but it is most likely due to underfitting since the training loss is also increased. It seems that 4-6M parameters is far from sufficient to capturing all of the tasks simultaneously, which motivates the need for a larger model.
+
+| Dataset | Model | MAE ↓ | Pearson ↑ | R² ↑ | MAE ↓ | Pearson ↑ | R² ↑ |
+|-----------|-------|-----------|-----------|-----------|---------|-----------|---------|
+| | Single-Task Model Multi-Task Model |
+| | | | | | | | |
+| Pcqm4m_g25 | GCN | 0.2362 ± 0.0003 | 0.8781 ± 0.0005 | 0.7803 ± 0.0006 | 0.2458 ± 0.0007 | 0.8701 ± 0.0002 | **0.8189 ± 0.0726** |
+| | GIN | 0.2270 ± 0.0003 | 0.8854 ± 0.0004 | 0.7912 ± 0.0006 | 0.2352 ± 0.0006 | 0.8802 ± 0.0007 | 0.7827 ± 0.0005 |
+| | GINE| **0.2223 ± 0.0007** | **0.8874 ± 0.0003** | 0.7949 ± 0.0001 | 0.2315 ± 0.0002 | 0.8823 ± 0.0002 | 0.7864 ± 0.0008 |
+| Pcqm4m_n4 | GCN | 0.2080 ± 0.0003 | 0.5497 ± 0.0010 | 0.2942 ± 0.0007 | 0.2040 ± 0.0001 | 0.4796 ± 0.0006 | 0.2185 ± 0.0002 |
+| | GIN | 0.1912 ± 0.0027 | **0.6138 ± 0.0088** | **0.3688 ± 0.0116** | 0.1966 ± 0.0003 | 0.5198 ± 0.0008 | 0.2602 ± 0.0012 |
+| | GINE| **0.1910 ± 0.0001** | 0.6127 ± 0.0003 | 0.3666 ± 0.0008 | 0.1941 ± 0.0003 | 0.5303 ± 0.0023 | 0.2701 ± 0.0034 |
+
+
+| | | BCE ↓ | AUROC ↑ | AP ↑ | BCE ↓ | AUROC ↑ | AP ↑ |
+|-----------|-------|-----------|-----------|-----------|---------|-----------|---------|
+| | Single-Task Model Multi-Task Model |
+| | | | | | | | |
+| Pcba\_1328 | GCN | **0.0316 ± 0.0000** | **0.7960 ± 0.0020** | **0.3368 ± 0.0027** | 0.0349 ± 0.0002 | 0.7661 ± 0.0031 | 0.2527 ± 0.0041 |
+| | GIN | 0.0324 ± 0.0000 | 0.7941 ± 0.0018 | 0.3328 ± 0.0019 | 0.0342 ± 0.0001 | 0.7747 ± 0.0025 | 0.2650 ± 0.0020 |
+| | GINE | 0.0320 ± 0.0001 | 0.7944 ± 0.0023 | 0.3337 ± 0.0027 | 0.0341 ± 0.0001 | 0.7737 ± 0.0007 | 0.2611 ± 0.0043 |
+| L1000\_vcap | GCN | 0.1900 ± 0.0002 | 0.5788 ± 0.0034 | 0.3708 ± 0.0007 | 0.1872 ± 0.0020 | 0.6362 ± 0.0012 | 0.4022 ± 0.0008 |
+| | GIN | 0.1909 ± 0.0005 | 0.5734 ± 0.0029 | 0.3731 ± 0.0014 | 0.1870 ± 0.0010 | 0.6351 ± 0.0014 | 0.4062 ± 0.0001 |
+| | GINE | 0.1907 ± 0.0006 | 0.5708 ± 0.0079 | 0.3705 ± 0.0015 | **0.1862 ± 0.0007** | **0.6398 ± 0.0043** | **0.4068 ± 0.0023** |
+| L1000\_mcf7 | GCN | 0.1869 ± 0.0003 | 0.6123 ± 0.0051 | 0.3866 ± 0.0010 | 0.1863 ± 0.0011 | **0.6401 ± 0.0021** | 0.4194 ± 0.0004 |
+| | GIN | 0.1862 ± 0.0003 | 0.6202 ± 0.0091 | 0.3876 ± 0.0017 | 0.1874 ± 0.0013 | 0.6367 ± 0.0066 | **0.4198 ± 0.0036** |
+| | GINE | **0.1856 ± 0.0005** | 0.6166 ± 0.0017 | 0.3892 ± 0.0035 | 0.1873 ± 0.0009 | 0.6347 ± 0.0048 | 0.4177 ± 0.0024 |
+
+## LargeMix training set loss
+
+Below is the loss on the training set. One can observe that the multi-task model always underfits the single-task, except on the two `L1000` datasets.
+
+This is not surprising as they contain two orders of magnitude more datapoints and pose a significant challenge for the relatively small models used in this analysis. This favors the Single dataset setup (which uses a model of the same size) and we conjecture larger models to bridge this gap moving forward.
+
+| | | CE or BCE loss in single-task $\downarrow$ | CE or BCE loss in multi-task $\downarrow$ |
+|------------|-------|-----------------------------------------|-----------------------------------------|
+| | | | |
+| **Pcqm4m\_g25** | GCN | **0.2660 ± 0.0005** | 0.2767 ± 0.0015 |
+| | GIN | **0.2439 ± 0.0004** | 0.2595 ± 0.0016 |
+| | GINE | **0.2424 ± 0.0007** | 0.2568 ± 0.0012 |
+| | | | |
+| **Pcqm4m\_n4** | GCN | **0.2515 ± 0.0002** | 0.2613 ± 0.0008 |
+| | GIN | **0.2317 ± 0.0003** | 0.2512 ± 0.0008 |
+| | GINE | **0.2272 ± 0.0001** | 0.2483 ± 0.0004 |
+| | | | |
+| **Pcba\_1328** | GCN | **0.0284 ± 0.0010** | 0.0382 ± 0.0005 |
+| | GIN | **0.0249 ± 0.0017** | 0.0359 ± 0.0011 |
+| | GINE | **0.0258 ± 0.0017** | 0.0361 ± 0.0008 |
+| | | | |
+| **L1000\_vcap** | GCN | 0.1906 ± 0.0036 | **0.1854 ± 0.0148** |
+| | GIN | 0.1854 ± 0.0030 | **0.1833 ± 0.0185** |
+| | GINE | **0.1860 ± 0.0025** | 0.1887 ± 0.0200 |
+| | | | |
+| **L1000\_mcf7** | GCN | 0.1902 ± 0.0038 | **0.1829 ± 0.0095** |
+| | GIN | 0.1873 ± 0.0033 | **0.1701 ± 0.0142** |
+| | GINE | 0.1883 ± 0.0039 | **0.1771 ± 0.0010** |
# UltraLarge Baseline
Coming soon!
diff --git a/docs/cli_references.md b/docs/cli_references.md
deleted file mode 100644
index b65bb2fba..000000000
--- a/docs/cli_references.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# CLI Reference
-
-This page provides documentation for our command line tools.
-
-::: mkdocs-click
- :module: graphium.cli
- :command: main_cli
- :style: table
- :prog_name: graphium
diff --git a/env.yml b/env.yml
index e49d071a4..fa4e89136 100644
--- a/env.yml
+++ b/env.yml
@@ -5,7 +5,7 @@ channels:
dependencies:
- python >=3.8
- pip
- - click
+ - typer
- loguru
- omegaconf >=2.0.0
- tqdm
@@ -66,10 +66,10 @@ dependencies:
- mkdocstrings
- mkdocstrings-python
- mkdocs-jupyter
- - mkdocs-click
- markdown-include
- mike >=1.0.0
- pip:
- lightning-graphcore # optional, for using IPUs only
- hydra-core>=1.3.2
+ - hydra-optuna-sweeper
diff --git a/expts/hydra-configs/architecture/toymix.yaml b/expts/hydra-configs/architecture/toymix.yaml
index c79325919..a62b839cd 100644
--- a/expts/hydra-configs/architecture/toymix.yaml
+++ b/expts/hydra-configs/architecture/toymix.yaml
@@ -78,7 +78,7 @@ datamodule:
featurization_n_jobs: 30
featurization_progress: True
featurization_backend: "loky"
- processed_graph_data_path: "../datacache/neurips2023-small/"
+ processed_graph_data_path: ${constants.datacache_path}
dataloading_from: ram
num_workers: 30 # -1 to use all
persistent_workers: False
diff --git a/expts/hydra-configs/finetuning/admet.yaml b/expts/hydra-configs/finetuning/admet.yaml
index 80fb20e35..7360707df 100644
--- a/expts/hydra-configs/finetuning/admet.yaml
+++ b/expts/hydra-configs/finetuning/admet.yaml
@@ -29,16 +29,16 @@ constants:
# For now, we assume a model is always fine-tuned on a single task at a time.
# You can override this value with any of the benchmark names in the TDC benchmark suite.
# See also https://tdcommons.ai/benchmark/admet_group/overview/
- task: &task lipophilicity_astrazeneca
+ task: lipophilicity_astrazeneca
name: finetuning_${constants.task}_gcn
wandb:
name: ${constants.name}
- project: *task
+ project: ${constants.task}
entity: multitask-gnn
save_dir: logs/${constants.task}
seed: 42
- max_epochs: 10
+ max_epochs: 100
data_dir: expts/data/admet/${constants.task}
raise_train_error: true
@@ -57,10 +57,10 @@ finetuning:
level: graph
# Pretrained model
- pretrained_model_name: dummy-pretrained-model
+ pretrained_model: dummy-pretrained-model
finetuning_module: task_heads # gnn
sub_module_from_pretrained: zinc # optional
- new_sub_module: lipophilicity_astrazeneca # optional
+ new_sub_module: ${constants.task} # optional
# keep_modules_after_finetuning_module: # optional
# graph_output_nn/graph: {}
diff --git a/expts/hydra-configs/hparam_search/optuna.yaml b/expts/hydra-configs/hparam_search/optuna.yaml
new file mode 100644
index 000000000..47811f3ec
--- /dev/null
+++ b/expts/hydra-configs/hparam_search/optuna.yaml
@@ -0,0 +1,54 @@
+# @package _global_
+#
+# For running a hyper-parameter search, we use the Optuna plugin for hydra.
+# This makes optuna available as a sweeper in hydra and integrates easily with the rest of the codebase.
+# For more info, see https://hydra.cc/docs/plugins/optuna_sweeper/
+#
+# To run a hyper-param search,
+# (1) Update this config, specifically the hyper-param search space;
+# (2) Run `graphium-train +hparam_search=optuna` from the command line.
+
+
+defaults:
+ - override /hydra/sweeper: optuna
+ # Optuna supports various sweepers (e.g. grid search, random search, TPE sampler)
+ - override /hydra/sweeper/sampler: tpe
+
+hyper_param_search:
+ # For the sweeper to work, the main process needs to return
+ # the objective value(s) (as a float) we are trying to optimize.
+
+ # Assuming this is a metric, the `objective` key specifies which metric.
+ # Optuna supports multi-parameter optimization as well.
+ # If configured correctly, you can specify multiple keys.
+ objective: loss/test
+
+ # Where to save results to
+ # NOTE (cwognum): Ideally, we would use the `hydra.sweep.dir` key, but they don't support remote paths.
+ # save_destination: gs://path/to/bucket
+ # overwrite_destination: false
+
+hydra:
+ # Run in multirun mode by default (i.e. actually use the sweeper)
+ mode: MULTIRUN
+
+ # Changes the working directory
+ sweep:
+ dir: hparam-search-results/${constants.name}
+ subdir: ${hydra.job.num}
+
+ # Sweeper config
+ sweeper:
+ sampler:
+ seed: ${constants.seed}
+ direction: minimize
+ study_name: ${constants.name}
+ storage: null
+ n_trials: 100
+ n_jobs: 1
+
+ # The hyper-parameter search space definition
+ # See https://hydra.cc/docs/plugins/optuna_sweeper/#search-space-configuration for the options
+ params:
+ predictor.optim_kwargs.lr: tag(log, interval(0.00001, 0.001))
+
diff --git a/expts/hydra-configs/training/model/toymix_gcn.yaml b/expts/hydra-configs/training/model/toymix_gcn.yaml
index 48eabe003..3c7a13d05 100644
--- a/expts/hydra-configs/training/model/toymix_gcn.yaml
+++ b/expts/hydra-configs/training/model/toymix_gcn.yaml
@@ -6,6 +6,7 @@ constants:
max_epochs: 100
data_dir: expts/data/neurips2023/small-dataset
raise_train_error: true
+ datacache_path: ../datacache/neurips2023-small/
trainer:
model_checkpoint:
diff --git a/expts/hydra-configs/training/model/toymix_gin.yaml b/expts/hydra-configs/training/model/toymix_gin.yaml
index ed2885efb..459694c9a 100644
--- a/expts/hydra-configs/training/model/toymix_gin.yaml
+++ b/expts/hydra-configs/training/model/toymix_gin.yaml
@@ -3,8 +3,10 @@
constants:
name: neurips2023_small_data_gin
seed: 42
+ max_epochs: 100
data_dir: expts/data/neurips2023/small-dataset
raise_train_error: true
+ datacache_path: ../datacache/neurips2023-small/
trainer:
model_checkpoint:
diff --git a/expts/main_run_multitask.py b/expts/main_run_multitask.py
index c68663a08..d854c2c3e 100644
--- a/expts/main_run_multitask.py
+++ b/expts/main_run_multitask.py
@@ -1,33 +1,5 @@
-# General imports
-import os
-from os.path import dirname, abspath
-from omegaconf import DictConfig, OmegaConf
-import timeit
-from loguru import logger
-from datetime import datetime
-from lightning.pytorch.utilities.model_summary import ModelSummary
-
-# Current project imports
-import graphium
-from graphium.config._loader import (
- load_datamodule,
- load_metrics,
- load_architecture,
- load_predictor,
- load_trainer,
- save_params_to_wandb,
- load_accelerator,
-)
-from graphium.utils.safe_run import SafeRun
-
import hydra
-
-# WandB
-import wandb
-
-# Set up the working directory
-MAIN_DIR = dirname(dirname(abspath(graphium.__file__)))
-os.chdir(MAIN_DIR)
+from omegaconf import DictConfig
@hydra.main(version_base=None, config_path="hydra-configs", config_name="main")
diff --git a/graphium/cli/__init__.py b/graphium/cli/__init__.py
index 8928b9836..0bea27c69 100644
--- a/graphium/cli/__init__.py
+++ b/graphium/cli/__init__.py
@@ -1,3 +1,3 @@
-from .data import data_cli
-from .finetune_utils import finetune_cli
-from .main import main_cli
+from .data import data_app
+from .finetune_utils import finetune_app
+from .main import app
diff --git a/graphium/cli/__main__.py b/graphium/cli/__main__.py
index 0baa7638c..3e6e96f3a 100644
--- a/graphium/cli/__main__.py
+++ b/graphium/cli/__main__.py
@@ -1,4 +1,4 @@
-from .main import main_cli
+from .main import app
if __name__ == "__main__":
- main_cli()
+ app()
diff --git a/graphium/cli/data.py b/graphium/cli/data.py
index a6003b58d..6884247cf 100644
--- a/graphium/cli/data.py
+++ b/graphium/cli/data.py
@@ -1,41 +1,22 @@
-import click
+import timeit
+from typing import List
+from omegaconf import OmegaConf
+import typer
+import graphium
from loguru import logger
+from hydra import initialize, compose
-import graphium
+from .main import app
+from graphium.config._loader import load_datamodule
+
+
+data_app = typer.Typer(help="Graphium datasets.")
+app.add_typer(data_app, name="data")
-from .main import main_cli
-
-
-@main_cli.group(name="data", help="Graphium datasets.")
-def data_cli():
- pass
-
-
-@data_cli.command(name="download", help="Download a Graphium dataset.")
-@click.option(
- "-n",
- "--name",
- type=str,
- required=True,
- help="Name of the graphium dataset to download.",
-)
-@click.option(
- "-o",
- "--output",
- type=str,
- required=True,
- help="Where to download the Graphium dataset.",
-)
-@click.option(
- "--progress",
- type=bool,
- is_flag=True,
- default=False,
- required=False,
- help="Whether to extract the dataset if it's a zip file.",
-)
-def download(name, output, progress):
+
+@data_app.command(name="download", help="Download a Graphium dataset.")
+def download(name: str, output: str, progress: bool = True):
args = {}
args["name"] = name
args["output_path"] = output
@@ -49,7 +30,32 @@ def download(name, output, progress):
logger.info(f"Dataset available at {fpath}.")
-@data_cli.command(name="list", help="List available Graphium dataset.")
+@data_app.command(name="list", help="List available Graphium dataset.")
def list():
logger.info("Graphium datasets:")
logger.info(graphium.data.utils.list_graphium_datasets())
+
+
+@data_app.command(name="prepare", help="Prepare a Graphium dataset.")
+def prepare_data(overrides: List[str]) -> None:
+ with initialize(version_base=None, config_path="../../expts/hydra-configs"):
+ cfg = compose(
+ config_name="main",
+ overrides=overrides,
+ )
+ cfg = OmegaConf.to_container(cfg, resolve=True)
+ st = timeit.default_timer()
+
+ # Checking that `processed_graph_data_path` is provided
+ path = cfg["datamodule"]["args"].get("processed_graph_data_path", None)
+ if path is None:
+ raise ValueError(
+ "Please provide `datamodule.args.processed_graph_data_path` to specify the caching dir."
+ )
+ logger.info(f"The caching dir is set to '{path}'")
+
+ # Data-module
+ datamodule = load_datamodule(cfg, "cpu")
+ datamodule.prepare_data()
+
+ logger.info(f"Data preparation took {timeit.default_timer() - st:.2f} seconds.")
diff --git a/graphium/cli/finetune_utils.py b/graphium/cli/finetune_utils.py
index 80d437f98..1e9f575dc 100644
--- a/graphium/cli/finetune_utils.py
+++ b/graphium/cli/finetune_utils.py
@@ -1,63 +1,48 @@
-import yaml
-import click
-import fsspec
+from typing import List, Optional
-from loguru import logger
-from hydra import compose, initialize
+import fsspec
+import typer
+import yaml
from datamol.utils import fs
+from hydra import compose, initialize
+from hydra.core.hydra_config import HydraConfig
+from loguru import logger
-from .main import main_cli
+from .main import app
from .train_finetune import run_training_finetuning
-
-@main_cli.group(name="finetune", help="Utility CLI for extra fine-tuning utilities.")
-def finetune_cli():
- pass
+finetune_app = typer.Typer(help="Utility CLI for extra fine-tuning utilities.")
+app.add_typer(finetune_app, name="finetune")
-@finetune_cli.command(name="admet")
-@click.argument("save_dir")
-@click.option("--wandb/--no-wandb", default=True, help="Whether to log to Weights & Biases.")
-@click.option(
- "--name",
- "-n",
- multiple=True,
- help="One or multiple benchmarks to filter on. See also --inclusive-filter/--exclusive-filter.",
-)
-@click.option(
- "--inclusive-filter/--exclusive-filter",
- default=True,
- help="Whether to include or exclude the benchmarks specified by `--name`.",
-)
-def benchmark_tdc_admet_cli(save_dir, wandb, name, inclusive_filter):
+@finetune_app.command(name="admet")
+def benchmark_tdc_admet_cli(
+ overrides: List[str],
+ name: Optional[List[str]] = None,
+ inclusive_filter: bool = True,
+):
"""
Utility CLI to easily fine-tune a model on (a subset of) the benchmarks in the TDC ADMET group.
- The results are saved to the SAVE_DIR.
+ A major limitation is that we cannot use all features of the Hydra CLI, such as multiruns.
"""
-
try:
from tdc.utils import retrieve_benchmark_names
except ImportError:
raise ImportError("TDC needs to be installed to use this CLI. Run `pip install PyTDC`.")
# Get the benchmarks to run this for
- if name is None:
+ if len(name) == 0:
name = retrieve_benchmark_names("admet_group")
- elif not inclusive_filter:
- name = [n for n in name if n not in retrieve_benchmark_names("admet_group")]
+ if not inclusive_filter:
+ name = [n for n in retrieve_benchmark_names("admet_group") if n not in name]
+
+ logger.info(f"Running fine-tuning for the following benchmarks: {name}")
results = {}
# Use the Compose API to construct the config
for n in name:
- overrides = [
- "+finetuning=admet",
- f"finetuning.task={n}",
- f"finetuning.finetuning_head.task={n}",
- ]
-
- if not wandb:
- overrides.append("~constants.wandb")
+ overrides += ["+finetuning=admet", f"constants.task={n}"]
with initialize(version_base=None, config_path="../../expts/hydra-configs"):
cfg = compose(
@@ -70,6 +55,9 @@ def benchmark_tdc_admet_cli(save_dir, wandb, name, inclusive_filter):
ret = {k: v.item() for k, v in ret.items()}
results[n] = ret
+ # Save to the results_dir by default or to the Hydra output_dir if needed.
+ # This distinction is needed, because Hydra's output_dir cannot be remote.
+ save_dir = cfg["constants"].get("results_dir", HydraConfig.get()["runtime"]["output_dir"])
fs.mkdir(save_dir, exist_ok=True)
path = fs.join(save_dir, "results.yaml")
logger.info(f"Saving results to {path}")
diff --git a/graphium/cli/fingerprints.py b/graphium/cli/fingerprints.py
new file mode 100644
index 000000000..62b078eb9
--- /dev/null
+++ b/graphium/cli/fingerprints.py
@@ -0,0 +1,6 @@
+from .main import app
+
+
+@app.command(name="fp")
+def get_fingerprints_from_model():
+ ...
diff --git a/graphium/cli/main.py b/graphium/cli/main.py
index 2161514e7..7cce5fce8 100644
--- a/graphium/cli/main.py
+++ b/graphium/cli/main.py
@@ -1,11 +1,8 @@
-import click
+import typer
-@click.group()
-@click.version_option()
-def main_cli():
- pass
+app = typer.Typer(add_completion=False)
if __name__ == "__main__":
- main_cli()
+ app()
diff --git a/graphium/cli/prepare_data.py b/graphium/cli/prepare_data.py
deleted file mode 100644
index 7a8c6eceb..000000000
--- a/graphium/cli/prepare_data.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import hydra
-import timeit
-
-from omegaconf import DictConfig, OmegaConf
-from loguru import logger
-
-from graphium.config._loader import load_datamodule, load_accelerator
-
-
-@hydra.main(version_base=None, config_path="../../expts/hydra-configs", config_name="main")
-def cli(cfg: DictConfig) -> None:
- """
- CLI endpoint for preparing the data in advance.
- """
- run_prepare_data(cfg)
-
-
-def run_prepare_data(cfg: DictConfig) -> None:
- """
- The main (pre-)training and fine-tuning loop.
- """
-
- cfg = OmegaConf.to_container(cfg, resolve=True)
- st = timeit.default_timer()
-
- # Checking that `processed_graph_data_path` is provided
- path = cfg["datamodule"]["args"].get("processed_graph_data_path", None)
- if path is None:
- raise ValueError(
- "Please provide `datamodule.args.processed_graph_data_path` to specify the caching dir."
- )
- logger.info(f"The caching dir is set to '{path}'")
-
- # Data-module
- datamodule = load_datamodule(cfg, "cpu")
- datamodule.prepare_data()
-
- logger.info(f"Data preparation took {timeit.default_timer() - st:.2f} seconds.")
-
-
-if __name__ == "__main__":
- cli()
diff --git a/graphium/cli/train_finetune.py b/graphium/cli/train_finetune.py
index 4772d735a..bf3705a92 100644
--- a/graphium/cli/train_finetune.py
+++ b/graphium/cli/train_finetune.py
@@ -1,34 +1,47 @@
-import hydra
-import wandb
+import os
+import time
import timeit
-
-from omegaconf import DictConfig, OmegaConf
-from loguru import logger
from datetime import datetime
+
+import fsspec
+import hydra
+import torch
+import wandb
+import yaml
+from datamol.utils import fs
+from hydra.core.hydra_config import HydraConfig
+from hydra.types import RunMode
from lightning.pytorch.utilities.model_summary import ModelSummary
+from loguru import logger
+from omegaconf import DictConfig, OmegaConf
from graphium.config._loader import (
+ load_accelerator,
+ load_architecture,
load_datamodule,
load_metrics,
- load_architecture,
load_predictor,
load_trainer,
- load_accelerator,
save_params_to_wandb,
)
-from graphium.finetuning import modify_cfg_for_finetuning, GraphFinetuning
+from graphium.finetuning import (
+ FINETUNING_CONFIG_KEY,
+ GraphFinetuning,
+ modify_cfg_for_finetuning,
+)
+from graphium.hyper_param_search import (
+ HYPER_PARAM_SEARCH_CONFIG_KEY,
+ extract_main_metric_for_hparam_search,
+)
from graphium.utils.safe_run import SafeRun
-FINETUNING_CONFIG_KEY = "finetuning"
-
-
@hydra.main(version_base=None, config_path="../../expts/hydra-configs", config_name="main")
def cli(cfg: DictConfig) -> None:
"""
The main CLI endpoint for training and fine-tuning Graphium models.
"""
- run_training_finetuning(cfg)
+ return run_training_finetuning(cfg)
def run_training_finetuning(cfg: DictConfig) -> None:
@@ -38,6 +51,18 @@ def run_training_finetuning(cfg: DictConfig) -> None:
cfg = OmegaConf.to_container(cfg, resolve=True)
+ dst_dir = cfg["constants"].get("results_dir")
+ hydra_cfg = HydraConfig.get()
+ output_dir = hydra_cfg["runtime"]["output_dir"]
+
+ if dst_dir is not None and fs.exists(dst_dir) and len(fs.get_mapper(dst_dir).fs.ls(dst_dir)) > 0:
+ logger.warning(
+ "The destination directory is not empty. "
+ "If files already exist, this would lead to a crash at the end of training."
+ )
+ # We pause here briefly, to make sure the notification is seen as there's lots of logs afterwards
+ time.sleep(5)
+
# Modify the config for finetuning
if FINETUNING_CONFIG_KEY in cfg:
cfg = modify_cfg_for_finetuning(cfg)
@@ -105,6 +130,12 @@ def run_training_finetuning(cfg: DictConfig) -> None:
with SafeRun(name="TRAINING", raise_error=cfg["constants"]["raise_train_error"], verbose=True):
trainer.fit(model=predictor, datamodule=datamodule)
+ # Save validation metrics - Base utility in case someone doesn't use a logger.
+ results = trainer.callback_metrics
+ results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()}
+ with fsspec.open(fs.join(output_dir, "val_results.yaml"), "w") as f:
+ yaml.dump(results, f)
+
# Determine the max num nodes and edges in testing
predictor.set_max_nodes_edges_per_graph(datamodule, stages=["test"])
@@ -119,7 +150,29 @@ def run_training_finetuning(cfg: DictConfig) -> None:
if wandb_cfg is not None:
wandb.finish()
- return trainer.callback_metrics
+ # Save test metrics - Base utility in case someone doesn't use a logger.
+ results = trainer.callback_metrics
+ results = {k: v.item() if torch.is_tensor(v) else v for k, v in results.items()}
+ with fsspec.open(fs.join(output_dir, "test_results.yaml"), "w") as f:
+ yaml.dump(results, f)
+
+ # When part of of a hyper-parameter search, we are very specific about how we save our results
+ # NOTE (cwognum): We also check if the we are in multi-run mode, as the sweeper is otherwise not active.
+ if HYPER_PARAM_SEARCH_CONFIG_KEY in cfg and hydra_cfg.mode == RunMode.MULTIRUN:
+ results = extract_main_metric_for_hparam_search(results, cfg[HYPER_PARAM_SEARCH_CONFIG_KEY])
+
+ # Copy the current working directory to remote
+ # By default, processes should just write results to Hydra's output directory.
+ # However, this currently does not support remote storage, which is why we copy the results here if needed.
+ # For more info, see also: https://github.com/facebookresearch/hydra/issues/993
+
+ if dst_dir is not None:
+ src_dir = hydra_cfg["runtime"]["output_dir"]
+ dst_dir = fs.join(dst_dir, fs.get_basename(src_dir))
+ fs.mkdir(dst_dir, exist_ok=True)
+ fs.copy_dir(src_dir, dst_dir)
+
+ return results
if __name__ == "__main__":
diff --git a/graphium/config/_loader.py b/graphium/config/_loader.py
index d049a6f4e..7ae6d0b0a 100644
--- a/graphium/config/_loader.py
+++ b/graphium/config/_loader.py
@@ -1,36 +1,35 @@
-from typing import Dict, Mapping, Tuple, Type, Union, Any, Optional, Callable
-
# Misc
import os
-import omegaconf
from copy import deepcopy
-from loguru import logger
-import yaml
+from typing import Any, Callable, Dict, Mapping, Optional, Tuple, Type, Union
+
import joblib
-import pathlib
-import warnings
+import mup
+import omegaconf
# Torch
import torch
-import mup
+import yaml
# Lightning
from lightning import Trainer
from lightning.pytorch.callbacks import EarlyStopping, ModelCheckpoint
-from lightning.pytorch.loggers import WandbLogger, Logger
+from lightning.pytorch.loggers import Logger, WandbLogger
+from loguru import logger
-# Graphium
-from graphium.utils.mup import set_base_shapes
+from graphium.data.datamodule import BaseDataModule, MultitaskFromSmilesDataModule
+from graphium.finetuning.finetuning_architecture import FullGraphFinetuningNetwork
from graphium.ipu.ipu_dataloader import IPUDataloaderOptions
-from graphium.trainer.metrics import MetricWrapper
+from graphium.ipu.ipu_utils import import_poptorch, load_ipu_options
from graphium.nn.architectures import FullGraphMultiTaskNetwork
-from graphium.finetuning.finetuning_architecture import FullGraphFinetuningNetwork
from graphium.nn.utils import MupMixin
+from graphium.trainer.metrics import MetricWrapper
from graphium.trainer.predictor import PredictorModule
+from graphium.utils.command_line_utils import get_anchors_and_aliases, update_config
+
+# Graphium
+from graphium.utils.mup import set_base_shapes
from graphium.utils.spaces import DATAMODULE_DICT
-from graphium.ipu.ipu_utils import import_poptorch, load_ipu_options
-from graphium.data.datamodule import MultitaskFromSmilesDataModule, BaseDataModule
-from graphium.utils.command_line_utils import update_config, get_anchors_and_aliases
def get_accelerator(
@@ -264,12 +263,12 @@ def load_architecture(
if model_class is FullGraphFinetuningNetwork:
finetuning_head_kwargs = config["finetuning"].pop("finetuning_head", None)
pretrained_overwriting_kwargs = config["finetuning"].pop("overwriting_kwargs")
- pretrained_model_name = pretrained_overwriting_kwargs.pop("pretrained_model_name")
+ pretrained_model = pretrained_overwriting_kwargs.pop("pretrained_model")
model_kwargs = {
"pretrained_model_kwargs": deepcopy(model_kwargs),
"pretrained_overwriting_kwargs": pretrained_overwriting_kwargs,
- "pretrained_model_name": pretrained_model_name,
+ "pretrained_model": pretrained_model,
"finetuning_head_kwargs": finetuning_head_kwargs,
}
@@ -409,7 +408,6 @@ def load_trainer(
# Define the early model checkpoing parameters
if "model_checkpoint" in cfg_trainer.keys():
- cfg_trainer["model_checkpoint"]["dirpath"] += str(cfg_trainer["seed"]) + "/"
callbacks.append(ModelCheckpoint(**cfg_trainer["model_checkpoint"]))
# Define the logger parameters
diff --git a/graphium/config/dummy_finetuning_from_task_head.yaml b/graphium/config/dummy_finetuning_from_task_head.yaml
index 27b56a683..048fa17aa 100644
--- a/graphium/config/dummy_finetuning_from_task_head.yaml
+++ b/graphium/config/dummy_finetuning_from_task_head.yaml
@@ -32,7 +32,7 @@ finetuning:
level: graph
# Pretrained model
- pretrained_model_name: dummy-pretrained-model
+ pretrained_model: dummy-pretrained-model
finetuning_module: task_heads
sub_module_from_pretrained: zinc # optional
new_sub_module: lipophilicity_astrazeneca # optional
diff --git a/graphium/data/dataset.py b/graphium/data/dataset.py
index 039d1b35a..03737964f 100644
--- a/graphium/data/dataset.py
+++ b/graphium/data/dataset.py
@@ -1,18 +1,16 @@
-from typing import Type, List, Dict, Union, Any, Callable, Optional, Tuple, Iterable
-
-from multiprocessing import Manager
-import numpy as np
-from functools import lru_cache
-from loguru import logger
-from copy import deepcopy
import os
-import numpy as np
-
-from datamol import parallelized, parallelized_with_batches
+from copy import deepcopy
+from functools import lru_cache
+from multiprocessing import Manager
+from typing import Any, Dict, List, Optional, Tuple, Union
+import fsspec
+import numpy as np
import torch
+from datamol import parallelized, parallelized_with_batches
+from loguru import logger
from torch.utils.data.dataloader import Dataset
-from torch_geometric.data import Data, Batch
+from torch_geometric.data import Batch, Data
from graphium.data.smiles_transform import smiles_to_unique_mol_ids
from graphium.features import GraphDict
@@ -290,7 +288,8 @@ def _load_metadata(self):
"_num_edges_list",
]
path = os.path.join(self.data_path, "multitask_metadata.pkl")
- attrs = torch.load(path)
+ with fsspec.open(path, "rb") as f:
+ attrs = torch.load(path)
if not set(attrs_to_load).issubset(set(attrs.keys())):
raise ValueError(
@@ -460,7 +459,8 @@ def load_graph_from_index(self, data_idx):
filename = os.path.join(
self.data_path, format(data_idx // 1000, "04d"), format(data_idx, "07d") + ".pkl"
)
- data_dict = torch.load(filename)
+ with fsspec.open(filename, "rb") as f:
+ data_dict = torch.load(f)
return data_dict
def merge(
diff --git a/graphium/finetuning/__init__.py b/graphium/finetuning/__init__.py
index 5ef566743..0bfaf0587 100644
--- a/graphium/finetuning/__init__.py
+++ b/graphium/finetuning/__init__.py
@@ -1,3 +1,6 @@
from .utils import modify_cfg_for_finetuning
from .finetuning import GraphFinetuning
from .finetuning_architecture import FullGraphFinetuningNetwork
+
+
+FINETUNING_CONFIG_KEY = "finetuning"
diff --git a/graphium/finetuning/finetuning_architecture.py b/graphium/finetuning/finetuning_architecture.py
index 10f918621..eec2aab11 100644
--- a/graphium/finetuning/finetuning_architecture.py
+++ b/graphium/finetuning/finetuning_architecture.py
@@ -7,15 +7,15 @@
from graphium.nn.utils import MupMixin
from graphium.trainer.predictor import PredictorModule
-from graphium.utils.spaces import FINETUNING_HEADS_DICT, GRAPHIUM_PRETRAINED_MODELS_DICT
+from graphium.utils.spaces import FINETUNING_HEADS_DICT
class FullGraphFinetuningNetwork(nn.Module, MupMixin):
def __init__(
self,
- pretrained_model_name: str,
- pretrained_model_kwargs: Dict[str, Any],
- pretrained_overwriting_kwargs: Dict[str, Any],
+ pretrained_model: Union[str, "PretrainedModel"],
+ pretrained_model_kwargs: Dict[str, Any] = {},
+ pretrained_overwriting_kwargs: Dict[str, Any] = {},
finetuning_head_kwargs: Optional[Dict[str, Any]] = None,
num_inference_to_average: int = 1,
last_layer_is_readout: bool = False,
@@ -29,8 +29,8 @@ def __init__(
Parameters:
- pretrained_model_name:
- Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT
+ pretrained_model:
+ A PretrainedModel or an identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT or a valid .ckpt checkpoint path
pretrained_model_kwargs:
Key-word arguments to instantiate a model of the same class as the pretrained model (e.g., FullGraphMultitaskNetwork))
@@ -67,16 +67,17 @@ def __init__(
self.num_inference_to_average = num_inference_to_average
self.last_layer_is_readout = last_layer_is_readout
self._concat_last_layers = None
- self.pretrained_model_name = pretrained_model_name
+ self.pretrained_model = pretrained_model
self.pretrained_overwriting_kwargs = pretrained_overwriting_kwargs
self.finetuning_head_kwargs = finetuning_head_kwargs
self.max_num_nodes_per_graph = None
self.max_num_edges_per_graph = None
self.finetuning_head = None
- self.pretrained_model = PretrainedModel(
- pretrained_model_name, pretrained_model_kwargs, pretrained_overwriting_kwargs
- )
+ if not isinstance(self.pretrained_model, PretrainedModel):
+ self.pretrained_model = PretrainedModel(
+ self.pretrained_model, pretrained_model_kwargs, pretrained_overwriting_kwargs
+ )
if finetuning_head_kwargs is not None:
self.finetuning_head = FinetuningHead(finetuning_head_kwargs)
@@ -135,7 +136,7 @@ def make_mup_base_kwargs(self, divide_factor: float = 2.0) -> Dict[str, Any]:
Dictionary with the kwargs to create the base model.
"""
kwargs = dict(
- pretrained_model_name=self.pretrained_model_name,
+ pretrained_model=self.pretrained_model,
pretrained_model_kwargs=None,
finetuning_head_kwargs=None,
num_inference_to_average=self.num_inference_to_average,
@@ -174,18 +175,18 @@ def set_max_num_nodes_edges_per_graph(self, max_nodes: Optional[int], max_edges:
class PretrainedModel(nn.Module, MupMixin):
def __init__(
self,
- pretrained_model_name: str,
+ pretrained_model: str,
pretrained_model_kwargs: Dict[str, Any],
pretrained_overwriting_kwargs: Dict[str, Any],
):
r"""
- Flexible class allowing to finetune pretrained models from GRAPHIUM_PRETRAINED_MODELS_DICT.
+ Flexible class allowing to finetune pretrained models from GRAPHIUM_PRETRAINED_MODELS_DICT or from a ckeckpoint path.
Can be any model that inherits from nn.Module, MupMixin and comes with a module map (e.g., FullGraphMultitaskNetwork)
Parameters:
- pretrained_model_name:
- Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT
+ pretrained_model:
+ Identifier of pretrained model within GRAPHIUM_PRETRAINED_MODELS_DICT or from a checkpoint path
pretrained_model_kwargs:
Key-word arguments to instantiate a model of the same class as the pretrained model (e.g., FullGraphMultitaskNetwork))
@@ -198,9 +199,7 @@ def __init__(
super().__init__()
# Load pretrained model
- pretrained_model = PredictorModule.load_from_checkpoint(
- GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name]
- ).model
+ pretrained_model = PredictorModule.load_pretrained_models(pretrained_model).model
pretrained_model.create_module_map()
# Initialize new model with architecture after desired modifications to architecture.
diff --git a/graphium/finetuning/utils.py b/graphium/finetuning/utils.py
index 605ca536f..b43dba7c5 100644
--- a/graphium/finetuning/utils.py
+++ b/graphium/finetuning/utils.py
@@ -1,10 +1,9 @@
-from typing import Union, List, Dict, Any
-
from copy import deepcopy
+from typing import Any, Dict, List, Union
+
from loguru import logger
-from graphium.trainer import PredictorModule
-from graphium.utils.spaces import GRAPHIUM_PRETRAINED_MODELS_DICT
+from graphium.trainer import PredictorModule
def filter_cfg_based_on_admet_benchmark_name(config: Dict[str, Any], names: Union[List[str], str]):
@@ -56,10 +55,8 @@ def modify_cfg_for_finetuning(cfg: Dict[str, Any]):
cfg_finetune = cfg["finetuning"]
# Load pretrained model
- pretrained_model_name = cfg_finetune["pretrained_model_name"]
- pretrained_predictor = PredictorModule.load_from_checkpoint(
- GRAPHIUM_PRETRAINED_MODELS_DICT[pretrained_model_name], device="cpu"
- )
+ pretrained_model = cfg_finetune["pretrained_model"]
+ pretrained_predictor = PredictorModule.load_pretrained_models(pretrained_model, device="cpu")
# Inherit shared configuration from pretrained
# Architecture
@@ -146,7 +143,7 @@ def modify_cfg_for_finetuning(cfg: Dict[str, Any]):
pretrained_overwriting_kwargs.pop(key, None)
finetuning_training_kwargs = deepcopy(cfg["finetuning"])
- drop_keys = ["task", "level", "pretrained_model_name", "sub_module_from_pretrained", "finetuning_head"]
+ drop_keys = ["task", "level", "pretrained_model", "sub_module_from_pretrained", "finetuning_head"]
for key in drop_keys:
finetuning_training_kwargs.pop(key, None)
diff --git a/graphium/hyper_param_search/__init__.py b/graphium/hyper_param_search/__init__.py
new file mode 100644
index 000000000..f7be11aff
--- /dev/null
+++ b/graphium/hyper_param_search/__init__.py
@@ -0,0 +1,3 @@
+from .results import extract_main_metric_for_hparam_search
+
+HYPER_PARAM_SEARCH_CONFIG_KEY = "hyper_param_search"
diff --git a/graphium/hyper_param_search/results.py b/graphium/hyper_param_search/results.py
new file mode 100644
index 000000000..6aab001f3
--- /dev/null
+++ b/graphium/hyper_param_search/results.py
@@ -0,0 +1,16 @@
+_OBJECTIVE_KEY = "objective"
+
+
+def extract_main_metric_for_hparam_search(results: dict, cfg: dict):
+ """Processes the results in the context of a hyper-parameter search."""
+
+ # Extract the objectives
+ objectives = cfg[_OBJECTIVE_KEY]
+ if isinstance(objectives, str):
+ objectives = [objectives]
+
+ # Extract the objective values
+ objective_values = [results[k] for k in objectives]
+ if len(objective_values) == 1:
+ objective_values = objective_values[0]
+ return objective_values
diff --git a/graphium/trainer/predictor.py b/graphium/trainer/predictor.py
index f79fb918e..ce6e9bb90 100644
--- a/graphium/trainer/predictor.py
+++ b/graphium/trainer/predictor.py
@@ -1,25 +1,29 @@
-from graphium.trainer.metrics import MetricWrapper
-from typing import Dict, List, Any, Union, Any, Callable, Tuple, Type, Optional
-from collections import OrderedDict
-import numpy as np
-from copy import deepcopy
import time
-from loguru import logger
+from copy import deepcopy
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
-import torch
-from torch import nn, Tensor
import lightning
-from torch_geometric.data import Data, Batch
+import numpy as np
+import torch
+from loguru import logger
from mup.optim import MuAdam
+from torch import Tensor, nn
+from torch_geometric.data import Batch, Data
from graphium.config.config_convert import recursive_config_reformating
-from graphium.trainer.predictor_options import EvalOptions, FlagOptions, ModelOptions, OptimOptions
-from graphium.trainer.predictor_summaries import TaskSummaries
from graphium.data.datamodule import BaseDataModule
+from graphium.trainer.metrics import MetricWrapper
+from graphium.trainer.predictor_options import (
+ EvalOptions,
+ FlagOptions,
+ ModelOptions,
+ OptimOptions,
+)
+from graphium.trainer.predictor_summaries import TaskSummaries
+from graphium.utils import fs
from graphium.utils.moving_average_tracker import MovingAverageTracker
-from graphium.utils.tensor import dict_tensor_fp16_to_fp32
-
from graphium.utils.spaces import GRAPHIUM_PRETRAINED_MODELS_DICT
+from graphium.utils.tensor import dict_tensor_fp16_to_fp32
class PredictorModule(lightning.LightningModule):
@@ -669,20 +673,28 @@ def list_pretrained_models():
return GRAPHIUM_PRETRAINED_MODELS_DICT
@staticmethod
- def load_pretrained_models(name: str, device: str = None):
+ def load_pretrained_models(name_or_path: str, device: str = None):
"""Load a pretrained model from its name.
Args:
- name: Name of the model to load, or full path of the model.
- List available from `graphium.trainer.PredictorModule.list_pretrained_models()`.
+ name: Name of the model to load or a valid checkpoint path. List available
+ from `graphium.trainer.PredictorModule.list_pretrained_models()`.
"""
- if name in GRAPHIUM_PRETRAINED_MODELS_DICT:
+ name = GRAPHIUM_PRETRAINED_MODELS_DICT.get(name_or_path)
+
+ if name is not None:
return PredictorModule.load_from_checkpoint(
- GRAPHIUM_PRETRAINED_MODELS_DICT[name], map_location=device
+ GRAPHIUM_PRETRAINED_MODELS_DICT[name_or_path], map_location=device
)
- else:
- return PredictorModule.load_from_checkpoint(name, map_location=device)
+
+ if name is None and not (fs.exists(name_or_path) and fs.get_extension(name_or_path) == "ckpt"):
+ raise ValueError(
+ f"The model '{name_or_path}' is not available. Choose from {set(GRAPHIUM_PRETRAINED_MODELS_DICT.keys())} "
+ "or pass a valid checkpoint (.ckpt) path."
+ )
+
+ return PredictorModule.load_from_checkpoint(name_or_path, map_location=device)
def set_max_nodes_edges_per_graph(self, datamodule: BaseDataModule, stages: Optional[List[str]] = None):
datamodule.setup()
diff --git a/mkdocs.yml b/mkdocs.yml
index e0759cebe..ce2715b61 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -42,7 +42,6 @@ nav:
- Pretrained Models: pretrained_models.md
- Contribute: contribute.md
- License: license.md
- - CLI: cli_references.md
theme:
name: material
@@ -72,7 +71,6 @@ markdown_extensions:
- pymdownx.tabbed
- pymdownx.tasklist
- pymdownx.details
- - mkdocs-click
- pymdownx.arithmatex:
generic: true
- toc:
diff --git a/notebooks/compare-pretraining-finetuning-performance.ipynb b/notebooks/compare-pretraining-finetuning-performance.ipynb
new file mode 100644
index 000000000..0f971e1b4
--- /dev/null
+++ b/notebooks/compare-pretraining-finetuning-performance.ipynb
@@ -0,0 +1,326 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import fsspec\n",
+ "import yaml\n",
+ "import pandas as pd\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns\n",
+ "from tqdm import tqdm\n",
+ "from typing import Literal\n",
+ "from dataclasses import dataclass, field\n",
+ "from collections import defaultdict\n",
+ "from graphium.utils import fs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ROOT_DIR = \"gs://graphium-private/pretrained-models/ToyMix/cas\"\n",
+ "\n",
+ "PT_TASKS = [\"qm9\", \"tox21\", \"zinc\"]\n",
+ "PT_FT_RELS = {\n",
+ " \"toymix_gcn_1\": {\"caco2\": \"finetuning_caco2_wang_gcn_1\", \"lipophilicity\": \"finetuning_lipophilicity_astrazeneca_gcn_1\"},\n",
+ " \"toymix_gcn_2\": {\"caco2\": \"finetuning_caco2_wang_gcn_2\", \"lipophilicity\": \"finetuning_lipophilicity_astrazeneca_gcn_2\"},\n",
+ "}\n",
+ "FT_METRICS = {\"caco2\": \"graph_caco2_wang/mae/test\", \"lipophilicity\": \"graph_lipophilicity_astrazeneca/r2_score/test\"}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "@dataclass\n",
+ "class FinetuningResult: \n",
+ " name: str\n",
+ " scores: dict[str, list[float]] = field(default_factory=lambda: defaultdict(list))\n",
+ "\n",
+ " def best(self, metric, minimize: bool = False):\n",
+ " return max(self.scores[metric]) if not minimize else min(self.scores[metric])\n",
+ "\n",
+ "\n",
+ "@dataclass\n",
+ "class PretrainingResult: \n",
+ " name: str\n",
+ " loss: dict[Literal[\"qm9\", \"zinc\", \"tox21\", \"all\"], float] = field(default_factory=dict)\n",
+ " ft_results: dict[str, FinetuningResult] = field(default_factory=dict)\n",
+ "\n",
+ " @property\n",
+ " def finetuning_tasks(self):\n",
+ " return sorted(list(self.ft_results.keys()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/cas.wognum/micromamba/envs/graphium/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
+ " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
+ "100%|██████████| 2/2 [00:20<00:00, 10.05s/it]\n"
+ ]
+ }
+ ],
+ "source": [
+ "globber, _ = fsspec.core.url_to_fs(ROOT_DIR)\n",
+ "\n",
+ "results = {}\n",
+ "\n",
+ "for pt_dir, ft_dirs in tqdm(PT_FT_RELS.items()):\n",
+ " \n",
+ " # Create a new results object\n",
+ " results[pt_dir] = PretrainingResult(name=pt_dir)\n",
+ "\n",
+ " # Parse the pre-training results\n",
+ " pt_results_path = fs.join(ROOT_DIR, pt_dir, \"results\", \"test_results.yaml\")\n",
+ " with fsspec.open(pt_results_path, \"r\") as f:\n",
+ " pt_results = yaml.safe_load(f)\n",
+ " results[pt_dir].loss = {k: pt_results[f\"graph_{k}/loss/test\"] for k in PT_TASKS}\n",
+ " results[pt_dir].loss[\"all\"] = pt_results[\"loss/test\"]\n",
+ "\n",
+ " # Parse the associated fine-tuning results\n",
+ " for ft_label, ft_dir in ft_dirs.items():\n",
+ "\n",
+ " # Create a new results object\n",
+ " ft_results = FinetuningResult(name=ft_label)\n",
+ " \n",
+ " # Find all results for all trials\n",
+ " ft_results_pattern = fs.join(ROOT_DIR, ft_dir, \"**\", \"test_results.yaml\")\n",
+ " paths = globber.glob(ft_results_pattern)\n",
+ "\n",
+ " # Save all scores\n",
+ " for path in paths: \n",
+ " with globber.open(path, \"r\") as f:\n",
+ " data = yaml.safe_load(f)\n",
+ " for k, v in data.items():\n",
+ " ft_results.scores[k].append(v)\n",
+ " \n",
+ " # Save the finetuning results to the pre-training results\n",
+ " results[pt_dir].ft_results[ft_label] = ft_results\n",
+ "\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def draw_boxplot(results: dict[str, FinetuningResult], fine_tuning_task: str, metric_label: str = None, loss_label: str = \"all\", ax=None):\n",
+ " if ax is None: \n",
+ " _, ax = plt.subplots()\n",
+ " if metric_label is None:\n",
+ " metric_label = FT_METRICS[fine_tuning_task]\n",
+ " for pt_label, pt_results in results.items():\n",
+ " positions = [round(pt_results.loss[loss_label], 3)]\n",
+ " data = pt_results.ft_results[fine_tuning_task].scores[metric_label]\n",
+ " ax.boxplot(data, positions=positions)\n",
+ " ax.set_xlabel(f\"Pre-training loss on {loss_label}\")\n",
+ " ax.set_ylabel(metric_label) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAGGCAYAAACqvTJ0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABT3klEQVR4nO3dfXzO9f////sxY4ZtcpazNeZsc5KclBhKqFRC75KKkPq+lXKWTsS7dEZnIokiSQmdUd4V8hZJEs2kNKxYpMlZzFnTtufvj36OT2sbx/HaseP13Ha7Xi67vNvreO11PI9jHbf363h0HMc8xhgjAAAAAAAAIIhC3F4AAAAAAAAASh6GUgAAAAAAAAg6hlIAAAAAAAAIOoZSAAAAAAAACDqGUgAAAAAAAAg6hlIAAAAAAAAIOoZSAAAAAAAACDqGUgAAAAAAAAi6ULcXUBDZ2dn69ddfFRERIY/H4/ZyADhkjNHRo0dVs2ZNhYQUn1k5jQKKh+LYKPoEFA/0CYCtfO1TkR5K/frrr4qOjnZ7GQACZPfu3apdu7bbywgYGgUUL8WpUfQJKF7oEwBbna1PRXooFRERIemvGxkZGenyagA4lZ6erujoaO9jurigUUDxUBwbRZ+A4oE+AbCVr30q0kOp0y/njIyMJFhAMVDcXqJNo4DipTg1ij4BxQt9AmCrs/WpeLzxGAAAAAAAAEUKQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEXajbCwCKkhMnTmjr1q25tp88eVKpqamqU6eOwsPDc10eFxencuXKBWOJAIohp+2R6A8Ad3DOBMBW9MkuDKUAP2zdulWtWrXy++cSExPVsmXLQlgRgJLAaXsk+gPAHZwzAbAVfbILQynAD3FxcUpMTMy1PTk5WX379tXcuXMVHx+f588BgFNO23P6ZwEg2DhnAmAr+mQXhlKAH8qVK3fG6Xh8fDzTcwABR3sAFDV0C4Ct6JNd+KBzAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQykAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAASd60OpPXv2qG/fvqpcubLKlSunCy64QImJiW4vCwDoEwBr0ScAtqJPAPwR6uaV//7770pISFCnTp20ZMkSVatWTT/99JMqVqzo5rIAgD4BsBZ9AmAr+gTAX64OpZ5++mlFR0dr9uzZ3m116tRxb0EA8P+jTwBsRZ8A2Io+AfCXq2/fW7x4sVq3bq0bbrhB1apVU4sWLTRz5sx898/IyFB6enqOLwAoDP72SaJRAIKDPgGwFX0C4C9Xh1I7duzQ9OnT1aBBAy1btkyDBw/W0KFD9cYbb+S5/4QJExQVFeX9io6ODvKKAZQU/vZJolEAgoM+AbAVfQLgL48xxrh15WXKlFHr1q21du1a77ahQ4dqw4YN+uqrr3Ltn5GRoYyMDO/36enpio6O1pEjRxQZGRmUNQN52bhxo1q1aqXExES1bNnS7eUUOenp6YqKirLqsexvnyQaheCjPcFhW6PoE4oyuhVY9AkIHPoUWL72ydVXStWoUUONGzfOsS0+Pl67du3Kc/+wsDBFRkbm+AKAwuBvnyQaBSA46BMAW9EnAP5y9YPOExIStG3bthzbtm/frpiYGJdWVHydOHFCW7duzbX95MmTSk1NVZ06dRQeHp7r8ri4OJUrVy4YSwSsQp/skV+/JBqGkok+ucPpuZREi1By0Cf30CgUVa4OpUaMGKF27dpp/Pjx6t27t9avX68ZM2ZoxowZbi6rWNq6datatWrl98/x0kWUVPTJHk77JdEwFE/0yR20CDg7+uQeGoWiytWh1IUXXqhFixZp9OjReuyxx1S3bl1NnjxZt9xyi5vLKpbi4uKUmJiYa3tycrL69u2ruXPnKj4+Ps+fA0oi+mSP/Pol0TCUTPTJHU7PpU7/LFAS0Cf30CgUVa4OpSTpmmuu0TXXXOP2Moq9cuXKnXH6HR8fz3Qc+Af6ZIez9UuiYSh56FPwcS4F+IY+uYNGoahy9YPOAQAAAAAAUDIxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNAxlAIAAAAAAEDQMZQCAAAAAABA0DGUAgAAAAAAQNA5GkrFxsbq4MGDubYfPnxYsbGxBV4UADhFnwDYij4BsBmNAuAGR0Op1NRUZWVl5dqekZGhPXv2FHhRAOAUfQJgK/oEwGY0CoAbQv3ZefHixd5/XrZsmaKiorzfZ2VlacWKFapTp07AFgcAvqJPAGxFnwDYjEYBcJNfQ6mePXtKkjwej/r375/jstKlS6tOnTqaOHFiwBYHAL6iTwBsRZ8A2IxGAXCTX0Op7OxsSVLdunW1YcMGValSpVAWBQD+ok8AbEWfANiMRgFwk19DqdN27tyZa9vhw4dVsWLFgq4HAAqEPgGwFX0CYDMaBcANjj7o/Omnn9bbb7/t/f6GG25QpUqVVKtWLX377bcBWxwA+Is+AbAVfQJgMxoFwA2OhlKvvPKKoqOjJUnLly/X//73Py1dulTdunXTfffdF9AFAoA/6BMAW9EnADajUQDc4Ojte2lpad5gffTRR+rdu7cuv/xy1alTR23atAnoAgHAH/QJgK3oEwCb0SgAbnD0SqlzzjlHu3fvliQtXbpUXbp0kSQZY5SVlRW41QGAn+gTAFvRJwA2o1EA3ODolVLXXXedbr75ZjVo0EAHDx5Ut27dJEmbNm1S/fr1A7pAAPAHfQJgK/oEwGY0CoAbHA2lJk2apDp16mj37t165plnVKFCBUl/veTzrrvuCugCAcAf9AmAregTAJvRKABucDSUKl26tEaNGpVr+/Dhwwu6HgAoEPoEwFb0CYDNaBQANzj6TClJevPNN9W+fXvVrFlTP//8syRp8uTJ+vDDDwO2OABwgj4BsBV9AmAzGgUg2BwNpaZPn66RI0eqW7duOnz4sPeD7ypWrKjJkycHcn0A4Bf6BMBW9AmAzWgUADc4Gkq9+OKLmjlzpsaMGaNSpUp5t7du3VrfffddwBYHAP6iTwBsRZ8A2IxGAXCDo6HUzp071aJFi1zbw8LCdPz48QIvCgCcok8AbEWfANiMRgFwg6OhVN26dbVp06Zc25csWaLGjRv7fJxx48bJ4/Hk+KpevbqTJQGAJPoEwF70CYDNaBQANzj663v33XefhgwZoj/++EPGGK1fv17z58/XhAkT9Oqrr/p1rCZNmuh///uf9/u/v1QUAPxFnwDYij4BsBmNAuAGR0OpgQMHKjMzU/fff79OnDihm2++WbVq1dILL7ygPn36+LeA0FAm5wAChj4BsBV9AmAzGgXADY6GUpJ0xx136I477tCBAweUnZ2tatWqOTpOSkqKatasqbCwMLVp00bjx49XbGxsnvtmZGQoIyPD+316erqj6wRQvLnRJ4lGATg7+gTAZjzHAxBsjj5T6rLLLtPhw4clSVWqVPHGKj09XZdddpnPx2nTpo3eeOMNLVu2TDNnztTevXvVrl07HTx4MM/9J0yYoKioKO9XdHS0k+UDKMbc6pNEowCcGX0CYDOe4wFwg6Oh1KpVq3Tq1Klc2//44w998cUXPh+nW7du+te//qVmzZqpS5cu+vjjjyVJc+bMyXP/0aNH68iRI96v3bt3O1k+gGLMrT5JNArAmdEnADbjOR4AN/j19r3Nmzd7//mHH37Q3r17vd9nZWVp6dKlqlWrluPFlC9fXs2aNVNKSkqel4eFhSksLMzx8QEUX273SaJRAPJGnwDYzO1G0SegZPNrKHXBBRd4/6xnXi/hDA8P14svvuh4MRkZGUpOTlaHDh0cHwNAyUSfANiKPgGwGY0C4Ca/hlI7d+6UMUaxsbFav369qlat6r2sTJkyqlatml9/7nPUqFHq3r27zjvvPO3bt09PPPGE0tPT1b9/f3+WBQD0CYC16BMAm9EoAG7yayj1yiuvqGfPnsrOzg7Ilf/yyy+66aabdODAAVWtWlUXX3yx1q1bp5iYmIAcH0DJQZ8A2Io+AbAZjQLgJr+GUmlpabrmmmtUqlQpde/eXT169FCXLl0cvwd4wYIFjn4OAP6JPgGwFX0CYDMaBcBNfv31vdmzZ+u3337TO++8o4oVK+ree+9VlSpVdN111+n111/XgQMHCmudAHBG9AmAregTAJvRKABu8msoJUkej0cdOnTQM888o61bt2r9+vW6+OKLNXPmTNWqVUsdO3bUc889pz179hTGegEgX/QJgK3oEwCb0SgAbvF7KPVP8fHxuv/++/Xll19q9+7d6t+/v7744gvNnz8/EOsDAMfoEwBb0ScANqNRAILFr8+U+qcff/xRP/30kzp27Kjw8HBVrVpVgwYN0qBBgwK1PgBwhD4BsBV9AmAzGgUgmBy9UurgwYPq0qWLGjZsqKuuukppaWmSpNtvv12jRo0K6AIBwB/0CYCt6BMAm9EoAG5wNJQaMWKEQkNDtWvXLpUrV867/cYbb9SSJUsCtjgA8Bd9AmAr+gTAZjQKgBscvX3v008/1bJly1S7du0c2xs0aKCff/45IAsDACfoEwBb0ScANqNRANzg6JVSx48fzzE9P+3AgQMKCwsr8KIAwCn6BMBW9AmAzWgUADc4Gkp17NhRb7zxhvd7j8ej7OxsPfvss+rUqVPAFgcA/qJPAGxFnwDYjEYBcIOjt+89++yzuvTSS/XNN9/o1KlTuv/++7VlyxYdOnRIX375ZaDXCAA+o08AbEWfANiMRgFwg6NXSjVu3FibN2/WRRddpK5du+r48eO67rrrlJSUpHr16gV6jQDgM/oEwFb0CYDNaBQANzh6pZQkVa9eXY8++mgg1wIAAUGfANiKPgGwGY0CEGyOh1KSdOLECe3atUunTp3Ksf38888v0KIAoKDoEwBb0ScANqNRAILJ0VBq//79GjhwoJYsWZLn5VlZWQVaFAA4RZ8A2Io+AbAZjQLgBkefKTV8+HD9/vvvWrduncLDw7V06VLNmTNHDRo00OLFiwO9RgDwGX0CYCv6BMBmNAqAGxy9Uuqzzz7Thx9+qAsvvFAhISGKiYlR165dFRkZqQkTJujqq68O9DoBwCf0CYCt6BMAm9EoAG5w9Eqp48ePq1q1apKkSpUqaf/+/ZKkZs2aaePGjYFbHQD4iT4BsBV9AmAzGgXADY6GUo0aNdK2bdskSRdccIFeeeUV7dmzRy+//LJq1KgR0AUCgD/oEwBb0ScANqNRANzg6O17w4cPV1pamiTpkUce0RVXXKG33npLZcqU0euvvx7I9QGAX+gTAFvRJwA2o1EA3OBoKHXLLbd4/7lFixZKTU3V1q1bdd5556lKlSoBWxwA+Is+AbAVfQJgMxoFwA2OhlL/VK5cObVs2TIQhwKAgKJPAGxFnwDYjEYBCAZHQyljjN577z2tXLlS+/btU3Z2do7LFy5cGJDFAYC/6BMAW9EnADajUQDc4GgoNWzYMM2YMUOdOnXSueeeK4/HE+h1AYAj9AmAregTAJvRKABucDSUmjt3rhYuXKirrroq0OsBgAKhTwBsRZ8A2IxGAXBDiJMfioqKUmxsbKDXAgAFRp8A2Io+AbAZjQLgBkdDqXHjxunRRx/VyZMnA70eACgQ+gTAVvQJgM1oFAA3OHr73g033KD58+erWrVqqlOnjkqXLp3j8o0bNwZkcQDgL/oEwFb0CYDNaBQANzgaSg0YMECJiYnq27cvH4IHwCr0CYCt6BMAm9EoAG5wNJT6+OOPtWzZMrVv3z7Q6wGAAqFPAGxFnwDYjEYBcIOjz5SKjo5WZGRkoNcCAAVGnwDYij4BsBmNAuAGR0OpiRMn6v7771dqamqAlwMABUOfANiKPgGwGY0C4AZHb9/r27evTpw4oXr16qlcuXK5PgTv0KFDAVkcAPiLPgGwFX0CYDMaBcANjoZSkyZN4oPvAFiJPgGwFX0CYDMaBcANfg2lPv30U3Xq1EkDBgwopOUAgDP0CYCt6BMAm9EoAG7y6zOlBg8erKpVq+rGG2/UvHnzdPjw4UJaFgD4hz4BsBV9AmAzGgXATX4NpXbs2KHVq1erWbNmmjx5sqpXr67OnTtrypQpfCAeAFfRJwC2ok8AbEajALjJ77++d/7552vs2LFav369duzYoRtuuEFLly5VfHy8mjdvrocffljffPNNYawVAM6IPgGwFX0CYDMaBcAtfg+l/q5mzZoaPHiwPvnkEx04cED/+c9/lJqaqiuvvFLjx48P1BoBwG/0CYCt6BMAm9EoAMHk6K/v5aV8+fK6/vrrdf311ys7O1sHDx4M1KEBoEDoEwBb0ScANqNRAAqbo6HUlClT8tzu8XhUtmxZNWjQQB06dPDrmBMmTNBDDz2kYcOGafLkyU6WBQCF0ieJRgEoOPoEwGY8xwPgBkdDqUmTJmn//v06ceKEzjnnHBljdPjwYZUrV04VKlTQvn37FBsbq5UrVyo6Ovqsx9uwYYNmzJih888/38lyAMAr0H2SaBSAwKBPAGzGczwAbnD0mVLjx4/XhRdeqJSUFB08eFCHDh3S9u3b1aZNG73wwgvatWuXqlevrhEjRpz1WMeOHdMtt9yimTNn6pxzznGyHADwCmSfJBoFIHDoEwCb8RwPgBscDaXGjh2rSZMmqV69et5t9evX13PPPafRo0erdu3aeuaZZ/Tll1+e9VhDhgzR1VdfrS5dupx134yMDKWnp+f4AoC/C2SfJBoFIHDoEwCb8RwPgBscvX0vLS1NmZmZubZnZmZq7969kv76qw1Hjx4943EWLFigjRs3asOGDT5d74QJE/Too4/6v2AAJUag+iTRKACBRZ8A2IzneADc4OiVUp06ddK///1vJSUlebclJSXpzjvv1GWXXSZJ+u6771S3bt18j7F7924NGzZMc+fOVdmyZX263tGjR+vIkSPer927dztZPoBiLBB9kmgUgMCjTwBsxnM8AG5wNJSaNWuWKlWqpFatWiksLExhYWFq3bq1KlWqpFmzZkmSKlSooIkTJ+Z7jMTERO3bt0+tWrVSaGioQkND9fnnn2vKlCkKDQ1VVlZWrp8JCwtTZGRkji8A+LtA9EmiUQACjz4BsBnP8QC4wdHb96pXr67ly5dr69at2r59u4wxiouLU6NGjbz7dOrU6YzH6Ny5s7777rsc2wYOHKi4uDg98MADKlWqlJOlASjhAtEniUYBCDz6BMBmPMcD4AZHQ6nT4uLiFBcX5+hnIyIi1LRp0xzbypcvr8qVK+faDgD+KkifJBoFoPDQJwA24zkegGByNJTKysrS66+/rhUrVmjfvn3Kzs7Ocflnn30WkMUBgL/oEwBb0ScANqNRANzgaCg1bNgwvf7667r66qvVtGlTeTyegCxm1apVATkOgJKrsPok0SgABUOfANiM53gA3OBoKLVgwQK98847uuqqqwK9HgAoEPoEwFb0CYDNaBQANzj663tlypRR/fr1A70WACgw+gTAVvQJgM1oFAA3OBpK3XvvvXrhhRdkjAn0egCgQOgTAFvRJwA2o1EA3ODo7Xtr1qzRypUrtWTJEjVp0kSlS5fOcfnChQsDsjgA8Bd9AmAr+gTAZjQKgBscDaUqVqyoXr16BXotAFBg9AmAregTAJvRKABucDSUmj17dqDXAQABQZ8A2Io+AbAZjQLgBkefKQUAAAAAAAAUhKNXSknSe++9p3feeUe7du3SqVOncly2cePGAi8MAJyiTwBsRZ8A2IxGAQg2R6+UmjJligYOHKhq1aopKSlJF110kSpXrqwdO3aoW7dugV4jAPiMPgGwFX0CYDMaBcANjoZS06ZN04wZMzR16lSVKVNG999/v5YvX66hQ4fqyJEjgV4jAPiMPgGwFX0CYDMaBcANjoZSu3btUrt27SRJ4eHhOnr0qCSpX79+mj9/fuBWBwB+ok8AbEWfANiMRgFwg6OhVPXq1XXw4EFJUkxMjNatWydJ2rlzp4wxgVsdAPiJPgGwFX0CYDMaBcANjoZSl112mf773/9KkgYNGqQRI0aoa9euuvHGG9WrV6+ALhAA/EGfANiKPgGwGY0C4AZHf31vxowZys7OliQNHjxYlSpV0po1a9S9e3cNHjw4oAsEAH/QJwC2ok8AbEajALjB0VAqJCREISH/9yKr3r17q3fv3gFbFAA4RZ8A2Io+AbAZjQLgBkdDqYSEBF1yySW69NJLlZCQoPLlywd6XQDgCH0CYCv6BMBmNAqAGxx9ptQ111yjjRs36vrrr9c555yjtm3b6sEHH9TSpUt17NixQK8RAHxGnwDYij4BsBmNAuAGR0Op0aNHa+nSpfr999+1evVq9ejRQ5s2bdK1116rypUrB3qNAOAz+gTAVvQJgM1oFAA3OHr73mkpKSn69ttv9e2332rz5s2KjIxUhw4dArU2AHCMPgGwFX0CYDMaBSCYHA2lbrzxRq1evVrZ2dnq2LGjOnbsqNGjR+v8888P9PoAwC/0CYCt6BMAm9EoAG5wNJR69913VaVKFQ0YMECdOnVShw4dVKFChUCvDQD8Rp8A2Io+AbAZjQLgBkefKXXo0CG9+uqryszM1NixY1WlShW1adNGDzzwgJYsWRLoNQKAz+gTAFvRJwA2o1EA3OBoKFWxYkVde+21ev7555WYmKgtW7aocePGev7553XNNdcEeo0A4DP6BMBW9AmAzWgUADc4evveoUOH9Pnnn2vVqlVatWqVtmzZokqVKqlHjx7q1KlToNcIAD6jTwBsRZ8A2IxGAXCDo6FU1apVVaVKFXXo0EF33HGHLr30UjVt2jTQawMAv9EnALaiTwBsRqMAuMHRUOrbb7/1KVBffvmlWrdurbCwMCdXAwB+o08AbEWfANiMRgFwg6PPlPJ1Yt6tWzft2bPHyVUAgCP0CYCt6BMAm9EoAG5wNJTylTGmMA8PAI7RJwC2ok8AbEajAARSoQ6lAAAAAAAAgLwwlAIAAAAAAEDQMZQCAAAAAABA0BXqUMrj8RTm4QHAMfoEwFb0CYDNaBSAQOKDzgGUSPQJgK3oEwCb0SgAgRRamAc/evRoYR4eAByjTwBsRZ8A2IxGAQgkv18p9e233+qJJ57QtGnTdODAgRyXpaen67bbbgvY4gDAH/QJgK3oEwCb0SgAbvFrKPXpp5/qoosu0oIFC/T0008rPj5eK1eu9F5+8uRJzZkzJ+CLBICzoU8AbEWfANiMRgFwk19DqXHjxmnUqFH6/vvvlZqaqvvvv1/XXnutli5dWljrAwCf0CcAtqJPAGxGowC4ya/PlNqyZYvefPNNSX/91YX77rtPtWvX1vXXX6/58+froosuKpRFAsDZ0CcAtqJPAGxGowC4ya+hVFhYmA4fPpxj20033aSQkBD16dNHEydODOTaAMBn9AmAregTAJvRKABu8uvtexdccEGO9xefduONN+rVV1/V0KFD/bry6dOn6/zzz1dkZKQiIyPVtm1bLVmyxK9jAIBEnwDYiz4BsBmNAuAmv14pdeedd2r16tV5XnbTTTdJkmbMmOHz8WrXrq2nnnpK9evXlyTNmTNHPXr0UFJSkpo0aeLP0gCUcPQJgK3oEwCb0SgAbvJrKNWrVy/16tUr38tvuukmb7h80b179xzfP/nkk5o+fbrWrVtHsAD4hT4BsBV9AmAzGgXATX4Npf4pMTFRycnJ8ng8io+PV8uWLR0fKysrS++++66OHz+utm3bFmRZAECfAFiLPgGwGY0CEEyOhlL79u1Tnz59tGrVKlWsWFHGGB05ckSdOnXSggULVLVqVZ+P9d1336lt27b6448/VKFCBS1atEiNGzfOc9+MjAxlZGR4v09PT3eyfADFmFt9kmgUgDOjTwBsxnM8AG7w64POT7vnnnuUnp6uLVu26NChQ/r999/1/fffKz093e8PwmvUqJE2bdqkdevW6c4771T//v31ww8/5LnvhAkTFBUV5f2Kjo52snwAxZhbfZJoFIAzo08AbMZzPABucDSUWrp0qaZPn674+HjvtsaNG+ull17y+y8rlClTRvXr11fr1q01YcIENW/eXC+88EKe+44ePVpHjhzxfu3evdvJ8gEUY271SaJRAM6MPgGwGc/xALjB0dv3srOzVbp06VzbS5curezs7AItyBiT4+WbfxcWFqawsLACHR/wVUpKio4ePerTvsnJyTn+1xcRERFq0KCBo7Uhf271SaJRKDh/uiM5a89pNCj46BOKq8I+Z5JoVjDwHA/FTTDOq2hTwTkaSl122WUaNmyY5s+fr5o1a0qS9uzZoxEjRqhz584+H+ehhx5St27dFB0draNHj2rBggVatWqVli5d6mRZQMCkpKSoYcOGfv9c3759/dp/+/btRCzA6BOKKqfdkfxvz2k0KLjoE4qjYJ0zSTSrsNEoFCfBPK+iTQXjaCg1depU9ejRQ3Xq1FF0dLQ8Ho927dqlZs2aae7cuT4f57ffflO/fv2UlpamqKgonX/++Vq6dKm6du3qZFlAwJyeqM+dOzfHS5jzc/LkSaWmpqpOnToKDw8/6/7Jycnq27evX5N7+IY+oajytzuS/+05jQa5gz6hOCrscyaJZgULjUJxEozzKtoUGI6GUtHR0dq4caOWL1+urVu3yhijxo0bq0uXLn4dZ9asWU6uHggaf/4MbkJCQiGvBr6gTyjq/P3z27Sn6KBPKM44Zyr6aBSKI86r7OdoKHVa165dmXgDsBJ9AmAr+gTAZjQKQDA5+ut7Q4cO1ZQpU3Jtnzp1qoYPH17QNQGAY/QJgK3oEwCb0SgAbnA0lHr//ffzfFlbu3bt9N577xV4UQDgFH0CYCv6BMBmNAqAGxwNpQ4ePKioqKhc2yMjI3XgwIECLwoAnKJPAGxFnwDYjEYBcIOjoVT9+vXz/JOeS5YsUWxsbIEXBQBO0ScAtqJPAGxGowC4wdEHnY8cOVJ333239u/fr8suu0yStGLFCk2cOFGTJ08O5PoAwC/0CYCt6BMAm9EoAG5wNJS67bbblJGRoSeffFKPP/64JKlOnTqaPn26br311oAuEAD8QZ8A2Io+AbAZjQLgBkdDKUm68847deedd2r//v0KDw9XhQoVArkuAHCMPgGwFX0CYDMaBSDYHA+lTqtatWog1oEASklJ0dGjR33aNzk5Ocf/+iIiIkINGjRwtDYgmOhT0eJPu05z0jCJjsF99MluhX0uJdEh2I1G2Y1GoThxPJR677339M4772jXrl06depUjss2btxY4IXBmZSUFDVs2NDvn+vbt69f+2/fvp1IwVr0qehx2q7T/G2YRMfgDvpkv2CdS0l0CPahUfajUShuHA2lpkyZojFjxqh///768MMPNXDgQP3000/asGGDhgwZEug1wg+nJ+Zz585VfHz8Wfc/efKkUlNTVadOHYWHh591/+TkZPXt29fvVzMAwUKfiiZ/23Wavw2T6BjcQ5+KhsI+l5LoEOxEo4oGGoXixtFQatq0aZoxY4ZuuukmzZkzR/fff79iY2P18MMP69ChQ4FeIxyIj49Xy5Ytfdo3ISGhkFcDBA99Ktr8addpNAxFBX0qWjiXQklDo4oWGoXiIsTJD+3atUvt2rWTJIWHh3snqP369dP8+fMDtzoA8BN9AmAr+gTAZjQKgBscDaWqV6+ugwcPSpJiYmK0bt06SdLOnTtljAnc6gDAT/QJgK3oEwCb0SgAbnA0lLrsssv03//+V5I0aNAgjRgxQl27dtWNN96oXr16BXSBAOAP+gTAVvQJgM1oFAA3OPpMqRkzZig7O1uSNHjwYFWqVElr1qxR9+7dNXjw4IAuEAD8QZ8A2Io+AbAZjQLgBkdDqZCQEIWE/N+LrHr37q3evXsHbFEA4BR9AmAr+gTAZjQKgBscvX1v9uzZevfdd3Ntf/fddzVnzpwCLwoAnKJPAGxFnwDYjEYBcIOjodRTTz2lKlWq5NperVo1jR8/vsCLAgCn6BMAW9EnADajUQDc4Ggo9fPPP6tu3bq5tsfExGjXrl0FXhQAOEWfANiKPgGwGY0C4AZHQ6lq1app8+bNubZ/++23qly5coEXBQBO0ScAtqJPAGxGowC4wdFQqk+fPho6dKhWrlyprKwsZWVl6bPPPtOwYcPUp0+fQK8RAHxGnwDYij4BsBmNAuAGR39974knntDPP/+szp07KzT0r0NkZ2fr1ltv5f3GAFxFnwDYij4BsBmNAuAGR0OpMmXK6O2339YTTzyhTZs2KTw8XM2aNVNMTEyg1wcAfqFPAGxFnwDYjEYBcIOjodRpDRo0UIMGDfK9PDIyUps2bVJsbGxBrgYA/EafANiKPgGwGY0CEEyOPlPKV8aYwjw8ADhGnwDYij4BsBmNAhBIhTqUAgAAAAAAAPLCUAoAAAAAAABBx1AKAAAAAAAAQVeoQymPx1OYhwcAx+gTAFvRJwA2o1EAAokPOgdQItEnALaiTwBsRqMABFKhDqWWLFmiWrVqFeZVAIAj9AmAregTAJvRKACBFOrkh7KysvT6669rxYoV2rdvn7Kzs3Nc/tlnn0mS2rdvX/AVAoAf6BMAW9EnADajUQDc4GgoNWzYML3++uu6+uqr1bRpU95XDMAa9AmAregTAJvRKABucDSUWrBggd555x1dddVVgV4PABQIfQJgK/oEwGY0CoAbHH2mVJkyZVS/fv1ArwUACow+AbAVfQJgMxoFwA2OhlL33nuvXnjhBf7yAgDr0CcAtqJPAGxGowC4wee371133XU5vv/ss8+0ZMkSNWnSRKVLl85x2cKFCwOzOgDwAX0CYCv6BMBmNAqA23weSkVFReX4vlevXgFfDAA4QZ8A2Io+AbAZjQLgNp+HUrNnzw74lU+YMEELFy7U1q1bFR4ernbt2unpp59Wo0aNAn5dAIov+gTAVvQJgM0C3Sj6BMBfjj5T6rR9+/bpiy++0Jo1a7Rv3z6/f/7zzz/XkCFDtG7dOi1fvlyZmZm6/PLLdfz48YIsCwDoEwBr0ScANitIo+gTAH/5/Eqpv0tPT9eQIUO0YMECZWVlSZJKlSqlG2+8US+99FKul4HmZ+nSpTm+nz17tqpVq6bExER17NjRydIAlHD0CYCt6BMAmwWiUfQJgL8cvVLq9ttv19dff62PPvpIhw8f1pEjR/TRRx/pm2++0R133OF4MUeOHJEkVapUyfExAJRs9AmAregTAJsVRqPoE4CzcfRKqY8//ljLli1T+/btvduuuOIKzZw5U1deeaWjhRhjNHLkSLVv315NmzbNc5+MjAxlZGR4v09PT3d0XQCKL7f6JNEoAGdGnwDYLNCNok8AfOHolVKVK1fO8+WbUVFROueccxwt5O6779bmzZs1f/78fPeZMGGCoqKivF/R0dGOrgtA8eVWnyQaBeDM6BMAmwW6UfQJgC8cDaXGjh2rkSNHKi0tzbtt7969uu+++/Sf//zH7+Pdc889Wrx4sVauXKnatWvnu9/o0aN15MgR79fu3budLB9AMeZWnyQaBeDM6BMAmwWyUfQJgK8cvX1v+vTp+vHHHxUTE6PzzjtPkrRr1y6FhYVp//79euWVV7z7bty4Md/jGGN0zz33aNGiRVq1apXq1q17xusNCwtTWFiYkyUDKCHc6pNEowCcGX0CYLNANIo+AfCXo6FUz549A3LlQ4YM0bx58/Thhx8qIiJCe/fulfTXS0TDw8MDch0AShb6BMBW9AmAzQLRKPoEwF+OhlKPPPJIQK58+vTpkqRLL700x/bZs2drwIABAbkOACULfQJgK/oEwGaBaBR9AuAvR0OpQDHGuHn1AJAv+gTAVvQJgK3oEwB/ORpKZWVladKkSXrnnXe0a9cunTp1Ksflhw4dCsjiAMBf9AmAregTAJvRKABucPTX9x599FE9//zz6t27t44cOaKRI0fquuuuU0hIiMaNGxfgJQKA7+gTAFvRJwA2o1EA3OBoKPXWW29p5syZGjVqlEJDQ3XTTTfp1Vdf1cMPP6x169YFeo0A4DP6BMBW9AmAzWgUADc4Gkrt3btXzZo1kyRVqFBBR44ckSRdc801+vjjjwO3OgDwE30CYCv6BMBmNAqAGxwNpWrXrq20tDRJUv369fXpp59KkjZs2KCwsLDArQ4A/ESfANiKPgGwGY0C4AZHQ6levXppxYoVkqRhw4bpP//5jxo0aKBbb71Vt912W0AXCAD+oE8AbEWfANiMRgFwg6O/vvfUU095//n6669XdHS0vvzyS9WvX1/XXnttwBYHAP6iTwBsRZ8A2IxGAXCD30OpP//8U//v//0//ec//1FsbKwkqU2bNmrTpk3AFwcA/qBPAGxFnwDYjEYBcIvfb98rXbq0Fi1aVBhrAYACoU8AbEWfANiMRgFwi+PPlPrggw8CvBQAKDj6BMBW9AmAzWgUADc4+kyp+vXr6/HHH9fatWvVqlUrlS9fPsflQ4cODcjiAMBf9AmAregTAJvRKABucDSUevXVV1WxYkUlJiYqMTExx2Uej4dgAXANfQJgK/oEwGY0CoAbHA2ldu7cGeh1AEBA0CcAtqJPAGxGowC4wdFQauTIkXlu93g8Klu2rOrXr68ePXqoUqVKBVocAPiLPgGwFX0CYDMaBcANjoZSSUlJ2rhxo7KystSoUSMZY5SSkqJSpUopLi5O06ZN07333qs1a9aocePGgV4zAOSLPgGwFX0CYDMaBcANjv76Xo8ePdSlSxf9+uuvSkxM1MaNG7Vnzx517dpVN910k/bs2aOOHTtqxIgRgV4vAJwRfQJgK/oEwGY0CoAbHA2lnn32WT3++OOKjIz0bouMjNS4ceP0zDPPqFy5cnr44YdzfUAeABQ2+gTAVvQJgM1oFAA3OBpKHTlyRPv27cu1ff/+/UpPT5ckVaxYUadOnSrY6gDAT/QJgK3oEwCb0SgAbnD89r3bbrtNixYt0i+//KI9e/Zo0aJFGjRokHr27ClJWr9+vRo2bBjItQLAWdEnALaiTwBsRqMAuMHRB52/8sorGjFihPr06aPMzMy/DhQaqv79+2vSpEmSpLi4OL366quBWykA+IA+AbAVfQJgMxoFwA2OhlIVKlTQzJkzNWnSJO3YsUPGGNWrV08VKlTw7nPBBRcEao0A4DP6BMBW9AmAzWgUADc4GkqdVqFCBZ1//vmBWgsABAx9AmAr+gTAZjQKQDA5+kwpAAAAAAAAoCAYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoXB1KrV69Wt27d1fNmjXl8Xj0wQcfuLkcAMiBRgGwFX0CYCv6BMAfrg6ljh8/rubNm2vq1KluLgMA8kSjANiKPgGwFX0C4I9QN6+8W7du6tatm5tLAIB80SgAtqJPAGxFnwD4g8+UAgAAAAAAQNC5+kopf2VkZCgjI8P7fXp6uourAYCcaBQAW9EnALaiT0DJVqReKTVhwgRFRUV5v6Kjo91eEgB40SgAtqJPAGxFn4CSrUgNpUaPHq0jR454v3bv3u32kgDAi0YBsBV9AmAr+gSUbEXq7XthYWEKCwtzexkAkCcaBcBW9AmAregTULK5OpQ6duyYfvzxR+/3O3fu1KZNm1SpUiWdd955Lq4MAGgUAHvRJwC2ok8A/OHqUOqbb75Rp06dvN+PHDlSktS/f3+9/vrrLq0KAP5CowDYij4BsBV9AuAPV4dSl156qYwxbi4BAPJFowDYij4BsBV9AuCPIvVB5wAAAAAAACgeGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOhC3V4AYCNP5h9qUT1E4Ye3S78GfnYbfni7WlQPkSfzj4AfG0DRVNjd+TsaBCBQgtEumgXAX7Sp6GAoBeSh7LFd2vjvCtLqf0urA3/8eEkb/11Bycd2SWoX+CsAUOQUdnf+jgYBCJRgtItmAfAXbSo6GEoBefijwnlq+coxvfXWW4qPiwv48ZO3btUtt9yiWVedF/BjAyiaCrs7f0eDAARKMNpFswD4izYVHQylgDyY0LJK2putkxUbSjUvCPjxT+7NVtLebJnQsgE/NoCiqbC783c0CECgBKNdNAuAv2hT0cEHnQMAAAAAACDoeKVUMcMHdAMoiviQbwC24MNxAdiMRqG4YShVzPAB3QCKIj7kG4At+HBcADajUShuGEoVM3xAN4CiiA/5BmALPhwXgM1oFIobhlLFDB/QDaAo4kO+AdiCD8cFYDMaheKGDzoHAAAAAABA0PFKKSAPJ06ckCRt3LjRp/1Pnjyp1NRU1alTR+Hh4WfdPzk5uUDrA1D8+Nsdyf/2nEaDAARKYZ8zSTQLgP+CcV5FmwKDoVQxwzAlMLZu3SpJuuOOOwr1eiIiIgr1+EBR4eTEQSpeT26C1Z2/o0FAbgxZ/BPMdtEsgEb5ijYVHQylihmGKYHRs2dPSVJcXJzKlSvn3Z6cnKy+ffv6fby5c+cqPj4+x7aIiAg1aNCgQOsEigsGMvl3R3LeHinv/kg0CMgPT2T8E4xzJolmAafRKN8UxnkVz+kKh8cYY9xehFPp6emKiorSkSNHFBkZ6fZyrHDgwAF98MEHDFMKyYkTJ7z/R/B3Z/svEHnFEP+nuD6Wi+vtKgz5tUtiICM5b49EfwKhOD6Wi+NtCpRAn0tJJfN8inOm4CiOj+XieJsCiUYVHH0KDl8fywylSggeeLBZcX0sF9fbFWz59UuiYQiO4vhYLo63qbAxHIaNiuNjuTjepmCgUbANQykARUZxfSwX19sFlDTF8bFcHG8TUBIVx8dycbxNQEnk62M5JIhrAgAAAAAAACQxlAIAAAAAAIALGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoGEoBAAAAAAAg6BhKAQAAAAAAIOgYSgEAAAAAACDoQt1eQEEYYyRJ6enpLq8EQEGcfgyffkwXFzQKKB6KY6PoE1A80CcAtvK1T0V6KHX06FFJUnR0tMsrARAIR48eVVRUlNvLCBgaBRQvxalR9AkoXugTAFudrU8eU4TH6tnZ2fr1118VEREhj8fj9nKKpPT0dEVHR2v37t2KjIx0ezlFFvdjwRhjdPToUdWsWVMhIcXnXcU0qvCV9MdeSb/9wVIcG0WfAofHoX+4vwKLPuFseMz5jvsqsHztU5F+pVRISIhq167t9jKKhcjISB54AcD96Fxx+a97f0ejgqekP/ZK+u0PhuLWKPoUeDwO/cP9FTj0Cb7gMec77qvA8aVPxWOcDgAAAAAAgCKFoRQAAAAAAACCjqFUCRcWFqZHHnlEYWFhbi+lSON+BNxR0h97Jf32Azbgcegf7i8guHjM+Y77yh1F+oPOAQAAAAAAUDTxSikAAAAAAAAEHUMpAAAAAAAABB1DKQAAAAAAAAQdQ6kibtq0aapbt67Kli2rVq1a6Ysvvsh334ULF6pr166qWrWqIiMj1bZtWy1btizXPq1bt1bFihVVvnx5XXDBBXrzzTdzHWvPnj3q27evKleurHLlyumCCy5QYmJiwG9fsPhzPw4YMEAejyfXV5MmTfLcf8GCBfJ4POrZs2eO7ePGjct1jOrVqwfyZgHW8+exl5aWpptvvlmNGjVSSEiIhg8fnmufP//8U4899pjq1aunsmXLqnnz5lq6dGmOfTIzMzV27FjVrVtX4eHhio2N1WOPPabs7OxA37yz8uf2r1q1Ks/2bN26Ncd+hw8f1pAhQ1SjRg2VLVtW8fHx+uSTT7yXr169Wt27d1fNmjXl8Xj0wQcfFNbNA4oEfx6Ha9asUUJCgipXrqzw8HDFxcVp0qRJOfbZsmWL/vWvf6lOnTryeDyaPHlyruMU5cdhYZwzna1bnDOhJAv0872/y+95ik3nSv7w576SpIyMDI0ZM0YxMTEKCwtTvXr19Nprr3kv9+W8sij33BYMpYqwt99+W8OHD9eYMWOUlJSkDh06qFu3btq1a1ee+69evVpdu3bVJ598osTERHXq1Endu3dXUlKSd59KlSppzJgx+uqrr7R582YNHDhQAwcOzBGz33//XQkJCSpdurSWLFmiH374QRMnTlTFihUL+yYXCn/vxxdeeEFpaWner927d6tSpUq64YYbcu37888/a9SoUerQoUOex2rSpEmOY3333XcBvW2Azfx97GVkZKhq1aoaM2aMmjdvnuc+Y8eO1SuvvKIXX3xRP/zwgwYPHqxevXrl6NzTTz+tl19+WVOnTlVycrKeeeYZPfvss3rxxRcL5Xbmx9/bf9q2bdtydKNBgwbey06dOqWuXbsqNTVV7733nrZt26aZM2eqVq1a3n2OHz+u5s2ba+rUqYV224Ciwt/HYfny5XX33Xdr9erVSk5O1tixYzV27FjNmDHDu8+JEycUGxurp556Kt/BSVF9HBbGOZMv3ZI4Z0LJVBjP90470/MUW86V/OHkvKp3795asWKFZs2apW3btmn+/PmKi4vzXu7LeWVR7blVDIqsiy66yAwePDjHtri4OPPggw/6fIzGjRubRx999Iz7tGjRwowdO9b7/QMPPGDat2/v32ItVtD7cdGiRcbj8ZjU1NQc2zMzM01CQoJ59dVXTf/+/U2PHj1yXP7II4+Y5s2bF2TpQJFWkMfeJZdcYoYNG5Zre40aNczUqVNzbOvRo4e55ZZbvN9fffXV5rbbbsuxz3XXXWf69u3rx+oLzt/bv3LlSiPJ/P777/kec/r06SY2NtacOnXKpzVIMosWLfJ1yUCxE4hzqV69euXbj5iYGDNp0qQz/nxRehwWxjmTL93inAklVWE93zvb8xRbzpX84e99tWTJEhMVFWUOHjyY7zF9Oa/8u6LUc5vwSqki6tSpU0pMTNTll1+eY/vll1+utWvX+nSM7OxsHT16VJUqVcrzcmOMVqxYoW3btqljx47e7YsXL1br1q11ww03qFq1amrRooVmzpzp/Ma4KBD346xZs9SlSxfFxMTk2P7YY4+patWqGjRoUL4/m5KSopo1a6pu3brq06ePduzY4f+NAIqgQDz28pKRkaGyZcvm2BYeHq41a9Z4v2/fvr1WrFih7du3S5K+/fZbrVmzRldddZXj6/VXQW5/ixYtVKNGDXXu3FkrV67McdnixYvVtm1bDRkyROeee66aNm2q8ePHKysrK+C3ASjqAtGhpKQkrV27VpdccklhLNEqhXXO5Gu3OGdCSVOYz/fO9jzFhnMlfzi5r04/p33mmWdUq1YtNWzYUKNGjdLJkye9+/hyXomCC3V7AXDmwIEDysrK0rnnnptj+7nnnqu9e/f6dIyJEyfq+PHj6t27d47tR44cUa1atZSRkaFSpUpp2rRp6tq1q/fyHTt2aPr06Ro5cqQeeughrV+/XkOHDlVYWJhuvfXWgt+4ICro/ZiWlqYlS5Zo3rx5ObZ/+eWXmjVrljZt2pTvz7Zp00ZvvPGGGjZsqN9++01PPPGE2rVrpy1btqhy5cqObg9QVASiYXm54oor9Pzzz6tjx46qV6+eVqxYoQ8//DDHk5sHHnhAR44cUVxcnEqVKqWsrCw9+eSTuummmxxfr7+c3P4aNWpoxowZatWqlTIyMvTmm2+qc+fOWrVqlfc/HOzYsUOfffaZbrnlFn3yySdKSUnRkCFDlJmZqYcffrjQbxdQlBSkQ7Vr19b+/fuVmZmpcePG6fbbby/MpVqhsM6ZfOkW50woiQrr+Z4vz1NsOFfyh5P7aseOHVqzZo3Kli2rRYsW6cCBA7rrrrt06NAh7+dK+XJeiYJjKFXEeTyeHN8bY3Jty8v8+fM1btw4ffjhh6pWrVqOyyIiIrRp0yYdO3ZMK1as0MiRIxUbG6tLL71U0l8T99atW2v8+PGS/vqv9lu2bNH06dOL3FDqNKf34+uvv66KFSvm+HDAo0ePqm/fvpo5c6aqVKmS789269bN+8/NmjVT27ZtVa9ePc2ZM0cjR470/0YARZDTx15+XnjhBd1xxx2Ki4uTx+NRvXr1NHDgQM2ePdu7z9tvv625c+dq3rx5atKkiTZt2qThw4erZs2a6t+/v+PrdsKf29+oUSM1atTI+33btm21e/duPffcc96hVHZ2tqpVq6YZM2aoVKlSatWqlX799Vc9++yzDKWAfDjp0BdffKFjx45p3bp1evDBB1W/fn1rn6wFWiDPmSTfusU5E0qyQD7f8/V5ik3nSv7w577Kzs6Wx+PRW2+9paioKEnS888/r+uvv14vvfSSwsPDfTqvRMExlCqiqlSpolKlSuWa/O7bty/XhPif3n77bQ0aNEjvvvuuunTpkuvykJAQ1a9fX5J0wQUXKDk5WRMmTPAOpWrUqKHGjRvn+Jn4+Hi9//77BbhF7ijI/WiM0WuvvaZ+/fqpTJky3u0//fSTUlNT1b17d++203+pIjQ0VNu2bVO9evVyHa98+fJq1qyZUlJSCnKTgCKhII+9M6latao++OAD/fHHHzp48KBq1qypBx98UHXr1vXuc9999+nBBx9Unz59JP31BOfnn3/WhAkTgnaiFajbf/HFF2vu3Lne72vUqKHSpUurVKlS3m3x8fHau3evTp06laNVQElXkMfh6aY0a9ZMv/32m8aNG1fsh1KFcc4kOesW50woCQrj+Z6vz1NsOFfyh5P7qkaNGqpVq5Z3ICX91R5jjH755Rc1aNDAp/NKFByfKVVElSlTRq1atdLy5ctzbF++fLnatWuX78/Nnz9fAwYM0Lx583T11Vf7dF3GGGVkZHi/T0hI0LZt23Lss3379lyfqVQUOL0fJenzzz/Xjz/+mOu92HFxcfruu++0adMm79e1116rTp06adOmTYqOjs7zeBkZGUpOTlaNGjUKdqOAIqAgjz1flC1bVrVq1VJmZqbef/999ejRw3vZiRMnFBKS8//+SpUqFdQ/cxyo25+UlJSjGQkJCfrxxx9z3Jbt27erRo0aDKSAfwjU4/Cf50nFVWGcM0nOusU5E0qCwni+5+vzFBvOlfzh5L5KSEjQr7/+qmPHjnm3bd++XSEhIapdu3aOfc90XokAcOPT1REYCxYsMKVLlzazZs0yP/zwgxk+fLgpX7689y+aPPjgg6Zfv37e/efNm2dCQ0PNSy+9ZNLS0rxfhw8f9u4zfvx48+mnn5qffvrJJCcnm4kTJ5rQ0FAzc+ZM7z7r1683oaGh5sknnzQpKSnmrbfeMuXKlTNz584N3o0PIH/vx9P69u1r2rRp49N15PVXLe69916zatUqs2PHDrNu3TpzzTXXmIiIiFx/xQ8orpw89pKSkkxSUpJp1aqVufnmm01SUpLZsmWL9/J169aZ999/3/z0009m9erV5rLLLjN169bN8Rfr+vfvb2rVqmU++ugjs3PnTrNw4UJTpUoVc//99wfldp/m7+2fNGmSWbRokdm+fbv5/vvvzYMPPmgkmffff9+7z65du0yFChXM3XffbbZt22Y++ugjU61aNfPEE0949zl69Kj3fpRknn/+eZOUlGR+/vnn4N14wBL+Pg6nTp1qFi9ebLZv3262b99uXnvtNRMZGWnGjBnj3ScjI8P7GKtRo4YZNWqUSUpKMikpKd59iurjsDDOmXzpFudMKKkK4/neP+X1PMWWcyV/+HtfHT161NSuXdtcf/31ZsuWLebzzz83DRo0MLfffrt3H1/OK4tqz23CUKqIe+mll0xMTIwpU6aMadmypfn888+9l/Xv399ccskl3u8vueQSIynXV//+/b37jBkzxtSvX9+ULVvWnHPOOaZt27ZmwYIFua73v//9r2natKkJCwszcXFxZsaMGYV5MwudP/ejMcYcPnzYhIeH+3y784r9jTfeaGrUqGFKly5tatasaa677rocT66BksDfx15eDYuJifFevmrVKhMfH2/CwsJM5cqVTb9+/cyePXtyHCM9Pd0MGzbMnHfeeaZs2bImNjbWjBkzxmRkZBTmTc2TP7f/6aefNvXq1fP2uX379ubjjz/Odcy1a9eaNm3amLCwMBMbG2uefPJJk5mZ6b185cqVZ/3/AqAk8edxOGXKFNOkSRNTrlw5ExkZaVq0aGGmTZtmsrKyvPvs3Lkzz8fY349TlB+HhXHOdLZucc6EkizQz/f+Ka/nKTadK/nD3z4lJyebLl26mPDwcFO7dm0zcuRIc+LECe/lvpxXFuWe28JjjDGF+EIsAAAAAAAAIBc+UwoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCrlceumlGj58uM/7p6amyuPxaNOmTYW2JklatWqVPB6PDh8+XKjXA8Ae9AhAUUO37FDSbi9QEHTLLh6PRx988IGk4N3XbmIoVUgGDBggj8cjj8ej0qVLKzY2VqNGjdLx48cL5bp69uwZsOMtXLhQjz/+uM/7R0dHKy0tTU2bNg3YGlBwK1asULt27RQREaEaNWrogQceUGZmptvLggvoEXwR6N/daRMmTNCFF16oiIgIVatWTT179tS2bdty7LNw4UJdccUVqlKlSrE/8YJv6BYKql27dkpLS1NUVJTbS0EJQbcAZxhKFaIrr7xSaWlp2rFjh5544glNmzZNo0aNynPfP//8s9DX4+t1VKpUSRERET4ft1SpUqpevbpCQ0OdLg0BtnnzZl111VW68sorlZSUpAULFmjx4sV68MEH3V4aXEKP4JbPP/9cQ4YM0bp167R8+XJlZmbq8ssvz3GSfvz4cSUkJOipp55ycaWwDd1CQZQpU0bVq1eXx+NxeykoQegW4IBBoejfv7/p0aNHjm233367qV69ujHGmEceecQ0b97czJo1y9StW9d4PB6TnZ1tDh8+bO644w5TtWpVExERYTp16mQ2bdqU7/U88sgjRlKOr5UrV5qdO3caSebtt982l1xyiQkLCzOvvfaaOXDggOnTp4+pVauWCQ8PN02bNjXz5s3LccxLLrnEDBs2zPt9TEyMefLJJ83AgQNNhQoVTHR0tHnllVe8l5++rqSkJGOMMStXrjSSzP/+9z/TqlUrEx4ebtq2bWu2bt2a43oef/xxU7VqVVOhQgUzaNAg88ADD5jmzZvne1tPH/f333/3bnvvvfdM48aNTZkyZUxMTIx57rnncvzMSy+9ZOrXr2/CwsJMtWrVzL/+9S/vZe+++65p2rSpKVu2rKlUqZLp3LmzOXbsWL7Xv2rVKnPhhReaMmXKmOrVq5sHHnjA/Pnnnznut3vuucfcd9995pxzzjHnnnuueeSRR/I9njHGZGZmmhEjRpioqChTqVIlc99995lbb701x787l1xyibn77rvNsGHDTMWKFU21atXMK6+8Yo4dO2YGDBhgKlSoYGJjY80nn3zi/ZnRo0eb1q1b57iuRYsWmbJly5r09PQzrgnFDz2iR2frUX6/O2OM2bx5s+nUqZN3bXfccYc5evSo934oXbq0Wb16tfdYzz33nKlcubL59ddf87yuffv2GUnm888/z3XZP39/KLnoFt3y5Tzqn787SSYmJibP2zt79mwTFRVlli5dauLi4kz58uXNFVdckatVs2bN8t4n1atXN0OGDDnjGoDT6Bbd8qVb69evN126dDGVK1c2kZGRpmPHjiYxMTHHPpLMokWLjDEl49yIoVQhyStK99xzj6lcubIx5q+YnP4/w40bN5pvv/3WZGdnm4SEBNO9e3ezYcMGs337dnPvvfeaypUrm4MHD+Z5PUePHjW9e/c2V155pUlLSzNpaWkmIyPD+y9vnTp1zPvvv2927Nhh9uzZY3755Rfz7LPPmqSkJPPTTz+ZKVOmmFKlSpl169Z5j5lXlCpVqmReeuklk5KSYiZMmGBCQkJMcnKyMSb/KLVp08asWrXKbNmyxXTo0MG0a9fOe8y5c+easmXLmtdee81s27bNPProoyYyMtKvKH3zzTcmJCTEPPbYY2bbtm1m9uzZJjw83MyePdsYY8yGDRtMqVKlzLx580xqaqrZuHGjeeGFF4wxxvz6668mNDTUPP/882bnzp1m8+bN5qWXXvI+0fqnX375xZQrV87cddddJjk52SxatMhUqVIlR3QuueQSExkZacaNG2e2b99u5syZYzwej/n000/zvU1PP/20iYqKMu+995754YcfzKBBg0xERESuoVRERIR5/PHHzfbt283jjz9uQkJCTLdu3cyMGTPM9u3bzZ133mkqV65sjh8/bowxZuTIkaZ9+/Y5rmvp0qU5nmii5KBH9OhsPcrvd3f8+HFTs2ZNc91115nvvvvOrFixwtStW9f079/f+7P33XefiYmJMYcPHzabNm0yYWFhZuHChfnedykpKUaS+e6773JdVhJOvOAbukW3fDmPOv07S0tLMz/++KOpX7++6devX563d/bs2aZ06dKmS5cuZsOGDSYxMdHEx8ebm2++2Xu8adOmmbJly5rJkyebbdu2mfXr15tJkyble/3A39EtuuVLt1asWGHefPNN88MPP3if/5177rk5XjjAUAoB8c8off3116Zy5cqmd+/expi/olS6dGmzb98+7z4rVqwwkZGR5o8//shxrHr16uWYTJ/tuoz5v395J0+efNa1XnXVVebee+/1fp9XlPr27ev9Pjs721SrVs1Mnz49x3XlNSk/7eOPPzaSzMmTJ40xxrRp0ybXf3lKSEjwK0o333yz6dq1a4597rvvPtO4cWNjjDHvv/++iYyMzPOVQYmJiUaSSU1Nzff6/u6hhx4yjRo1MtnZ2d5tL730kqlQoYLJysoyxvx1v/1zEHThhReaBx54IN/j1qhRwzz11FPe7//8809Tu3btXEOpvx83MzPTlC9f3nvSZcxfJ2WSzFdffWWMMWbZsmUmJCTEzJs3z2RmZppffvnFtG/f3kjK9V9GUPzRI3pkzNl7lNfvbsaMGeacc87J8V8RP/74YxMSEmL27t1rjDEmIyPDtGjRwvTu3ds0adLE3H777fleR3Z2tunevXuutZ1WEk684Bu6RbeMOXu3TsvOzja9evUyrVq1MidOnMjz9s6ePdtIMj/++GOONZx77rne72vWrGnGjBnj020C/olu0S1jfO/WaZmZmSYiIsL897//9W4raUMpPlOqEH300UeqUKGCypYtq7Zt26pjx4568cUXvZfHxMSoatWq3u8TExN17NgxVa5cWRUqVPB+7dy5Uz/99JN27dqVY/v48ePPuobWrVvn+D4rK0tPPvmkzj//fO/1fPrpp9q1a9cZj3P++ed7/9nj8ah69erat2+fzz9To0YNSfL+zLZt23TRRRfl2P+f359NcnKyEhIScmxLSEhQSkqKsrKy1LVrV8XExCg2Nlb9+vXTW2+9pRMnTkiSmjdvrs6dO6tZs2a64YYbNHPmTP3+++9nvK62bdvm+FyChIQEHTt2TL/88kuet/n07c7vfjpy5IjS0tLUtm1b77bQ0NBcv7N/HrdUqVKqXLmymjVr5t127rnnSvq/+/fyyy/Xs88+q8GDByssLEwNGzbU1Vdf7f15lDz0iB6dqUdnuq7mzZurfPnyOa4rOzvb+2HlZcqU0dy5c/X+++/r5MmTmjx5cr7Hu/vuu7V582bNnz/fr3WgZKJbdMvXbj300EP66quv9MEHHyg8PDzf/cqVK6d69erlefx9+/bp119/VefOnc96fUB+6BbdOlu39u3bp8GDB6thw4aKiopSVFSUjh07dtbfR3HGJ5MVok6dOmn69OkqXbq0atasqdKlS+e4/O8n+ZKUnZ2tGjVqaNWqVbmOVbFiRVWsWDHHXySqVKnSWdfwz+uYOHGiJk2apMmTJ6tZs2YqX768hg8frlOnTp3xOP9cu8fjUXZ2ts8/c/rB/Pef+ecHTxpjzni8fzLGnPEYERER2rhxo1atWqVPP/1UDz/8sMaNG6cNGzaoYsWKWr58udauXatPP/1UL774osaMGaOvv/5adevW9eu6/r7dyf3ki7yOe7b7d+TIkRoxYoTS0tJ0zjnnKDU1VaNHj87z9qH4o0f0yEmP8rquvx/vtLVr10qSDh06pEOHDuX6XUvSPffco8WLF2v16tWqXbu2X+tAyUS36JYv99PcuXM1adIkrVq16qxtyev4p9dxpmEW4Cu6RbfOdj8NGDBA+/fv1+TJkxUTE6OwsDC1bdv2rL+P4oxXShWi8uXLq379+oqJicn1L2teWrZsqb179yo0NFT169fP8VWlSpVc209HqUyZMsrKyvJpTV988YV69Oihvn37qnnz5oqNjVVKSkqBbqcTjRo10vr163Ns++abb/w6RuPGjbVmzZoc29auXauGDRt6Xw0UGhqqLl266JlnntHmzZuVmpqqzz77TNJfwUhISNCjjz6qpKQklSlTRosWLcr3utauXZsjemvXrlVERIRq1arl17pPi4qKUo0aNbRu3TrvtszMTCUmJjo6Xl48Ho9q1qyp8PBwzZ8/X9HR0WrZsmXAjo+igx7ljx79Ja/fXePGjbVp06Ycfynvyy+/VEhIiBo2bChJ+umnnzRixAjNnDlTF198sW699dYcJ2PGGN19991auHChPvvsMwbj8Bndyh/d+stXX32l22+/Xa+88oouvvhix8eR/noyW6dOHa1YsaJAx0HJRrfyR7f+8sUXX2jo0KG66qqr1KRJE4WFhenAgQOOj1ccMJSySJcuXdS2bVv17NlTy5YtU2pqqtauXauxY8ee8QFbp04dbd68Wdu2bdOBAwfO+Kc/69ev750QJycn69///rf27t1bGDfnjO655x7NmjVLc+bMUUpKip544glt3rzZrz/be++992rFihV6/PHHtX37ds2ZM0dTp071/tnVjz76SFOmTNGmTZv0888/64033lB2drYaNWqkr7/+WuPHj9c333yjXbt2aeHChdq/f7/i4+PzvK677rpLu3fv1j333KOtW7fqww8/1COPPKKRI0cqJMT5w2jYsGF66qmntGjRIm3dulV33XWXDh8+7Ph4f/fss8/qu+++05YtW/T444/rqaee0pQpU3j7HnxCj0pej/L63d1yyy0qW7as+vfvr++//14rV67UPffco379+uncc89VVlaW+vXrp8svv1wDBw7U7Nmz9f3332vixIne4w4ZMkRz587VvHnzFBERob1792rv3r06efKkd59Dhw5p06ZN+uGHHyT99RL/TZs2ufLvA4ouulWyurV371716tVLffr00RVXXOFty/79+x0dT5LGjRuniRMnasqUKUpJSdHGjRtzvPUKCDS6VbK6Jf31+3jzzTeVnJysr7/+WrfcckuJf6UmQymLeDweffLJJ+rYsaNuu+02NWzYUH369FFqaqr3M4Pycscdd6hRo0Zq3bq1qlatqi+//DLfff/zn/+oZcuWuuKKK3TppZeqevXq6tmzZyHcmjO75ZZbNHr0aI0aNUotW7bUzp07NWDAAJUtW9bnY7Rs2VLvvPOOFixYoKZNm+rhhx/WY489pgEDBkj66yWvCxcu1GWXXab4+Hi9/PLLmj9/vpo0aaLIyEitXr1aV111lRo2bKixY8dq4sSJ6tatW57XVatWLX3yySdav369mjdvrsGDB2vQoEEaO3Zsge6He++9V7feeqsGDBigtm3bKiIiQr169SrQMU9bsmSJOnTooNatW+vjjz/Whx9+6MrvGkUTPSp5Pcrrd1euXDktW7ZMhw4d0oUXXqjrr79enTt31tSpUyVJTz75pFJTUzVjxgxJUvXq1fXqq69q7Nix3rcbTJ8+XUeOHNGll16qGjVqeL/efvtt73UvXrxYLVq08H72XZ8+fdSiRQu9/PLLBbpNKFnoVsnq1tatW/Xbb79pzpw5Odpy4YUXOj5m//79NXnyZE2bNk1NmjTRNddc48orSlBy0K2S1S1Jeu211/T777+rRYsW6tevn4YOHapq1aoV6JhFncf4+0ZOoJB07dpV1atX15tvvun2Ulw1YMAAHT58WB988IHbSwFKLHoEoKihWwCKGroFiQ86h0tOnDihl19+WVdccYVKlSql+fPn63//+5+WL1/u9tIAlDD0CEBRQ7cAFDV0C/lhKAVXnH6p6hNPPKGMjAw1atRI77//vrp06eL20gCUMPQIQFFDtwAUNXQL+eHtewAAAAAAAAg6PugcAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABBx1AKAAAAAAAAQcdQCgAAAAAAAEHHUAoAAAAAAABB9/8BTFHE/p4+VYsAAAAASUVORK5CYII=",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "fig, axs = plt.subplots(nrows=1, ncols=4, figsize=(12, 4))\n",
+ "for i, loss_label in enumerate(PT_TASKS + [\"all\"]):\n",
+ " draw_boxplot(results, \"caco2\", loss_label=loss_label, ax=axs[i])\n",
+ "plt.tight_layout()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAGGCAYAAACqvTJ0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABjMklEQVR4nO3deZyN9f//8ecxmBmMdSxjacbaEFlbpBKh1CfRIhXhg+/HR6XSxkeLRJRKScmWFqUNrSp7KokGH6mxhMmeJDO2Rmbevz/6zfk0zeCca65zrvfMedxvt7llrnOd67zO6Dy65t1ZfMYYIwAAAAAAACCMink9AAAAAAAAACIPi1IAAAAAAAAIOxalAAAAAAAAEHYsSgEAAAAAACDsWJQCAAAAAABA2LEoBQAAAAAAgLBjUQoAAAAAAABhx6IUAAAAAAAAwq641wOEU3Z2tnbv3q24uDj5fD6vxwHw/xljdOjQIVWvXl3FikXmWjl9AuwV6Y2iT4C9Ir1PEo0CbBVonyJqUWr37t2qVauW12MAOIkdO3aoZs2aXo/hCfoE2C9SG0WfAPtFap8kGgXY7nR9iqhFqbi4OEl//lDKli3r8TQAcmRkZKhWrVr+x2gkok+AvSK9UfQJsFek90miUYCtAu1TRC1K5Tyds2zZsgQLsFAkP+WaPgH2i9RG0SfAfpHaJ4lGAbY7XZ8i84XHAAAAAAAA8BSLUgAAAAAAAAg7FqUAAAAAAAAQdixKAQAAAAAAIOxYlAIAAAAAAEDYOVqUqlOnjn799dc82w8ePKg6deoUeCgAcIo+AbAVfQJgMxoFwAuOFqXS0tKUlZWVZ3tmZqZ27dpV4KEAwCn6BMBW9AmAzWgUAC8UD2bnDz74wP/nzz77TOXKlfN/n5WVpUWLFikpKcm14QAgUPQJgK3oEwCb0SgAXgpqUapr166SJJ/Pp969e+e6rESJEkpKStJTTz3l2nAAECj6BMBW9AmAzWgUAC8FtSiVnZ0tSapdu7ZWrVql+Pj4kAwFAMGiTwBsRZ8A2IxGAfBSUItSObZt25Zn28GDB1W+fPmCzgMABUKfANiKPgGwGY0C4AVHi1KPP/64kpKSdMMNN0iSrr/+es2ePVsJCQmaN2+emjZt6uqQCL2jR49qw4YNebYfO3ZMaWlpSkpKUmxsbK7LkpOTVapUqXCNCASEPkUeJ/2SaBjCjz4VbbQIhR2NKtpoFGzlaFFq8uTJmjlzpiRpwYIFWrhwoT799FO9/fbbuvfeezV//nxXh0TobdiwQS1btgzqOikpKWrRokWIJgKcoU+Rx0m/JBqG8KNPRRstQmFHo4o2GgVbOVqU2rNnj2rVqiVJ+uijj9S9e3d16tRJSUlJOu+881wdEOGRnJyslJSUPNtTU1PVs2dPzZw5Uw0bNsxzHcA29CnyOOlXzvWAcKJPRRstQmFHo4o2GgVbOVqUqlChgnbs2KFatWrp008/1ahRoyRJxhhlZWW5OiDCo1SpUqdcAW/YsCEr5CgU6FPkoV8oLOhT0UaLUNjRqKKNRsFWjhalrrnmGt10002qX7++fv31V3Xu3FmStHbtWtWrV8/VAQEgGPQJgK3oEwCb0SgAXnC0KDV+/HglJSVpx44deuKJJ1SmTBlJfz7lc9CgQa4OCADBoE8AbEWfANiMRgHwgqNFqRIlSuiee+7Js/3OO+8s6DwAUCD0CYCt6BMAm9EoAF4o5vSKr732mi688EJVr15dP/30kyTpmWee0fvvv+/acADgBH0CYCv6BMBmNApAuDlalJo0aZKGDBmizp076+DBg/43vitfvryeeeYZN+cDgKDQJwC2ok8AbEajAHjB0aLUc889p6lTp2r48OGKioryb2/VqpW+++4714YDgGDRJwC2ok8AbEajAHjB0aLUtm3b1Lx58zzbo6OjdeTIkQIPBQBO0ScAtqJPAGxGowB4wdGiVO3atbV27do82z/55BM1atSooDMBgGP0CYCt6BMAm9EoAF5w9Ol79957r2699Vb9/vvvMsZo5cqVmjVrlsaMGaNp06a5PSMABIw+AbAVfQJgMxoFwAuOFqX69u2rEydO6L777tPRo0d10003qUaNGnr22WfVo0cPt2cEgIDRJwC2ok8AbEajAHjB0aKUJA0YMEADBgzQ/v37lZ2drSpVqrg5FwA4Rp8A2Io+AbAZjQIQbo7eU6p9+/Y6ePCgJCk+Pt4fq4yMDLVv39614QAgWPQJgK3oEwCb0SgAXnC0KLV06VIdP348z/bff/9dX3zxRYGHAgCn6BMAW9EnADajUQC8ENTL99atW+f/8w8//KC9e/f6v8/KytKnn36qGjVquDcdQmLz5s06dOhQQPumpqbm+ufpxMXFqX79+o5nA5yiT0VfMO2Sgu9XDjoGt9GnoieU51ISHUJ40aiih0ahMAlqUapZs2by+Xzy+Xz5PoUzNjZWzz33nGvDwX2bN29WgwYNgr5ez549A95306ZNRAphR5+KNqftkoLrVw46BjfRp6IlHOdSEh1C+NCoooVGobAJalFq27ZtMsaoTp06WrlypSpXruy/rGTJkqpSpYqioqJcHxLuyVkxnzlzpho2bHja/Y8dO6a0tDQlJSUpNjb2lPumpqaqZ8+eQT2TAXALfSragm2XFFy/ctAxhAJ9KlpCeS4l0SGEH40qWmgUCpugFqUmT56srl27Kjs7O1TzIEwaNmyoFi1aBLRvmzZtQjwNUHD0KTIE0y6JfsEO9Klo4lwKRQWNKppoFAqLoN7ofM+ePfrHP/6hhIQE/d///Z8+/vhjZWZmhmo2AAgYfQJgK/oEwGY0CoCXglqUmjFjhn7++We9/fbbKl++vO6++27Fx8frmmuu0csvv6z9+/eHak6/F154QbVr11ZMTIxatmzJJ0EAkESfANjLhj5JNApA/mxoFH0CIldQL9+TJJ/Pp4suukgXXXSRnnjiCaWmpurDDz/U1KlT9a9//UvnnXeeunTpohtvvNH1T2l46623dOedd+qFF15QmzZtNHnyZHXu3Fk//PCDzjjjDFdvCwjE0aNHtWHDhjzbT/Xa7OTkZJUqVSpcI0YU+oRIcbL2SKd/bwga5A0v+yTRKHjPyTmTRLPChXMoRDL65DHjop9//tlMmzbNdOnSxYwbN87NQxtjjDn33HPNwIEDc21LTk42Q4cODej66enpRpJJT093fbbCIiUlxUgyKSkpherYtsq5z8F8RdLPJ1DheGzSp8ItXH0pLB1z0h4a5FyoH5+h7pMxBWsUfcot1J0oLB0KltNuFbWfg9s4h6JRf0ejgkefQiPQx2bQz5T6qx9//FFbtmzRxRdfrNjYWFWuXFn9+vVTv379CnLYfB0/flwpKSkaOnRoru2dOnXS8uXL871OZmZmrtdDZ2RkuD4XIltycrJSUlLybM/5VIr8PvUiOTk5XONFNPqEouxk7ZFO3Z+c68Jb4eyTFHyj6BNCwck5U871EF6cQyHS0CdvOVqU+vXXX3XDDTdo8eLF8vl82rx5s+rUqaP+/furQoUKevLJJ92eU/v371dWVpaqVq2aa3vVqlW1d+/efK8zZswYPfLII67PAuQoVarUKT/VIthPCkPB0SdEgtO1R6I/NvKiT1LwjaJPCAXOmezHORQiFX3yVlBvdJ7jrrvuUvHixbV9+/Zcr6G84YYb9Mknn7g2XH58Pl+u740xebblGDZsmNLT0/1fO3bsCOlsALxHnwDYyss+SYE3ij4BkYlzKABecPRMqfnz5+uzzz5TzZo1c22vX7++fvrpJ1cG+7v4+HhFRUXlWTHft29fnpX1HNHR0YqOjg7JPIWV78Tval6tmGIPbpJ2O1qTPKnYg5vUvFox+U787upxgWDQp6IplO36KzqGUPKiT1LwjaJPpxbqHtEheIVzqKKBRqGwcbQodeTIkXzfZX7//v0hC0TJkiXVsmVLLViwQN26dfNvX7Bgga6++uqQ3GZRFHN4u1b/q4y07F/SMneP3VDS6n+VUerh7ZIucPfgQIDoU9EUynb9FR1DKHnRJ4lGuS3UPaJD8ArnUEUDjUJh42hR6uKLL9arr76qRx99VNKfT7fMzs7WuHHj1K5dO1cH/KshQ4aoV69eatWqlVq3bq0pU6Zo+/btGjhwYMhus6j5vcwZajH5sF5//XU1dPmN2VI3bNDNN9+s6VcUzY9u3bx5sw4dOhTQvqmpqbn+eTpxcXGqX7++49nwP/SpaAplu/7Kxo4F0x4p+P5INChcvOqTRKPcFOoe2dihYIXynEmiWaHCOVTRQKNOjT7Zx9Gi1Lhx43TJJZfo22+/1fHjx3Xffffp+++/14EDB/TVV1+5PaPfDTfcoF9//VUjR47Unj171LhxY82bN0+JiYkhu82ixhSP0Zq92TpWvoFUvZmrxz62N1tr9mbLFI9x9bg22Lx5sxo0aBD09Xr27Bnwvps2bSJgLqBPRVMo2/VXtnXMaXuk4Poj0aBw8KpPEo1yU6h7ZFuHghWOcyaJZoUC51BFA406OfpkJ0eLUo0aNdK6des0adIkRUVF6ciRI7rmmmt06623KiEhwe0Zcxk0aJAGDRoU0tsA/i5nNf1kHwf6d8eOHVNaWpqSkpIUGxt7yn1zPmo0mGdC4OToE4qSYNsjBdcfiQaFk5d9kmgUwiOU50wSzQolzqFQ1NEnOwW9KPXHH3+oU6dOmjx5Mh/FiYgTzMeBtmnTJsTT4O/oE4qqYD+KmP7Yhz4h0nDOVLjQKEQS+mSXoN+Ov0SJElq/fv1JP6ITALxCnwDYij4BsBmNAuAVR58Recstt2j69OluzwIABUafANiKPgGwGY0C4AVH7yl1/PhxTZs2TQsWLFCrVq1UunTpXJc//fTTrgwHAMGiTwBsRZ8A2IxGAfCCo0Wp9evX+1+DuWnTplyX8ZRPAF6iTwBsRZ8A2IxGAfCCo0WpJUuWuD0HwuTo0aOSpNWrVwe0f7CfIgd4jT4VTcG2Swr+E1MkOobQok9FQyjPpSQ6BO/QqKKBRqGwcbQo9Vc7d+6Uz+dTjRo13JgHIbZhwwZJ0oABA0J2G3FxcSE7NhAM+lR0hKNdf0XHEGr0qfAKV4/oELxEowovGoXCxtGiVHZ2tkaNGqWnnnpKhw8flvTnv5R33323hg8frmLFHL1/OsKga9eukqTk5GSVKlXqtPunpqaqZ8+emjlzpho2bHja/ePi4lS/fv2Cjgk4Rp+KpmDbJQXfrxx0DKFCn4qGUJ9LSXQI3qBRRQONQmHjaFFq+PDhmj59usaOHas2bdrIGKOvvvpKI0aM0O+//67Ro0e7PSdcEh8fr/79+wd9vYYNG/pfYw7YjD4VTU7bJdEv2IM+FQ2cS6GoolFFA41CYeNoUeqVV17RtGnT1KVLF/+2pk2bqkaNGho0aBDBAuAZ+gTAVvQJgM1oFAAvOHoO5oEDB5ScnJxne3Jysg4cOFDgoQDAKfoEwFb0CYDNaBQALzhalGratKkmTpyYZ/vEiRPVtGnTAg8FAE7RJwC2ok8AbEajAHjB0cv3nnjiCV155ZVauHChWrduLZ/Pp+XLl2vHjh2aN2+e2zMCQMDoEwBb0ScANqNRALzg6JlSbdu21caNG9WtWzcdPHhQBw4c0DXXXKONGzfqoosucntGAAgYfQJgK/oEwGY0CoAXHD1TSpJq1KjBm90BsBJ9AmAr+gTAZjQKQLg5eqbUjBkz9M477+TZ/s477+iVV14p8FAA4BR9AmAr+gTAZjQKgBccLUqNHTtW8fHxebZXqVJFjz32WIGHAgCn6BMAW9EnADajUQC84GhR6qefflLt2rXzbE9MTNT27dsLPBQAOEWfANiKPgGwGY0C4AVHi1JVqlTRunXr8mz/73//q0qVKhV4KABwij4BsBV9AmAzGgXAC44WpXr06KHBgwdryZIlysrKUlZWlhYvXqw77rhDPXr0cHtGAAgYfQJgK/oEwGY0CoAXHH363qhRo/TTTz/p0ksvVfHifx4iOztbt9xyC683BuAp+gTAVvQJgM1oFAAvOFqUKlmypN566y2NGjVKa9euVWxsrJo0aaLExES35wOAoNAnALaiTwBsRqMAeMHRolSO+vXrq379+srKytJ3332nsmXLqkKFCm7NBgCO0ScAtqJPAGxGowCEk6P3lLrzzjs1ffp0SVJWVpbatm2rFi1aqFatWlq6dKmb8wFAUOgTAFvRJwA2o1EAvODomVLvvvuuevbsKUn68MMPtXXrVm3YsEGvvvqqhg8frq+++srVIRF6R48e1YYNG/JsT01NzfXPv0pOTlapUqVCPhsQDPoUeZz0S6JhCD/6VLTRIhR2NKpoo1GwlaNFqf3796tatWqSpHnz5ql79+5q0KCB+vXrpwkTJrg6IMJjw4YNatmy5Ukvz/kP1F+lpKSoRYsWoRwLCBp9ijxO+iXRMIQffSraaBEKOxpVtNEo2MrRolTVqlX1ww8/KCEhQZ9++qleeOEFSX+uvkZFRbk6IMIjOTlZKSkpebYfO3ZMaWlpSkpKUmxsbJ7rALahT5HHSb9yrgeEE30q2mgRCjsaVbTRKNjK0aJU37591b17dyUkJMjn86ljx46SpG+++YZ/aQupUqVKnXQFvE2bNmGeBnCOPkUe+oXCgj4VbbQIhR2NKtpoFGzlaFFqxIgRaty4sXbs2KHrr79e0dHRkqSoqCgNHTrU1QEBIBj0CYCt6BMAm9EoAF5wtCglSdddd12ebb179871fZMmTTRv3jzVqlXL6c0AQNDoEwBb0ScANqNRAMKtWCgPnpaWpj/++COUNwEAjtAnALaiTwBsRqMAuCmki1IAAAAAAABAfliUAgAAAAAAQNixKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMIupItSkydPVtWqVUN5EwDgCH0CYCv6BMBmNAqAmxwtSu3cuVOHDx/Os/2PP/7QsmXL/N/fdNNNKl26tPPpACBI9AmAregTAJvRKABeCGpRas+ePTr33HOVmJio8uXLq3fv3rnCdeDAAbVr1871IQHgdOgTAFvRJwA2o1EAvBTUotTQoUMVFRWlb775Rp9++ql++OEHXXLJJfrtt9/8+xhjXB8SAE6HPgGwFX0CYDMaBcBLQS1KLVy4UM8++6xatWqlDh066Msvv1TNmjXVvn17HThwQJLk8/lCMigAnAp9AmAr+gTAZjQKgJeCWpRKT09XhQoV/N9HR0fr3XffVVJSktq1a6d9+/a5PiAABII+AbAVfQJgMxoFwEtBLUrVqVNH69aty7WtePHieuedd1SnTh394x//cHW4vxo9erQuuOAClSpVSuXLlw/Z7QAonOgTAFt52SeJRgE4Nc6hAHgpqEWpzp07a8qUKXm250SrWbNmbs2Vx/Hjx3X99dfr3//+d8huA0DhRZ8A2MrLPkk0CsCpcQ4FwEvFg9l59OjROnr0aP4HKl5cc+bM0c6dO10Z7O8eeeQRSdLLL78ckuMDKNzoEwBbedkniUYBODXOoQB4KahnShUvXlyxsbGqU6eOfvjhhzyXR0VFKTEx0bXhACBQ9AmAregTAJvRKABeCuqZUpJUokQJZWZmFopPYMjMzFRmZqb/+4yMDA+nARBq9AmAregTAJvRKABeCeqZUjluv/12Pf744zpx4kSBbnzEiBHy+Xyn/Pr2228dH3/MmDEqV66c/6tWrVoFmheA/egTAFu51ScptI2iT0Bk4hwKgBeCfqaUJH3zzTdatGiR5s+fryZNmqh06dK5Lp8zZ05Ax7ntttvUo0ePU+6TlJTkZERJ0rBhwzRkyBD/9xkZGUQLKOLoEwBbudUnKbSNok9AZOIcCoAXHC1KlS9fXtdee22Bbzw+Pl7x8fEFPs7JREdHKzo6OmTHB2Af+gTAVm71SQpto+gTEJk4hwLghaAWpQ4fPqwyZcpoxowZoZrnpLZv364DBw5o+/btysrK0tq1ayVJ9erVU5kyZcI+DwC70CcAtvKyTxKNAnBqnEMB8FJQi1Lx8fFq166dunTpoquvvlrVq1cP1Vx5PPTQQ3rllVf83zdv3lyStGTJEl1yySVhmwOAnegTAFt52SeJRgE4Nc6hAHgpqDc637hxo6644grNnj1btWvX1jnnnKNHH31U69atC9V8fi+//LKMMXm+iBUAiT4BsJeXfZJoFIBT4xwKgJeCWpRKTEzU7bffroULF2rfvn0aMmSIvv/+e1188cWqXbu27rjjDi1evFhZWVmhmhcA8kWfANiKPgGwGY0C4KWgFqX+qly5crrxxhv15ptvav/+/Zo8ebKys7PVt29fVa5cWa+//rqbcwJAwOgTAFvRJwA2o1EAws3Rp+/lOUjx4urUqZM6deqk5557TmvWrNGJEyfcODQAFAh9AmAr+gTAZjQKQDgUaFHq6NGj2r59u44fP55re84b1AFFhe/E72perZhiD26Sdjt+gmG+Yg9uUvNqxeQ78burx4109AlFQSjbk4MGhR99QlEW6m7RrNCjUSiq6JOdHC1K/fLLL+rbt68++eSTfC/n9cYoamIOb9fqf5WRlv1LWubusRtKWv2vMko9vF3SBe4ePALRJxQloWxPDhoUPvQJkSDU3aJZoUOjUNTRJzs5WpS688479dtvv2nFihVq166d5s6dq59//lmjRo3SU0895faMgOd+L3OGWkw+rNdff10Nk5NdPXbqhg26+eabNf2KM1w9bqSiTyhKQtmeHDQofOgTIkGou0WzQodGoaijT3ZytCi1ePFivf/++zrnnHNUrFgxJSYmqmPHjipbtqzGjBmjK6+80u05AU+Z4jFaszdbx8o3kKo3c/XYx/Zma83ebJniMa4eN1LRJxQloWxPDhoUPvQJkSDU3aJZoUOjUNTRJzs5eiHlkSNHVKVKFUlSxYoV9csvv0iSmjRpotWrV7s3HQAEiT4BsBV9AmAzGgXAC44Wpc4880xt3LhRktSsWTNNnjxZu3bt0osvvqiEhARXBwSAYNAnALaiTwBsRqMAeMHxe0rt2bNHkvTwww/rsssu0+uvv66SJUvq5ZdfdnM+AAgKfQJgK/oEwGY0CoAXHC1K3Xzzzf4/N2/eXGlpadqwYYPOOOMMxcfHuzYcAASLPgGwFX0CYDMaBcALjhal/q5UqVJq0aKFG4cCAFfRJwC2ok8AbEajAISDo/eUuu666zR27Ng828eNG6frr7++wEMBgFP0CYCt6BMAm9EoAF5wtCj1+eef5/uRoJdffrmWLVtW4KEAwCn6BMBW9AmAzWgUAC84WpQ6fPiwSpYsmWd7iRIllJGRUeChAMAp+gTAVvQJgM1oFAAvOFqUaty4sd56660829988001atSowEMBgFP0CYCt6BMAm9EoAF5w9EbnDz74oK699lpt2bJF7du3lyQtWrRIs2bN0jvvvOPqgAAQDPoEwFb0CYDNaBQALzhalOrSpYvee+89PfbYY3r33XcVGxurs88+WwsXLlTbtm3dnhEAAkafANiKPgGwGY0C4AVHi1KSdOWVV+b7RngA4DX6BMBW9AmAzWgUgHBz9J5SknTw4EFNmzZN//nPf3TgwAFJ0urVq7Vr1y7XhgMAJ+gTAFvRJwA2o1EAws3RM6XWrVunDh06qFy5ckpLS1P//v1VsWJFzZ07Vz/99JNeffVVt+cEgIDQJwC2ok8AbEajAHjB0TOlhgwZoj59+mjz5s2KiYnxb+/cubOWLVvm2nAAECz6BMBW9AmAzWgUAC84WpRatWqV/vWvf+XZXqNGDe3du7fAQwGAU/QJgK3oEwCb0SgAXnC0KBUTE6OMjIw82zdu3KjKlSsXeCgAcIo+AbAVfQJgMxoFwAuOFqWuvvpqjRw5Un/88Yckyefzafv27Ro6dKiuvfZaVwcEgGDQJwC2ok8AbEajAHjB0aLUk08+qV9++UVVqlTRsWPH1LZtW9WrV09xcXEaPXq02zMCQMDoEwBb0ScANqNRALzg6NP3ypYtqy+//FKLFy/W6tWrlZ2drRYtWqhDhw5uzwcAQaFPAGxFnwDYjEYB8IKjRakc7du3V/v27d2aBQBcQ58A2Io+AbAZjQIQTo4XpRYtWqRFixZp3759ys7OznXZSy+9VODBAMAp+gTAVvQJgM1oFIBwc7Qo9cgjj2jkyJFq1aqVEhIS5PP53J4LAByhTwBsRZ8A2IxGAfCCo0WpF198US+//LJ69erl9jwAUCD0CYCt6BMAm9EoAF5w9Ol7x48f1wUXXOD2LABQYPQJgK3oEwCb0SgAXnC0KNW/f3+98cYbbs8CAAVGnwDYij4BsBmNAuAFRy/f+/333zVlyhQtXLhQZ599tkqUKJHr8qefftqV4QAgWPQJgK3oEwCb0SgAXnC0KLVu3To1a9ZMkrR+/fpcl/GGeAC8RJ8A2Io+AbAZjQLgBUeLUkuWLHF7DgBwBX0CYCv6BMBmNAqAFxy9p1SOH3/8UZ999pmOHTsmSTLGuDIUABQUfQJgK/oEwGY0CkA4OVqU+vXXX3XppZeqQYMGuuKKK7Rnzx5Jf7453t133+3qgAAQDPoEwFb0CYDNaBQALzhalLrrrrtUokQJbd++XaVKlfJvv+GGG/Tpp5+6NhwABIs+AbAVfQJgMxoFwAuO3lNq/vz5+uyzz1SzZs1c2+vXr6+ffvrJlcEAwAn6BMBW9AmAzWgUAC84eqbUkSNHcq2e59i/f7+io6MLPBQAOEWfANiKPgGwGY0C4AVHi1IXX3yxXn31Vf/3Pp9P2dnZGjdunNq1a+facAAQLPoEwFb0CYDNaBQALzh6+d64ceN0ySWX6Ntvv9Xx48d133336fvvv9eBAwf01VdfuT2j0tLS9Oijj2rx4sXau3evqlevrp49e2r48OEqWbKk67cHoPCiTwBsFe4+STQKQOA4hwLgBUeLUo0aNdJ///tfvfjii4qKitKRI0d0zTXX6NZbb1VCQoLbM2rDhg3Kzs7W5MmTVa9ePa1fv14DBgzQkSNH9OSTT7p+ewAKL/oEwFbh7pNEowAEjnMoAF5wtCglSQkJCXrkkUfcnOWkLr/8cl1++eX+7+vUqaONGzdq0qRJBAtAHvQJgK3C2SeJRgEIDudQAMLN0XtK1alTR3379lVmZmau7fv371edOnVcGex00tPTVbFixbDcFoDCgz4BsJUNfZJoFID82dAo+gREHkeLUmlpafrqq6900UUXac+ePf7tWVlZYfm40C1btui5557TwIEDT7lfZmamMjIycn0BKNroEwBbed0nKbBG0ScgMnndKM6hgMjkaFHK5/Pp008/Vc2aNdWqVSutWrXK0Y2PGDFCPp/vlF/ffvttruvs3r1bl19+ua6//nr179//lMcfM2aMypUr5/+qVauWozkBFB70CYCt3OqTFNpG0ScgMnEOBcALjt5TyhijMmXKaM6cORo2bJjatm2rKVOmqGPHjkEd57bbblOPHj1OuU9SUpL/z7t371a7du3UunVrTZky5bTHHzZsmIYMGeL/PiMjg2gBRRx9AmArt/okhbZR9AmITJxDAfCCo0Upn8/n//OYMWN01llnacCAAbrxxhuDOk58fLzi4+MD2nfXrl1q166dWrZsqRkzZqhYsdM/ySs6OlrR0dFBzQSgcKNPAGzlVp+k0DaKPgGRiXMoAF5w/Eypv+rZs6fq1q2rbt26uTLU3+3evVuXXHKJzjjjDD355JP65Zdf/JdVq1YtJLcJoHCiTwBsFe4+STQKQOA4hwLgBUeLUtnZ2Xm2tW7dWv/973+1YcOGAg/1d/Pnz9ePP/6oH3/8UTVr1sx12d/jCSCy0ScAtgp3nyQaBSBwnEMB8IKjNzo/mapVq6pt27ZuHlKS1KdPHxlj8v0CgEDQJwC2ClWfJBoFoOA4hwIQSo6eKSVJ7777rt5++21t375dx48fz3XZ6tWrCzwYADhFnwDYij4BsBmNAhBujp4pNWHCBPXt21dVqlTRmjVrdO6556pSpUraunWrOnfu7PaMABAw+gTAVvQJgM1oFAAvOFqUeuGFFzRlyhRNnDhRJUuW1H333acFCxZo8ODBSk9Pd3tGAAgYfQJgK/oEwGY0CoAXHC1Kbd++XRdccIEkKTY2VocOHZIk9erVS7NmzXJvOgAIEn0CYCv6BMBmNAqAFxwtSlWrVk2//vqrJCkxMVErVqyQJG3bto03pgPgKfoEwFb0CYDNaBQALzhalGrfvr0+/PBDSVK/fv101113qWPHjrrhhhvUrVs3VwcEgGDQJwC2ok8AbEajAHjB0afvTZkyRdnZ2ZKkgQMHqmLFivryyy911VVXaeDAga4OCADBoE8AbEWfANiMRgHwQtCLUidOnNDo0aP1z3/+U7Vq1ZIkde/eXd27d3d9OAAIBn0CYCv6BMBmNAqAV4J++V7x4sU1btw4ZWVlhWIeAHCMPgGwFX0CYDMaBcArjt5TqkOHDlq6dKnLowBAwdEnALaiTwBsRqMAeMHRe0p17txZw4YN0/r169WyZUuVLl061+VdunRxZTgACBZ9AmAr+gTAZjQKgBccLUr9+9//liQ9/fTTeS7z+Xw87ROAZ+gTAFvRJwA2o1EAvOBoUSrnUxkAwDb0CYCt6BMAm9EoAF5w9J5Sr776qjIzM/NsP378uF599dUCDwUATtEnALaiTwBsRqMAeMHRolTfvn2Vnp6eZ/uhQ4fUt2/fAg8FAE7RJwC2ok8AbEajAHjB0aKUMUY+ny/P9p07d6pcuXIFHgoAnKJPAGxFnwDYjEYB8EJQ7ynVvHlz+Xw++Xw+XXrppSpe/H9Xz8rK0rZt23T55Ze7PiQAnA59AmAr+gTAZjQKgJeCWpTq2rWrJGnt2rW67LLLVKZMGf9lJUuWVFJSkq699lpXBwSAQNAnALaiTwBsRqMAeCmoRamHH35YkpSUlKQePXooOjo6JEMBQLDoEwBb0ScANqNRALzk6D2l2rdvr19++cX//cqVK3XnnXdqypQprg0GAE7QJwC2ok8AbEajAHjB0aLUTTfdpCVLlkiS9u7dqw4dOmjlypX6z3/+o5EjR7o6IAAEgz4BsBV9AmAzGgXAC44WpdavX69zzz1XkvT222+rSZMmWr58ud544w29/PLLbs4HAEGhTwBsRZ8A2IxGAfCCo0WpP/74w/9a44ULF6pLly6SpOTkZO3Zs8e96QAgSPQJgK3oEwCb0SgAXnC0KHXWWWfpxRdf1BdffKEFCxb4PyJ09+7dqlSpkqsDAkAw6BMAW9EnADajUQC84GhR6vHHH9fkyZN1ySWX6MYbb1TTpk0lSR988IH/KZ8A4AX6BMBW9AmAzWgUAC8Ud3KlSy65RPv371dGRoYqVKjg3/5///d/KlWqlGvDAUCw6BMAW9EnADajUQC84GhRSpKioqJyxUqSkpKSCjoPABQYfQJgK/oEwGY0CkC4OV6Uevfdd/X2229r+/btOn78eK7LVq9eXeDBAMAp+gTAVvQJgM1oFIBwc/SeUhMmTFDfvn1VpUoVrVmzRueee64qVaqkrVu3qnPnzm7PCAABo08AbEWfANiMRgHwgqNFqRdeeEFTpkzRxIkTVbJkSd13331asGCBBg8erPT0dLdnBICA0ScAtqJPAGxGowB4wdGi1Pbt23XBBRdIkmJjY3Xo0CFJUq9evTRr1iz3pgOAINEnALaiTwBsRqMAeMHRolS1atX066+/SpISExO1YsUKSdK2bdtkjHFvOgAIEn0CYCv6BMBmNAqAFxwtSrVv314ffvihJKlfv36666671LFjR91www3q1q2bqwMCQDDoEwBb0ScANqNRALzg6NP3pkyZouzsbEnSwIEDVbFiRX355Ze66qqrNHDgQFcHBIBg0CcAtqJPAGxGowB4wdGiVLFixVSs2P+eZNW9e3d17949z36DBg3SyJEjFR8f73xCAAgCfQJgK/oEwGY0CoAXHL18L1AzZ85URkZGKG8CAByhTwBsRZ8A2IxGAXBTSBeleEM8ALaiTwBsRZ8A2IxGAXBTSBelAAAAAAAAgPywKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMIupItSPXv2VNmyZUN5EwDgCH0CYCv6BMBmNAqAmxwtSiUlJWnkyJHavn37KfebNGmS4uPjHQ0GAE7QJwC2ok8AbEajAHjB0aLU3Xffrffff1916tRRx44d9eabbyozM9Pt2XLp0qWLzjjjDMXExCghIUG9evXS7t27Q3qbAAof+gTAVl70SaJRAALDORQALzhalLr99tuVkpKilJQUNWrUSIMHD1ZCQoJuu+02rV692u0ZJUnt2rXT22+/rY0bN2r27NnasmWLrrvuupDcFoDCiz4BsJUXfZJoFIDAcA4FwBPGBcePHzfPPPOMiY6ONsWKFTNnn322mT59usnOznbj8Pl6//33jc/nM8ePHw/4Ounp6UaSSU9PD9lcKJpSUlKMJJOSklKojl1YhPKxSZ9QmIWjDzTo9EL1+PSiT8YE3yj6hGCEuik0KzfOoWgUAkefwivQx2bxgixo/fHHH5o7d65mzJihBQsW6Pzzz1e/fv20e/duDR8+XAsXLtQbb7xRkJvI14EDB/T666/rggsuUIkSJU66X2ZmZq6nnGZkZLg+CwA70ScAtvKqT1JgjaJPQGTjHApAODlalFq9erVmzJihWbNmKSoqSr169dL48eOVnJzs36dTp066+OKLXRtUku6//35NnDhRR48e1fnnn6+PPvrolPuPGTNGjzzyiKszALAbfQJgK6/6JAXXKPoERCbOoQB4wdF7Sp1zzjnavHmzJk2apJ07d+rJJ5/MFStJatSokXr06HHK44wYMUI+n++UX99++61//3vvvVdr1qzR/PnzFRUVpVtuuUXGmJMef9iwYUpPT/d/7dixw8ndBVCI0CcAtnKrT1JoG0WfgMjEORQALzh6ptTWrVuVmJh4yn1Kly6tGTNmnHKf22677bRRS0pK8v85Pj5e8fHxatCggRo2bKhatWppxYoVat26db7XjY6OVnR09CmPD6BooU8AbOVWn6TQNoo+AZGJcygAXnC0KNWuXTutWrVKlSpVyrX94MGDatGihbZu3RrQcXIC5ETO6nk4PkoZQOFBnwDYyq0+STQKgPs4hwLgBUeLUmlpacrKysqzPTMzU7t27SrwUH+3cuVKrVy5UhdeeKEqVKigrVu36qGHHlLdunVPuoIOIDLRJwC2CnefJBoFIHCcQwHwQlCLUh988IH/z5999pnKlSvn/z4rK0uLFi3K9VRMt8TGxmrOnDl6+OGHdeTIESUkJOjyyy/Xm2++yVM3AUiiTwDs5VWfJBoF4PQ4hwLgpaAWpbp27SpJ8vl86t27d67LSpQooaSkJD311FOuDZejSZMmWrx4sevHBVB00CcAtvKqTxKNAnB6nEMB8FJQi1LZ2dmSpNq1a2vVqlWOXysMAG6jTwBsRZ8A2IxGAfCSo/eU2rZtm9tzAIAr6BMAW9EnADajUQC8EPCi1IQJE/R///d/iomJ0YQJE0657+DBgws8GAAEij4BsBV9AmAzGgXAawEvSo0fP14333yzYmJiNH78+JPu5/P5CBaAsKJPAGxFnwDYjEYB8FrAi1J/fTonT+0EYBP6BMBW9AmAzWgUAK8V83oAAAAAAAAARB5Hi1LXXXedxo4dm2f7uHHjdP311xd4KABwij4BsBV9AmAzGgXAC44WpT7//HNdeeWVebZffvnlWrZsWYGHAgCn6BMAW9EnADajUQC84GhR6vDhwypZsmSe7SVKlFBGRkaBhwIAp+gTAFvRJwA2o1EAvOBoUapx48Z666238mx/88031ahRowIPBQBO0ScAtqJPAGxGowB4IeBP3/urBx98UNdee622bNmi9u3bS5IWLVqkWbNm6Z133nF1QAAIBn0CYCv6BMBmNAqAFxwtSnXp0kXvvfeeHnvsMb377ruKjY3V2WefrYULF6pt27ZuzwgAAaNPAGxFnwDYjEYB8IKjRSlJuvLKK/N9IzwA8Bp9AmAr+gTAZjQKQLg5ek8pAAAAAAAAoCACfqZUxYoVtWnTJsXHx6tChQry+Xwn3ffAgQOuDAcAgaBPAGxFnwDYjEYB8FrAi1Ljx49XXFycJOmZZ54J1TwAEDT6BMBW9AmAzWgUAK8FvCjVu3fvfP8MAF6jTwBsRZ8A2IxGAfBawItSGRkZAR+0bNmyjoYBACfoEwBb0ScANqNRALwW8KJU+fLlT/kaY0kyxsjn8ykrK6vAgwFAoOgTAFvRJwA2o1EAvBbwotSSJUtCOQcAOEafANiKPgGwGY0C4LWAF6Xatm0byjkAwDH6BMBW9AmAzWgUAK8FvCi1bt06NW7cWMWKFdO6detOue/ZZ59d4MEAmxw9elSStHr16oD2P3bsmNLS0pSUlKTY2NhT7puamlrg+SIdfUJRFWx7pOD6I9GgUKNPiDShPGeSaJbbaBQiCX2yU8CLUs2aNdPevXtVpUoVNWvWTD6fT8aYPPvxemMURRs2bJAkDRgwIGS3kfNxvAgefUJRFY725KBBoUGfEGnC1S2a5Q4ahUhCn+wU8KLUtm3bVLlyZf+fgUjStWtXSVJycrJKlSp12v1TU1PVs2dPzZw5Uw0bNjzt/nFxcapfv35Bx4xY9AlFVbDtkYLvj0SDQok+IdKE+pxJolluolGIJPTJTgEvSiUmJub7ZyASxMfHq3///kFfr2HDhmrRokUIJsJf0ScUVU7bI9EfW9AnRBrOmQoXGoVIQp/sFPCi1N9t2rRJS5cu1b59+5SdnZ3rsoceeqjAgwGAU/QJgK3oEwCb0SgA4eZoUWrq1Kn697//rfj4eFWrVk0+n89/mc/nI1gAPEOfANiKPgGwGY0C4AVHi1KjRo3S6NGjdf/997s9DwAUCH0CYCv6BMBmNAqAF4o5udJvv/2m66+/3u1ZAKDA6BMAW9EnADajUQC84GhR6vrrr9f8+fPdngUACow+AbAVfQJgMxoFwAsBv3xvwoQJ/j/Xq1dPDz74oFasWKEmTZqoRIkSufYdPHiwexMCwGnQJwC2ok8AbEajAHgt4EWp8ePH5/q+TJky+vzzz/X555/n2u7z+QgWgLCiTwBsRZ8A2IxGAfBawItS27ZtC+UcAOAYfQJgK/oEwGY0CoDXHL2n1F8ZY2SMcWMWAHAVfQJgK/oEwGY0CkC4OF6Umj59uho3bqyYmBjFxMSocePGmjZtmpuzAYAj9AmAregTAJvRKADhFvDL9/7qwQcf1Pjx43X77berdevWkqSvv/5ad911l9LS0jRq1ChXhwSAQNEnALaiTwBsRqMAeMHRotSkSZM0depU3Xjjjf5tXbp00dlnn63bb7+dYAHwDH0CYCv6BMBmNAqAFxy9fC8rK0utWrXKs71ly5Y6ceJEgYcCAKfoEwBb0ScANqNRALzgaFGqZ8+emjRpUp7tU6ZM0c0331zgoQDAKfoEwFb0CYDNaBQALzh6+Z7055vgzZ8/X+eff74kacWKFdqxY4duueUWDRkyxL/f008/XfApASAI9AmAregTAJvRKADh5mhRav369WrRooUkacuWLZKkypUrq3Llylq/fr1/P5/P58KIABA4+gTAVvQJgM1oFAAvOFqUWrJkidtzAIAr6BMAW9EnADajUQC84Og9pf5q586d2rVrlxuzBCQzM1PNmjWTz+fT2rVrw3a7AAof+gTAVuHuk0SjAASOcygA4eJoUSo7O1sjR45UuXLllJiYqDPOOEPly5fXo48+quzsbLdnzOW+++5T9erVQ3obAAov+gTAVl72SaJRAE6NcygAXnD08r3hw4dr+vTpGjt2rNq0aSNjjL766iuNGDFCv//+u0aPHu32nJKkTz75RPPnz9fs2bP1ySefhOQ2ABRu9AmArbzqk0SjAJwe51AAvOBoUeqVV17RtGnT1KVLF/+2pk2bqkaNGho0aFBIgvXzzz9rwIABeu+991SqVKmArpOZmanMzEz/9xkZGa7PBcAu9AmArbzokxR8o+gTEJk4hwLgBUcv3ztw4ICSk5PzbE9OTtaBAwcKPNTfGWPUp08fDRw4UK1atQr4emPGjFG5cuX8X7Vq1XJ9NgB2oU8AbBXuPknOGkWfgMjEORQALzhalGratKkmTpyYZ/vEiRPVtGnTgI8zYsQI+Xy+U359++23eu6555SRkaFhw4YFNeewYcOUnp7u/9qxY0dQ1wdQ+NAnALZyq09SaBtFn4DIxDkUAC84evneE088oSuvvFILFy5U69at5fP5tHz5cu3YsUPz5s0L+Di33XabevToccp9kpKSNGrUKK1YsULR0dG5LmvVqpVuvvlmvfLKK/leNzo6Os91ABRt9AmArdzqkxTaRtEnIDJxDgXACz5jjHFyxd27d+v555/Xhg0bZIxRo0aNNGjQoJB8asL27dtzvVZ49+7duuyyy/Tuu+/qvPPOU82aNQM6TkZGhsqVK6f09HSVLVvW9TmBHKtXr1bLli2VkpKiFi1aeD2O9dx+bNInRDL64z43H5/h7JPkTqPoE0KJZhUM51A0CqFDnwom0Memo2dKSVL16tVD+ikxf3XGGWfk+r5MmTKSpLp16wYcKwCRgz4BsFU4+yTRKADB4RwKQLg5XpT67bffNH36dKWmpsrn86lhw4bq27evKlas6OZ8ABA0+gTAVvQJgM1oFIBwc/RG559//rlq166tCRMm6LffftOBAwc0YcIE1a5dW59//rnbM+aRlJQkY4yaNWsW8tsCULjQJwC28rpPEo0CcHJeN4o+AZHJ0TOlbr31VnXv3l2TJk1SVFSUJCkrK0uDBg3SrbfeqvXr17s6JAAEij4BsBV9AmAzGgXAC46eKbVlyxbdfffd/lhJUlRUlIYMGaItW7a4NhwABIs+AbAVfQJgMxoFwAuOFqVatGih1NTUPNtTU1N5uiUAT9EnALaiTwBsRqMAeMHRy/cGDx6sO+64Qz/++KPOP/98SdKKFSv0/PPPa+zYsVq3bp1/37PPPtudSQEgAPQJgK3oEwCb0SgAXnC0KHXjjTdKku677758L/P5fDLGyOfzKSsrq2ATAkAQ6BMAW9EnADajUQC84GhRatu2bW7PAQCuoE8AbEWfANiMRgHwgqNFqcTERLfnAABX0CcAtqJPAGxGowB4IeBFqQ8++ECdO3dWiRIl9MEHH5xy3y5duhR4MAAIFH0CYCv6BMBmNAqA1wJelOratav27t2rKlWqqGvXrifdj9cYAwg3+gTAVvQJgM1oFACvBbwolZ2dne+fAcBr9AmAregTAJvRKABeK+b1AAAAAAAAAIg8AT9TasKECQEfdPDgwY6GAQAn6BMAW9EnADajUQC8FvCi1Pjx4wPaz+fzESwAYUWfANiKPgGwGY0C4LWAF6W2bdsWyjkAwDH6BMBW9AmAzWgUAK+F9D2lypYtq61bt4byJgDAEfoEwFb0CYDNaBQAN4V0UcoYE8rDA4Bj9AmAregTAJvRKABu4tP3AAAAAAAAEHYsSgEAAAAAACDsWJQCAAAAAABA2IV0Ucrn84Xy8ADgGH0CYCv6BMBmNAqAm3ijcwARiT4BsBV9AmAzGgXATSFdlPrkk09Uo0aNUN4EADhCnwDYij4BsBmNAuCm4k6ulJWVpZdfflmLFi3Svn37lJ2dnevyxYsXS5IuvPDCgk8IAEGgTwBsRZ8A2IxGAfCCo0WpO+64Qy+//LKuvPJKNW7cmNcVA7AGfQJgK/oEwGY0CoAXHC1Kvfnmm3r77bd1xRVXuD0PABQIfQJgK/oEwGY0CoAXHL2nVMmSJVWvXj23ZwGAAqNPAGxFnwDYjEYB8IKjRam7775bzz77LJ+8AMA69AmAregTAJvRKABeCPjle9dcc02u7xcvXqxPPvlEZ511lkqUKJHrsjlz5rgzHQAEgD4BsBV9AmAzGgXAawEvSpUrVy7X9926dXN9GABwgj4BsBV9AmAzGgXAawEvSs2YMSOUcwCAY/QJgK3oEwCb0SgAXnP06Xs59u3bp40bN8rn86lBgwaqUqWKW3MBQIHQJwC2ok8AbEajAISTozc6z8jIUK9evVSjRg21bdtWF198sWrUqKGePXsqPT3d7RkBIGD0CYCt6BMAm9EoAF5wtCjVv39/ffPNN/roo4908OBBpaen66OPPtK3336rAQMGuD0jAASMPgGwFX0CYDMaBcALjl6+9/HHH+uzzz7ThRde6N922WWXaerUqbr88stdGw4AgkWfANiKPgGwGY0C4AVHz5SqVKlSnk9qkP789IYKFSoUeCgAcIo+AbAVfQJgMxoFwAuOFqUeeOABDRkyRHv27PFv27t3r+699149+OCDrg0HAMGiTwBsRZ8A2IxGAfCCo5fvTZo0ST/++KMSExN1xhlnSJK2b9+u6Oho/fLLL5o8ebJ/39WrV7szKQAEgD4BsBV9AmAzGgXAC44Wpbp27eryGADgDvoEwFb0CYDNaBQALzhalHr44YfdngMAXEGfANiKPgGwGY0C4AVH7ykFAAAAAAAAFISjZ0plZWVp/Pjxevvtt7V9+3YdP3481+UHDhxwZTgACBZ9AmAr+gTAZjQKgBccPVPqkUce0dNPP63u3bsrPT1dQ4YM0TXXXKNixYppxIgRLo8IAIGjTwBsRZ8A2IxGAfCCo0Wp119/XVOnTtU999yj4sWL68Ybb9S0adP00EMPacWKFW7PKElKSkqSz+fL9TV06NCQ3BaAwos+AbCVF32SaBSAwHAOBcALjl6+t3fvXjVp0kSSVKZMGaWnp0uS/vGPf+jBBx90b7q/GTlypAYMGOD/vkyZMiG7LQCFE30CYCuv+iTRKACnxzkUAC84eqZUzZo1tWfPHklSvXr1NH/+fEnSqlWrFB0d7d50fxMXF6dq1ar5vwgWgL+jTwBs5VWfJBoF4PQ4hwLgBUeLUt26ddOiRYskSXfccYcefPBB1a9fX7fccov++c9/ujrgXz3++OOqVKmSmjVrptGjR+d58z0AoE8AbOVVnyQaBeD0OIcC4AVHL98bO3as/8/XXXedatWqpa+++kr16tVTly5dXBvur+644w61aNFCFSpU0MqVKzVs2DBt27ZN06ZNO+l1MjMzlZmZ6f8+IyMjJLMBsAd9AmArL/okBd8o+gREJs6hAHjCBOn48eOmT58+ZsuWLcFeNY+HH37YSDrl16pVq/K97rvvvmskmf379wd9/PT09ALPDpxKSkqKkWRSUlK8HqVQSE9Pd+WxSZ8A+hMKbjTKzT4ZE9pG0SeEE80qGM6haBRChz4VTKB98hljTLALWeXLl9fq1atVp06dYK+ay/79+7V///5T7pOUlKSYmJg823ft2qWaNWtqxYoVOu+88/K9bn6r6LVq1VJ6errKli1boNmBU1m9erVatmyplJQUtWjRwutxrJeRkaFy5cq58tikT4h09Md9bjXKrT5JoW0UfUI40ayC4RyKRiF06FPBBNonRy/f69atm9577z0NGTLE8YCSFB8fr/j4eEfXXbNmjSQpISHhpPtER0eH/I1DAdiFPgGwlVt9kkLbKPoERCbOoQB4wdGiVL169fToo49q+fLlatmypUqXLp3r8sGDB7syXI6vv/5aK1asULt27VSuXDmtWrVKd911l7p06aIzzjjD1dsCULjRJwC2CnefJBoFIHCcQwHwgqNFqWnTpql8+fJKSUlRSkpKrst8Pp/rwYqOjtZbb72lRx55RJmZmUpMTNSAAQN03333uXo7QLCOHj2qDRs25Nmempqa659/lZycrFKlSoV8tkhFnxAJTtYe6dT9kWiQl8LdJ4lGwR5OzpkkmhVOnEMhUtEnbzl6T6nCys3XXAPS/15nHAxek5wXj01+BgiOk/bkoEHBi/THZ6Tff7jDabdo1qnx+ORngIKjT6ER0veUOtnrjH0+n2JiYlSvXj1dffXVqlixopPDA4VGcnJynv+TJEnHjh1TWlqakpKSFBsbm+c6CB36hEhwsvZIp+5PznXhDfqESObknCnneggPGoVIRZ+85eiZUu3atdPq1auVlZWlM888U8YYbd68WVFRUUpOTtbGjRvl8/n05ZdfqlGjRqGY2xFW0QE7ufnYpE8A3ObW45M+AXAb51A0CrBVoI/NYk4OfvXVV6tDhw7avXu3UlJStHr1au3atUsdO3bUjTfeqF27duniiy/WXXfd5fgOAIAT9AmAregTAJvRKABecPRMqRo1amjBggV5Vsi///57derUSbt27dLq1avVqVMn7d+/37VhC4pVdMBObj426RMAt7n1+KRPANzGORSNAmwV0mdKpaena9++fXm2//LLL8rIyJAklS9fXsePH3dyeABwjD4BsBV9AmAzGgXAC45fvvfPf/5Tc+fO1c6dO7Vr1y7NnTtX/fr1U9euXSVJK1euVIMGDdycFQBOiz4BsBV9AmAzGgXAC45evnf48GHdddddevXVV3XixAlJUvHixdW7d2+NHz9epUuX1tq1ayVJzZo1c3PeAuGpnYCd3Hxs0icAbnPr8UmfALiNcygaBdgq0Memo0WpHIcPH9bWrVtljFHdunVVpkwZp4cKC4IF2CkUj036BMAtbj8+6RMAt3AORaMAWwX62CxekBspU6aMzj777IIcAgBCgj4BsBV9AmAzGgUgnBy9pxQAAAAAAABQECxKAQAAAAAAIOxYlAIAAAAAAEDYsSgFAAAAAACAsCvQG50XNjkfNJiRkeHxJAD+KucxWYAPAy306BNgr0hvFH0C7BXpfZJoFGCrQPsUUYtShw4dkiTVqlXL40kA5OfQoUMqV66c12N4gj4B9ovURtEnwH6R2ieJRgG2O12ffCaCltWzs7O1e/duxcXFyefzeT1OoZCRkaFatWppx44dKlu2rNfjFBr83IJjjNGhQ4dUvXp1FSsWma8qpk/ui9THYaTe71CK9EbRp4LhMXlq/HwKJtL7JNGoguIxeHL8bAom0D5F1DOlihUrppo1a3o9RqFUtmxZHogO8HMLXKT+370c9Cl0IvVxGKn3O1QiuVH0yR08Jk+Nn49zkdwniUa5hcfgyfGzcS6QPkXmcjoAAAAAAAA8xaIUAAAAAAAAwo5FKZxSdHS0Hn74YUVHR3s9SqHCzw3wXqQ+DiP1fgO24jF5avx8AG/xGDw5fjbhEVFvdA4AAAAAAAA78EwpAAAAAAAAhB2LUgAAAAAAAAg7FqUAAAAAAAAQdixKFXEvvPCCateurZiYGLVs2VJffPHFSfedM2eOOnbsqMqVK6ts2bJq3bq1Pvvsszz7tGrVSuXLl1fp0qXVrFkzvfbaa3mOtWvXLvXs2VOVKlVSqVKl1KxZM6WkpLh+/0IlmJ9bnz595PP58nydddZZ+e7/5ptvyufzqWvXrrm2jxgxIs8xqlWr5ubdAgqVYB6He/bs0U033aQzzzxTxYoV05133plnnz/++EMjR45U3bp1FRMTo6ZNm+rTTz/Ntc+JEyf0wAMPqHbt2oqNjVWdOnU0cuRIZWdnu333TiqY+7106dJ8+7Nhw4Zc+x08eFC33nqrEhISFBMTo4YNG2revHn+y5ctW6arrrpK1atXl8/n03vvvRequwcUOsE8Jr/88ku1adNGlSpVUmxsrJKTkzV+/Phc+3z//fe69tprlZSUJJ/Pp2eeeSbPcQrTYzIU50ynaxbnTMD/uP373l+d7PcWG86XAhHMz0aSMjMzNXz4cCUmJio6Olp169bVSy+95L88kHPJwtRvW7AoVYS99dZbuvPOOzV8+HCtWbNGF110kTp37qzt27fnu/+yZcvUsWNHzZs3TykpKWrXrp2uuuoqrVmzxr9PxYoVNXz4cH399ddat26d+vbtq759++aK2W+//aY2bdqoRIkS+uSTT/TDDz/oqaeeUvny5UN9l10R7M/t2Wef1Z49e/xfO3bsUMWKFXX99dfn2fenn37SPffco4suuijfY5111lm5jvXdd9+5et+AwiLYx2FmZqYqV66s4cOHq2nTpvnu88ADD2jy5Ml67rnn9MMPP2jgwIHq1q1brsY9/vjjevHFFzVx4kSlpqbqiSee0Lhx4/Tcc8+F5H7+XbD3O8fGjRtztaN+/fr+y44fP66OHTsqLS1N7777rjZu3KipU6eqRo0a/n2OHDmipk2bauLEiSG7b0BhFOxjsnTp0rrtttu0bNkypaam6oEHHtADDzygKVOm+Pc5evSo6tSpo7Fjx550IaWwPCZDcc4USLMkzpkAKTS/7+U41e8tXp8vBcLJOVX37t21aNEiTZ8+XRs3btSsWbOUnJzsvzyQc8nC0m+rGBRZ5557rhk4cGCubcnJyWbo0KEBH6NRo0bmkUceOeU+zZs3Nw888ID/+/vvv99ceOGFwQ1rkYL+3ObOnWt8Pp9JS0vLtf3EiROmTZs2Ztq0aaZ3797m6quvznX5ww8/bJo2bVqQ0YEioyCPw7Zt25o77rgjz/aEhAQzceLEXNuuvvpqc/PNN/u/v/LKK80///nPXPtcc801pmfPnkFM71yw93vJkiVGkvntt99OesxJkyaZOnXqmOPHjwc0gyQzd+7cQEcGijQ3zqW6det20oYkJiaa8ePHn/L6Nj8mQ3HOFEizOGcC/hSq3/dO93uL1+dLgQj2Z/PJJ5+YcuXKmV9//fWkxwzkXPKvbO63TXimVBF1/PhxpaSkqFOnTrm2d+rUScuXLw/oGNnZ2Tp06JAqVqyY7+XGGC1atEgbN27UxRdf7N/+wQcfqFWrVrr++utVpUoVNW/eXFOnTnV+Z8LIjZ/b9OnT1aFDByUmJubaPnLkSFWuXFn9+vU76XU3b96s6tWrq3bt2urRo4e2bt0a/J0ACjk3Hof5yczMVExMTK5tsbGx+vLLL/3fX3jhhVq0aJE2bdokSfrvf/+rL7/8UldccYXj2w1UQe538+bNlZCQoEsvvVRLlizJddkHH3yg1q1b69Zbb1XVqlXVuHFjPfbYY8rKynL9PgBFiRstWrNmjZYvX662bduGYkRPheqcKdBmcc6ESBfK3/dO93uLl+dLgXDys8n5HfaJJ55QjRo11KBBA91zzz06duyYf59AziURvOJeD4DQ2L9/v7KyslS1atVc26tWraq9e/cGdIynnnpKR44cUffu3XNtT09PV40aNZSZmamoqCi98MIL6tixo//yrVu3atKkSRoyZIj+85//aOXKlRo8eLCio6N1yy23FPzOhVBBf2579uzRJ598ojfeeCPX9q+++krTp0/X2rVrT3rd8847T6+++qoaNGign3/+WaNGjdIFF1yg77//XpUqVXJ0f4DCyI1+5eeyyy7T008/rYsvvlh169bVokWL9P777+f6Ref+++9Xenq6kpOTFRUVpaysLI0ePVo33nij49sNlJP7nZCQoClTpqhly5bKzMzUa6+9pksvvVRLly71/8+CrVu3avHixbr55ps1b948bd68WbfeeqtOnDihhx56KOT3CyisCtKimjVr6pdfftGJEyc0YsQI9e/fP5SjeiJU50yBNItzJiB0v+8F8nuLl+dLgXDys9m6dau+/PJLxcTEaO7cudq/f78GDRqkAwcO+N9XKpBzSQSPRakizufz5freGJNnW35mzZqlESNG6P3331eVKlVyXRYXF6e1a9fq8OHDWrRokYYMGaI6derokksukfTninurVq302GOPSfrz/+B///33mjRpkvWLUjmc/txefvlllS9fPtebAR46dEg9e/bU1KlTFR8ff9Lrdu7c2f/nJk2aqHXr1qpbt65eeeUVDRkyJPg7ARRyTh+HJ/Pss89qwIABSk5Ols/nU926ddW3b1/NmDHDv89bb72lmTNn6o033tBZZ52ltWvX6s4771T16tXVu3dvx7cdjGDu95lnnqkzzzzT/33r1q21Y8cOPfnkk/5FqezsbFWpUkVTpkxRVFSUWrZsqd27d2vcuHEsSgEBcNKiL774QocPH9aKFSs0dOhQ1atXz5pf1tzm5jmTFFizOGcC/sfN3/cC/b3FhvOlQATzs8nOzpbP59Prr7+ucuXKSZKefvppXXfddXr++ecVGxsb0LkkgseiVBEVHx+vqKioPCvB+/bty7Ni/HdvvfWW+vXrp3feeUcdOnTIc3mxYsVUr149SVKzZs2UmpqqMWPG+BelEhIS1KhRo1zXadiwoWbPnl2AexQeBfm5GWP00ksvqVevXipZsqR/+5YtW5SWlqarrrrKvy3nkymKFy+ujRs3qm7dunmOV7p0aTVp0kSbN28uyF0CCp2CPA5PpXLlynrvvff0+++/69dff1X16tU1dOhQ1a5d27/Pvffeq6FDh6pHjx6S/vxl56efftKYMWNCfpLl1v0+//zzNXPmTP/3CQkJKlGihKKiovzbGjZsqL179+r48eO5egXgfwrymMzpSpMmTfTzzz9rxIgRRW5RKhTnTJKzZnHOhEgUit/3Av29xcvzpUA4+dkkJCSoRo0a/gUp6c/2GGO0c+dO1a9fP6BzSQSP95QqokqWLKmWLVtqwYIFubYvWLBAF1xwwUmvN2vWLPXp00dvvPGGrrzyyoBuyxijzMxM//dt2rTRxo0bc+2zadOmPO+xZCOnPzdJ+vzzz/Xjjz/mee11cnKyvvvuO61du9b/1aVLF7Vr105r165VrVq18j1eZmamUlNTlZCQULA7BRQyBXkcBiImJkY1atTQiRMnNHv2bF199dX+y44ePapixXL/pzEqKiosH3Hs1v1es2ZNrm60adNGP/74Y677sGnTJiUkJLAgBZyCW4/Jv58nFRWhOGeSnDWLcyZEolD8vhfo7y1eni8FwsnPpk2bNtq9e7cOHz7s37Zp0yYVK1ZMNWvWzLXvqc4l4YAX766O8HjzzTdNiRIlzPTp080PP/xg7rzzTlO6dGn/J5wMHTrU9OrVy7//G2+8YYoXL26ef/55s2fPHv/XwYMH/fs89thjZv78+WbLli0mNTXVPPXUU6Z48eJm6tSp/n1WrlxpihcvbkaPHm02b95sXn/9dVOqVCkzc+bM8N35Agj255ajZ8+e5rzzzgvoNvL7FIu7777bLF261GzdutWsWLHC/OMf/zBxcXF5PsUPiAROHodr1qwxa9asMS1btjQ33XSTWbNmjfn+++/9l69YscLMnj3bbNmyxSxbtsy0b9/e1K5dO9cn1/Xu3dvUqFHDfPTRR2bbtm1mzpw5Jj4+3tx3331W3u/x48ebuXPnmk2bNpn169eboUOHGklm9uzZ/n22b99uypQpY2677TazceNG89FHH5kqVaqYUaNG+fc5dOiQ/+cnyTz99NNmzZo15qeffgrL/QZsFexjcuLEieaDDz4wmzZtMps2bTIvvfSSKVu2rBk+fLh/n8zMTP/jLSEhwdxzzz1mzZo1ZvPmzf59CstjMhTnTIE0i3Mm4E+h+H3v7/L7vcXr86VABPuzOXTokKlZs6a57rrrzPfff28+//xzU79+fdO/f3//PoGcSxaWftuERaki7vnnnzeJiYmmZMmSpkWLFubzzz/3X9a7d2/Ttm1b//dt27Y1kvJ89e7d27/P8OHDTb169UxMTIypUKGCad26tXnzzTfz3O6HH35oGjdubKKjo01ycrKZMmVKKO+m64L5uRljzMGDB01sbGzA9zO/uN9www0mISHBlChRwlSvXt1cc801uX6hBiJNsI/D/PqVmJjov3zp0qWmYcOGJjo62lSqVMn06tXL7Nq1K9cxMjIyzB133GHOOOMMExMTY+rUqWOGDx9uMjMzQ3lXcwnmfj/++OOmbt26/iZfeOGF5uOPP85zzOXLl5vzzjvPREdHmzp16pjRo0ebEydO+C9fsmTJafsPRKpgHpMTJkwwZ511lilVqpQpW7asad68uXnhhRdMVlaWf59t27bl+3j763EK02MyFOdMp2sW50zA/7j9+97f5fd7iw3nS4EItk+pqammQ4cOJjY21tSsWdMMGTLEHD161H95IOeShanftvAZY0wIn4gFAAAAAAAA5MF7SgEAAAAAACDsWJQCAAAAAABA2LEoBQAAAAAAgLBjUQoAAAAAAABhx6IUAAAAAAAAwo5FKQAAAAAAAIQdi1IAAAAAAAAIOxalAAAAAAAAEHYsSiFgl1xyie68886A909LS5PP59PatWtDNpMkLV26VD6fTwcPHgzp7QCwBz0CUNjQLTtE2v0FCoJu2cXn8+m9996TFL6fdTiwKBVmffr0kc/nk8/nU4kSJVSnTh3dc889OnLkSEhuq2vXrq4db86cOXr00UcD3r9WrVras2ePGjdu7NoMKLhFixbpggsuUFxcnBISEnT//ffrxIkTXo8FD9AjBMLtv7scY8aM0TnnnKO4uDhVqVJFXbt21caNG3PtM2fOHF122WWKj48vMideKBi6hYK64IILtGfPHpUrV87rURAh6BZwaixKeeDyyy/Xnj17tHXrVo0aNUovvPCC7rnnnnz3/eOPP0I+T6C3UbFiRcXFxQV83KioKFWrVk3Fixd3Ohpctm7dOl1xxRW6/PLLtWbNGr355pv64IMPNHToUK9Hg0foEbzy+eef69Zbb9WKFSu0YMECnThxQp06dcp1kn7kyBG1adNGY8eO9XBS2IZuoSBKliypatWqyefzeT0KIgjdAk7BIKx69+5trr766lzb+vfvb6pVq2aMMebhhx82TZs2NdOnTze1a9c2Pp/PZGdnm4MHD5oBAwaYypUrm7i4ONOuXTuzdu3ak97Oww8/bCTl+lqyZInZtm2bkWTeeust07ZtWxMdHW1eeukls3//ftOjRw9To0YNExsbaxo3bmzeeOONXMds27atueOOO/zfJyYmmtGjR5u+ffuaMmXKmFq1apnJkyf7L8+5rTVr1hhjjFmyZImRZBYuXGhatmxpYmNjTevWrc2GDRty3c6jjz5qKleubMqUKWP69etn7r//ftO0adOT3tec4/7222/+be+++65p1KiRKVmypElMTDRPPvlkrus8//zzpl69eiY6OtpUqVLFXHvttf7L3nnnHdO4cWMTExNjKlasaC699FJz+PDhk97+0qVLzTnnnGNKlixpqlWrZu6//37zxx9/5Pq53X777ebee+81FSpUMFWrVjUPP/zwSY9njDEnTpwwd911lylXrpypWLGiuffee80tt9yS69+dtm3bmttuu83ccccdpnz58qZKlSpm8uTJ5vDhw6ZPnz6mTJkypk6dOmbevHn+6wwbNsy0atUq123NnTvXxMTEmIyMjFPOhKKHHtGj0/XoZH93xhizbt06065dO/9sAwYMMIcOHfL/HEqUKGGWLVvmP9aTTz5pKlWqZHbv3p3vbe3bt89IMp9//nmey/7+94fIRbfoViDnUX//u5NkEhMT872/M2bMMOXKlTOffvqpSU5ONqVLlzaXXXZZnlZNnz7d/zOpVq2aufXWW085A5CDbtGtQLq1cuVK06FDB1OpUiVTtmxZc/HFF5uUlJRc+0gyc+fONcYUrXMjFqXCLL8o3X777aZSpUrGmD9jkvMfw9WrV5v//ve/Jjs727Rp08ZcddVVZtWqVWbTpk3m7rvvNpUqVTK//vprvrdz6NAh0717d3P55ZebPXv2mD179pjMzEz/v7xJSUlm9uzZZuvWrWbXrl1m586dZty4cWbNmjVmy5YtZsKECSYqKsqsWLHCf8z8olSxYkXz/PPPm82bN5sxY8aYYsWKmdTUVGPMyaN03nnnmaVLl5rvv//eXHTRReaCCy7wH3PmzJkmJibGvPTSS2bjxo3mkUceMWXLlg0qSt9++60pVqyYGTlypNm4caOZMWOGiY2NNTNmzDDGGLNq1SoTFRVl3njjDZOWlmZWr15tnn32WWOMMbt37zbFixc3Tz/9tNm2bZtZt26def755/2/aP3dzp07TalSpcygQYNMamqqmTt3romPj88VnbZt25qyZcuaESNGmE2bNplXXnnF+Hw+M3/+/JPep8cff9yUK1fOvPvuu+aHH34w/fr1M3FxcXkWpeLi4syjjz5qNm3aZB599FFTrFgx07lzZzNlyhSzadMm8+9//9tUqlTJHDlyxBhjzJAhQ8yFF16Y67Y+/fTTXL9oInLQI3p0uh6d7O/uyJEjpnr16uaaa64x3333nVm0aJGpXbu26d27t/+69957r0lMTDQHDx40a9euNdHR0WbOnDkn/dlt3rzZSDLfffddnsuK0okXCoZu0a1AzqNy/s727NljfvzxR1OvXj3Tq1evfO/vjBkzTIkSJUyHDh3MqlWrTEpKimnYsKG56aab/Md74YUXTExMjHnmmWfMxo0bzcqVK8348eNPevvAX9EtuhVItxYtWmRee+0188MPP/h//6tatWquJw6wKAVX/D1K33zzjalUqZLp3r27MebPKJUoUcLs27fPv8+iRYtM2bJlze+//57rWHXr1s21Mn262zLmf//yPvPMM6ed9YorrjB33323//v8otSzZ0//99nZ2aZKlSpm0qRJuW4rv5XyHB9//LGRZI4dO2aMMea8887L83+e2rRpE1SUbrrpJtOxY8dc+9x7772mUaNGxhhjZs+ebcqWLZvvM4NSUlKMJJOWlnbS2/ur//znP+bMM8802dnZ/m3PP/+8KVOmjMnKyjLG/Plz+/tC0DnnnGPuv//+kx43ISHBjB071v/9H3/8YWrWrJlnUeqvxz1x4oQpXbq0/6TLmD9PyiSZr7/+2hhjzGeffWaKFStm3njjDXPixAmzc+dOc+GFFxpJef7PCIo+ekSPjDl9j/L7u5syZYqpUKFCrv+L+PHHH5tixYqZvXv3GmOMyczMNM2bNzfdu3c3Z511lunfv/9JbyM7O9tcddVVeWbLUZROvFAwdItuGXP6buXIzs423bp1My1btjRHjx7N9/7OmDHDSDI//vhjrhmqVq3q/7569epm+PDhAd0n4O/oFt0yJvBu5Thx4oSJi4szH374oX9bUV2U4j2lPPDRRx+pTJkyiomJUevWrXXxxRfrueee81+emJioypUr+79PSUnR4cOHValSJZUpU8b/tW3bNm3ZskXbt2/Ptf2xxx477QytWrXK9X1WVpZGjx6ts88+23878+fP1/bt2095nLPPPtv/Z5/Pp2rVqmnfvn0BXychIUGS/NfZuHGjzj333Fz7//3700lNTVWbNm1ybWvTpo02b96srKwsdezYUYmJiapTp4569eql119/XUePHpUkNW3aVJdeeqmaNGmi66+/XlOnTtVvv/12yttq3bp1rvclaNOmjQ4fPqydO3fme59z7vfJfk7p6enas2ePWrdu7d9WvHjxPH9nfz9uVFSUKlWqpCZNmvi3Va1aVdL/fr6dOnXSuHHjNHDgQEVHR6tBgwa68sor/ddH5KFH9OhUPTrVbTVt2lSlS5fOdVvZ2dn+NysvWbKkZs6cqdmzZ+vYsWN65plnTnq82267TevWrdOsWbOCmgORiW7RrUC79Z///Edff/213nvvPcXGxp50v1KlSqlu3br5Hn/fvn3avXu3Lr300tPeHnAydItuna5b+/bt08CBA9WgQQOVK1dO5cqV0+HDh0/791EU8A5kHmjXrp0mTZqkEiVKqHr16ipRokSuy/96ki9J2dnZSkhI0NKlS/Mcq3z58ipfvnyuTySqWLHiaWf4+2089dRTGj9+vJ555hk1adJEpUuX1p133qnjx4+f8jh/n93n8yk7Ozvg6+Q8mP96nb+/8aQx5pTH+ztjzCmPERcXp9WrV2vp0qWaP3++HnroIY0YMUKrVq1S+fLltWDBAi1fvlzz58/Xc889p+HDh+ubb75R7dq1g7qtv2538nMKRH7HPd3Pd8iQIbrrrru0Z88eVahQQWlpaRo2bFi+9w9FHz2iR056lN9t/fV4OZYvXy5JOnDggA4cOJDn71qSbr/9dn3wwQdatmyZatasGdQciEx0i24F8nOaOXOmxo8fr6VLl562LfkdP2eOUy1mAYGiW3TrdD+nPn366JdfftEzzzyjxMRERUdHq3Xr1qf9+ygKeKaUB0qXLq169eopMTExz7+s+WnRooX27t2r4sWLq169erm+4uPj82zPiVLJkiWVlZUV0ExffPGFrr76avXs2VNNmzZVnTp1tHnz5gLdTyfOPPNMrVy5Mte2b7/9NqhjNGrUSF9++WWubcuXL1eDBg38zwYqXry4OnTooCeeeELr1q1TWlqaFi9eLOnPYLRp00aPPPKI1qxZo5IlS2ru3Lknva3ly5fnit7y5csVFxenGjVqBDV3jnLlyikhIUErVqzwbztx4oRSUlIcHS8/Pp9P1atXV2xsrGbNmqVatWqpRYsWrh0fhQc9Ojl69Kf8/u4aNWqktWvX5vqkvK+++krFihVTgwYNJElbtmzRXXfdpalTp+r888/XLbfckutkzBij2267TXPmzNHixYtZGEfA6NbJ0a0/ff311+rfv78mT56s888/3/FxpD9/mU1KStKiRYsKdBxENrp1cnTrT1988YUGDx6sK664QmeddZaio6O1f/9+x8crTFiUKgQ6dOig1q1bq2vXrvrss8+Ulpam5cuX64EHHjjlAzYpKUnr1q3Txo0btX///lN+9Ge9evX8K8Spqan617/+pb1794bi7pzS7bffrunTp+uVV17R5s2bNWrUKK1bty6oj+29++67tWjRIj366KPatGmTXnnlFU2cONH/sasfffSRJkyYoLVr1+qnn37Sq6++quzsbJ155pn65ptv9Nhjj+nbb7/V9u3bNWfOHP3yyy9q2LBhvrc1aNAg7dixQ7fffrs2bNig999/Xw8//LCGDBmiYsWcP7zuuOMOjR07VnPnztWGDRs0aNAgHTx40PHx/mrcuHH67rvv9P333+vRRx/V2LFjNWHCBF6+h4DQo8jrUX5/dzfffLNiYmLUu3dvrV+/XkuWLNHtt9+uXr16qWrVqsrKylKvXr3UqVMn9e3bVzNmzND69ev11FNP+Y976623aubMmXrjjTcUFxenvXv3au/evTp27Jh/nwMHDmjt2rX64YcfJP35FP+1a9d68u8DCi+6FVnd2rt3r7p166YePXrosssu87fll19+cXQ8SRoxYoSeeuopTZgwQZs3b9bq1atzvfQKcBvdiqxuSX/+fbz22mtKTU3VN998o5tvvjlinqnJolQh4PP5NG/ePF188cX65z//qQYNGqhHjx5KS0vzv2dQfgYMGKAzzzxTrVq1UuXKlfXVV1+ddN8HH3xQLVq00GWXXaZLLrlE1apVU9euXUNwb07t5ptv1rBhw3TPPfeoRYsW2rZtm/r06aOYmJiAj9GiRQu9/fbbevPNN9W4cWM99NBDGjlypPr06SPpz6e8zpkzR+3bt1fDhg314osvatasWTrrrLNUtmxZLVu2TFdccYUaNGigBx54QE899ZQ6d+6c723VqFFD8+bN08qVK9W0aVMNHDhQ/fr10wMPPFCgn8Pdd9+tW265RX369FHr1q0VFxenbt26FeiYOT755BNddNFFatWqlT7++GO9//77nvxdo3CiR5HXo/z+7kqVKqXPPvtMBw4c0DnnnKPrrrtOl156qSZOnChJGj16tNLS0jRlyhRJUrVq1TRt2jQ98MAD/pcbTJo0Senp6brkkkuUkJDg/3rrrbf8t/3BBx+oefPm/ve+69Gjh5o3b64XX3yxQPcJkYVuRVa3NmzYoJ9//lmvvPJKrracc845jo/Zu3dvPfPMM3rhhRd01lln6R//+IcnzyhB5KBbkdUtSXrppZf022+/qXnz5urVq5cGDx6sKlWqFOiYhYXPBPuCTSDMOnbsqGrVqum1117zehRP9enTRwcPHtR7773n9ShAxKJHAAobugWgsKFbkYU3OodVjh49qhdffFGXXXaZoqKiNGvWLC1cuFALFizwejQAEYYeAShs6BaAwoZugUUpWCXnqaqjRo1SZmamzjzzTM2ePVsdOnTwejQAEYYeAShs6BaAwoZugZfvAQAAAAAAIOx4o3MAAAAAAACEHYtSAAAAAAAACDsWpQAAAAAAABB2LEoBAAAAAAAg7FiUAgAAAAAAQNixKAUAAAAAAICwY1EKAAAAAAAAYceiFAAAAAAAAMKORSkAAAAAAACE3f8DHGTvV2Jked0AAAAASUVORK5CYII=",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "fig, axs = plt.subplots(nrows=1, ncols=4, figsize=(12, 4))\n",
+ "for i, loss_label in enumerate(PT_TASKS + [\"all\"]):\n",
+ " draw_boxplot(results, \"lipophilicity\", loss_label=loss_label, ax=axs[i])\n",
+ "plt.tight_layout()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " toymix_gcn_1 \n",
+ " toymix_gcn_2 \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " caco2 \n",
+ " 0.502565 \n",
+ " 0.523741 \n",
+ " \n",
+ " \n",
+ " lipophilicity \n",
+ " 0.478970 \n",
+ " 0.041120 \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGdCAYAAACPX3D5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAz3UlEQVR4nO3deVhV5f7//xcgk6iIIoiKgFrOI1qhOXQ0yzLTsmzSyilLLaO+Fulx6hRm5tD55NjRstScU0tTMqdS00PY6JBNKIIKOGQm4/r94a992htQ2K7NBtfz0bWuy33vte71Xl4tfPO+73UvD8MwDAEAAMvydHcAAADAvUgGAACwOJIBAAAsjmQAAACLIxkAAMDiSAYAALA4kgEAACyOZAAAAIsjGQAAwOIquDuAv2T9uMvdIQBljmdQTXeHAJRJ3sH1XNp/TvrPpvXl6ljNUGaSAQAAyoz8PHdHUKpIBgAAcGTkuzuCUsWcAQAALI7KAAAAjvKtVRkgGQAAwIHBMAEAALASKgMAADhimAAAAItjmAAAAFgJlQEAAByx6BAAABbHMAEAALASKgMAADiy2NMEVAYAAHBgGPmmbSU1a9YsRUVFyc/PT9HR0dq5c2eR+27btk0eHh4FtoMHD5bonFQGAABw5KbKwLJlyzRq1CjNmjVLHTp00Ny5c9WjRw/98MMPqlu3bpHHHTp0SFWqVLF9rlGjRonOS2UAAIAyYtq0aRo0aJAGDx6sxo0ba8aMGQoPD9fs2bMve1xISIhq1qxp27y8vEp0XpIBAAAcGfmmbVlZWTp37pzdlpWVVeCU2dnZSkxMVPfu3e3au3fvrl27dl023NatWyssLExdu3bV1q1bS3y5JAMAADjKzzNti4+PV2BgoN0WHx9f4JTp6enKy8tTaGioXXtoaKjS0tIKDTMsLEzz5s3TqlWrtHr1ajVs2FBdu3bVjh07SnS5zBkAAMCF4uLiFBsba9fm6+tb5P4eHh52nw3DKND2l4YNG6phw4a2zzExMTp69KimTp2qTp06FTtGkgEAAByZuOiQr6/vZf/x/0twcLC8vLwKVAFOnjxZoFpwOTfddJPef//9EsXIMAEAAI7y883bisnHx0fR0dFKSEiwa09ISFD79u2L3U9SUpLCwsKKvb9EZQAAgDIjNjZW/fv3V9u2bRUTE6N58+YpOTlZw4YNk3RpyCElJUWLFi2SJM2YMUORkZFq2rSpsrOz9f7772vVqlVatWpVic5LMgAAgCM3vZugX79+ysjI0KRJk5SamqpmzZppw4YNioiIkCSlpqYqOTnZtn92draef/55paSkyN/fX02bNtXHH3+sO+64o0Tn9TAMwzD1SpyU9ePlH5sArMgzqKa7QwDKJO/gei7tP+ubTab15dviNtP6chXmDAAAYHEMEwAA4MAw8twdQqkiGQAAwJGb5gy4C8kAAACOeIUxAACwEioDAAA4YpgAAACLy7fWBEKGCQAAsDgqAwAAOGKYAAAAi+NpAgAAYCVUBgAAcMQwAQAAFscwAQAAsBIqAwAAOLJYZYBkAAAAB7y1EAAAq7NYZYA5AwAAWByVAQAAHPFoIQAAFscwAQAAsBIqAwAAOGKYAAAAi2OYAAAAWAmVAQAAHDFMAACAxTFMAAAArITKAAAAjixWGSAZAADAEXMGAACwOItVBpgzAACAxVEZAADAEcMEAABYHMMEAADASqgMAADgiGECAAAsjmECAABgJVQGAABwZLHKAMkAAACODMPdEZQqhgkAALA4KgMAADhimAAAAIsjGQAAwOIsts4AcwYAALA4KgMAADhimAAAAIvj0UIAAGAlVAYAAHDEMAEAABZnsWSAYQIAACyuxMnAxx9/rMGDB2v06NE6ePCg3XenT5/WP/7xD9OCAwDALYx887ZyoETJwJIlS3T33XcrLS1Nu3fvVuvWrbV48WLb99nZ2dq+fbvpQQIAUJqMfMO0rTwo0ZyBqVOnavr06Ro5cqQkaeXKlXr88cd18eJFDRo0yCUBAgBQ6iw2Z6BEycDhw4fVs2dP2+e+ffsqODhYvXr1Uk5Ojvr06WN6gAAAwLVKlAxUqVJFJ06cUFRUlK2tS5cuWr9+vXr27Kljx46ZHiAAAKWunIz1m6VEcwZuuOEGbdy4sUB7586dtX79es2YMcOsuAAAcJ98w7ytHChRMvDss8/Kz8+v0O+6dOmijz76SAMGDDAlMAAAUDo8DKNsLMCc9eMud4cAlDmeQTXdHQJQJnkH13Np/xf+/ZRpfVUcOcu0vlzF6RUI8/Ly9OGHH+rAgQPy8PBQ48aNdffdd8vLy8vM+AAAKH08TXBlR44c0Z133qljx46pYcOGMgxDhw8fVnh4uD7++GPVr1/f7DgBAICLOLUc8dNPP6169erp6NGj+uqrr5SUlKTk5GRFRUXp6aefNjtGAABKl2GYt5UDTlUGtm/frj179qhatWq2turVq2vy5Mnq0KGDacHBNT74+DO9s3qj0jPPqH7d2ho95CFFN7u+0H33fXNQg156rUD72tmvKio8zNWhAi7zweqPtHDJSp3KyFSDqAi98PQTim7VrNB99371jQaOfKFA+7ol81QvIlyStHLdRq3buEVHfvlNktSkYQM988Rjat6koesuAq7DMMGV+fr66vfffy/Qfv78efn4+Fx1UHCdT3Z8qSnzl2jMk/3Vusl1WrFxm56aME0fznpFYSHVizxu3dx4Varob/scVKVyaYQLuMTGT7dr8sy5GvvccLVu0UQrPtygYc//U+ven6uwmiFFHvfR0vmqFFDR9jmoaqDtz/u++kZ33NpFrZo1lo+vjxYsXqGhz47Rh+/PUWiNYJdeD3C1nBom6Nmzp4YOHaovv/xShmHIMAzt2bNHw4YNU69evcyOESZa9OFm9bm1k+69rbPqhdfSC0MfUs3galq+4bPLHlctsIqCgwJtm5cXL7xE+bVo2Rrd07O7+va6XfUj6+rFUcNUM6SGPljz8WWPqxZUVcHVq9m2v0+Yfm3CC3rgnp5qdH191YsI18QXnlF+fr72/He/i68GLmGxdQacqgy8+eabevTRRxUTEyNvb29JUm5urnr16qWZM2eaGiDMk5OTqwNHftWgvnfYtce0bqr9B3+67LH3PzNe2dk5qhdeS0MfuEs3tGjsylABl8nJydEPh37UoEfus2tvf0Mbff3dD5c99r7HRygrO1v1I+vqiUcf1A3RLYvc9+LFLOXm5imQKlr5xAqEV1a1alWtXbtWhw8f1sqVK7VixQodOnRIa9asUWBg4JU7gFucPve78vLzVT2oil179aBApZ8+W+gxwdUCNW7EY5oWN1zTXxqhyDo1NWTM6/rvd4dKI2TAdKfPnFNeXr6qVwuya68eVFXpGacLPaZG9Wqa8MLTmv7KWM149Z+KrFtHg56J03/3f1vkeabPWaiQGtUV07a1qfGjlLixMjBr1ixFRUXJz89P0dHR2rlzZ7GO++KLL1ShQgW1atWqxOd0ep0BSWrQoIEaNGhQ4uOysrKUlZVl35idLV/mG5QKD3nYfTYMQx4ehe8bVSdMUXX+N1GwZeMGSjuVqXdXf6K2zZgYhfLLw+F/ekNGgba/REXUUVREHdvnVs0aK+3kKb2zZJXatmpeYP8Fi1doQ8I2Lfy/KfL15ecaim/ZsmUaNWqUZs2apQ4dOmju3Lnq0aOHfvjhB9WtW7fI486ePasBAwaoa9euOnHiRInP61RloG/fvpo8eXKB9tdff1333XdfIUfYi4+PV2BgoN02Zc57zoSCEgiqUllenp4FqgCZZ86petXiV3RaNKqv5OMl/58NKAuCqlaRl5en0jMy7dozT59V9WpVi91Pi6aN9Nux4wXaFy5ZqfmLlmne9FfUsEFUIUeiPDDy803bSmLatGkaNGiQBg8erMaNG2vGjBkKDw/X7NmzL3vcE088oYceekgxMTFOXa9TycD27dt15513Fmi//fbbtWPHjiseHxcXp7Nnz9pto4f1dyYUlIC3dwU1bhCp3fu/t2vfs/8HtWpU/IWiDv6UrOBqDAehfPL29laThtdp974ku/bd+75Sy2ZNit3PwcM/qUb1anZtCxav1Nx3lmrOGy+rWePCH9dFOWHiMEFWVpbOnTtntxWojkvKzs5WYmKiunfvbtfevXt37dpV9JL9Cxcu1E8//aTx48c7fblOJQNFPULo7e2tc+fOXfF4X19fValSxW5jiKB0DOjdXas379CazTv089HjmjJ/qVJPZei+O26RJM18Z4VeemO+bf/31m7WZ7u/0m8paTryW4pmvrNCn+76rx7s2dVdlwBctQH9+mjV+k1a/dEm/fRrsl6bOVepJ06pX59Lk2unz16ouJen2vZ/b9kabdmxS78dTdGRn3/T9NkLlbDtCz147122fRYsXqF/z39XL8c9q9phoUrPyFR6RqYuXPiz1K8PZUth1fD4+PgC+6WnpysvL0+hoaF27aGhoUpLSyu07x9//FEvvviiFi9erAoVnB/5d+rIZs2aadmyZRo3bpxd+wcffKAmTYqfWaP03d7pRp35/Q/N/WCdTmWeVYOI2nprwrOqFXLpOehTp88q7VSGbf+cnFy9sWCZTmaclq+Pj+rXraW3xo9Sx3ZFz6IGyroe3Trr7LnfNWfhEp3KyNR19SI1e+ok1ap56YdwekamUk+ctO2fk5urqf/3tk6eypCvr48aREVo1usT1an9DbZ9Plj9kXJycvXs2FfszvXkwIc1fNAjpXNhMI+JTxPExcUpNjbWrs3X17fI/QvMZzEKn8+Sl5enhx56SBMnTtT1119dJcqptxauW7dO9957rx566CH94x//kCRt2bJFS5cu1YoVK9S7d+8SB8JbC4GCeGshUDhXv7Xwj0kPm9ZXwLjFxdovOztbFStW1IoVK9SnTx9b+zPPPKP9+/dr+/btdvufOXNGQUFBdutd5OfnyzAMeXl5afPmzbZ/o6/EqcpAr1699OGHH+rVV1/VypUr5e/vrxYtWujTTz9V586dnekSAABL8/HxUXR0tBISEuySgYSEBN19990F9q9SpYq+/db+8dZZs2bps88+08qVKxUVVfwJrE4PMNx5552FTiIEAKDcc9O7CWJjY9W/f3+1bdtWMTExmjdvnpKTkzVs2DBJl4YcUlJStGjRInl6eqpZM/v3aYSEhMjPz69A+5Vc1ToDAABck9y0jHC/fv2UkZGhSZMmKTU1Vc2aNdOGDRsUEREhSUpNTVVycrLp53VqzkBeXp6mT5+u5cuXKzk5WdnZ2XbfZ2ZmFnFk0ZgzABTEnAGgcC6fMzDuAdP6Cpj0gWl9uYpTjxZOnDhR06ZN0/3336+zZ88qNjZW99xzjzw9PTVhwgSTQwQAoJQZ+eZt5YBTycDixYs1f/58Pf/886pQoYIefPBBvf322xo3bpz27NljdowAAJQui7210KlkIC0tTc2bX1qPu1KlSjp79tLytj179tTHH1/+FaAAAJR17lqO2F2cSgbq1Kmj1NRUSZdeVrR582ZJ0r59+y67kAIAACh7nEoG+vTpoy1btki6tBjCP//5T1133XUaMGCABg4caGqAAACUOosNEzj1aOHf31jYt29fhYeH64svvlCDBg3Uq1cv04IDAMAtysk/4mZxqjIQHx+vBQsW2D7feOONio2NVXp6ul577TXTggMAAK7nVDIwd+5cNWrUqEB706ZNNWfOnKsOCgAAt7LYo4VODROkpaUpLCysQHuNGjVsEwsBACi3GCa4sr/mCDj64osvVKtWrasOCgAAlB6nKgODBw/WqFGjlJOTY/cK49GjR+u5554zNUAAAEqbYbHKgFPJwOjRo5WZmamnnnrK9l4CPz8/vfDCC4qLizM1QAAASp3FkgGnXlT0l/Pnz+vAgQPy9/fXddddd1ULDvGiIqAgXlQEFM7VLyr6/emepvVV+c2PTOvLVa7qFcaVKlVSu3btzIoFAICyoZwsI2yWq0oGAAC4JllsmIBkAAAARxZLBpx6tBAAAFw7qAwAAODgKubWl0skAwAAOGKYAAAAWAmVAQAAHFmsMkAyAACAA6stR8wwAQAAFkdlAAAARxarDJAMAADgyFqrETNMAACA1VEZAADAgdUmEJIMAADgiGQAAACLY84AAACwEioDAAA4YM4AAABWxzABAACwEioDAAA4YJgAAACrY5gAAABYCZUBAAAcGBarDJAMAADgyGLJAMMEAABYHJUBAAAcMEwAAIDVkQwAAGBtVqsMMGcAAACLozIAAIADq1UGSAYAAHBgtWSAYQIAACyOygAAAI4MD3dHUKpIBgAAcMAwAQAAsBQqAwAAODDyGSYAAMDSGCYAAACWQmUAAAAHBk8TAABgbVYbJiAZAADAgdUmEDJnAAAAi6MyAACAA8NwdwSli2QAAAAHDBMAAABLoTIAAIADq1UGSAYAAHBgtTkDDBMAAGBxVAYAAHDAMAEAABZnteWIGSYAAMDiqAwAAODAau8moDIAAICDfMPDtK2kZs2apaioKPn5+Sk6Olo7d+4sct/PP/9cHTp0UPXq1eXv769GjRpp+vTpJT4nlQEAABy4a87AsmXLNGrUKM2aNUsdOnTQ3Llz1aNHD/3www+qW7dugf0DAgI0YsQItWjRQgEBAfr888/1xBNPKCAgQEOHDi32eT0Mo2w8TZn14y53hwCUOZ5BNd0dAlAmeQfXc2n/hxr1MK2vhgc3FnvfG2+8UW3atNHs2bNtbY0bN1bv3r0VHx9frD7uueceBQQE6L333iv2eRkmAADAgZHvYdqWlZWlc+fO2W1ZWVkFzpmdna3ExER1797drr179+7atat4vzAnJSVp165d6ty5c4mul2QAAAAHhmHeFh8fr8DAQLutsN/y09PTlZeXp9DQULv20NBQpaWlXTbeOnXqyNfXV23bttXw4cM1ePDgEl0vcwYAAHChuLg4xcbG2rX5+voWub+Hh/18BcMwCrQ52rlzp86fP689e/boxRdfVIMGDfTggw8WO0aSAQAAHJi5AqGvr+9l//H/S3BwsLy8vApUAU6ePFmgWuAoKipKktS8eXOdOHFCEyZMKFEywDABAAAO3PFooY+Pj6Kjo5WQkGDXnpCQoPbt2xe7H8MwCp2TcDlUBgAAKCNiY2PVv39/tW3bVjExMZo3b56Sk5M1bNgwSZeGHFJSUrRo0SJJ0ltvvaW6deuqUaNGki6tOzB16lSNHDmyROclGQAAwIG71hno16+fMjIyNGnSJKWmpqpZs2basGGDIiIiJEmpqalKTk627Z+fn6+4uDj98ssvqlChgurXr6/JkyfriSeeKNF5WWcAKMNYZwAonKvXGfgm8i7T+mrx63rT+nIV5gwAAGBxDBMAAODAmXcKlGckAwAAOHDXnAF3IRkAAMBB2ZhNV3qYMwAAgMVRGQAAwAFzBtzEOJ3i7hCAMse/6X3uDgEok3KzXftvhtXmDDBMAACAxZWZygAAAGUFwwQAAFicxR4mYJgAAACrozIAAIADhgkAALA4niYAAACWQmUAAAAH+e4OoJSRDAAA4MCQtYYJSAYAAHCQb7FnC5kzAACAxVEZAADAQT7DBAAAWJvV5gwwTAAAgMVRGQAAwAGPFgIAYHEMEwAAAEuhMgAAgAOGCQAAsDirJQMMEwAAYHFUBgAAcGC1CYQkAwAAOMi3Vi5AMgAAgCOrLUfMnAEAACyOygAAAA4s9gZjkgEAABzxaCEAALAUKgMAADjI97DWBEKSAQAAHFhtzgDDBAAAWByVAQAAHFhtAiHJAAAADqy2AiHDBAAAWByVAQAAHFhtOWKSAQAAHFjtaQKSAQAAHDBnAAAAWAqVAQAAHPBoIQAAFme1OQMMEwAAYHFUBgAAcGC1CYQkAwAAOLDanAGGCQAAsDgqAwAAOLBaZYBkAAAAB4bF5gwwTAAAgMVRGQAAwAHDBAAAWBzJAAAAFscKhAAAwFKoDAAA4IAVCAEAsDirzRlgmAAAAIujMgAAgAOrVQZIBgAAcMDTBMXwxx9/mB0HAABwE6eSgdDQUA0cOFCff/652fEAAOB2+R7mbeWBU8nA0qVLdfbsWXXt2lXXX3+9Jk+erOPHj5sdGwAAbpFv4lYeOJUM3HXXXVq1apWOHz+uJ598UkuXLlVERIR69uyp1atXKzc31+w4AQCwhFmzZikqKkp+fn6Kjo7Wzp07i9x39erVuvXWW1WjRg1VqVJFMTEx2rRpU4nPeVWPFlavXl3PPvusvv76a02bNk2ffvqp+vbtq1q1amncuHG6cOHC1XQPAIBbGCZuJbFs2TKNGjVKY8aMUVJSkjp27KgePXooOTm50P137NihW2+9VRs2bFBiYqJuueUW3XXXXUpKSirReT0Mw3B60mRaWpoWLVqkhQsXKjk5WX369NGgQYN0/PhxTZ48WWFhYdq8eXOx+rq4d4WzYQDXrEo3j3J3CECZlJud4tL+X4l42LS+xvy2uNj73njjjWrTpo1mz55ta2vcuLF69+6t+Pj4YvXRtGlT9evXT+PGjSv2eZ16tHD16tVauHChNm3apCZNmmj48OF65JFHVLVqVds+rVq1UuvWrZ3pHgAAtzJzrD8rK0tZWVl2bb6+vvL19bVry87OVmJiol588UW79u7du2vXrl3FOld+fr5+//13VatWrUQxOjVM8Pjjj6tWrVr64osvtH//fo0YMcIuEZCkevXqacyYMc50DwDANSM+Pl6BgYF2W2G/5aenpysvL0+hoaF27aGhoUpLSyvWud544w398ccfuv/++0sUo1OVgdTUVFWsWPGy+/j7+2v8+PHOdA8AgFuZuehQXFycYmNj7docqwJ/5+Fh/zyiYRgF2gqzdOlSTZgwQWvXrlVISEiJYnQqGahcubJSU1MLnCwjI0MhISHKy8tzplsAAMoEM4cJChsSKExwcLC8vLwKVAFOnjxZoFrgaNmyZRo0aJBWrFihbt26lThGp4YJippzmJWVJR8fH2e6BADA0nx8fBQdHa2EhAS79oSEBLVv377I45YuXarHHntMS5Ys0Z133unUuUtUGXjzzTclXSphvP3226pUqZLtu7y8PO3YsUONGjVyKhAAAMoKd60cGBsbq/79+6tt27aKiYnRvHnzlJycrGHDhkm6NOSQkpKiRYsWSbqUCAwYMEAzZ87UTTfdZKsq+Pv7KzAwsNjnLVEyMH36dEmXKgNz5syRl5eX7TsfHx9FRkZqzpw5JekSAIAyJ99Nryrq16+fMjIyNGnSJKWmpqpZs2basGGDIiIiJF2as/f3NQfmzp2r3NxcDR8+XMOHD7e1P/roo3rnnXeKfV6n1hm45ZZbtHr1agUFBZX00CKxzgBQEOsMAIVz9ToDYyMfMq2vf/26xLS+XMWpCYRbt241Ow4AAMoMq73CuNjJQGxsrF5++WUFBAQUeETC0bRp0646MAAA3KW8vGDILMVOBpKSkpSTk2P7c1GK8ywkAAAoO4qdDPx9aIBhAgDAtcxdEwjdxal1Bs6ePavMzMwC7ZmZmTp37txVBwUAgDu5662F7uJUMvDAAw/ogw8+KNC+fPlyPfDAA1cdFAAA7pRv4lYeOJUMfPnll7rlllsKtHfp0kVffvnlVQcFAABKj1OPFmZlZSk3N7dAe05Ojv7888+rDgoAAHdizkAxtGvXTvPmzSvQPmfOHEVHR191UAAAuJPV5gw4VRl45ZVX1K1bN3399dfq2rWrJGnLli3at2+fNm/ebGqAAADAtZyqDHTo0EG7d+9WeHi4li9frvXr16tBgwb65ptv1LFjR7NjBACgVFltAqFTlQFJatWqlRYvXmxmLAAAlAlGuSnwm6PYycC5c+dUpUoV258v56/9AABA2VfsZCAoKEipqakKCQlR1apVC1122DAMeXh4KC8vz9QgAQAoTeWlvG+WYicDn332mapVqyaJ5YgBANc2qz1aWOxkoHPnzoX+GQAAlG/FTga++eabYnfaokULp4IBAKAssFZdoATJQKtWreTh4SHDuPxfEXMGypZln36pdz7eqfSz51W/dohGP3KH2jSMvOJxSYd/06BX/qMGdUK0/JURdt+9/8kuLd+yV2kZZ1S1ckXd2q6Znr7/Vvn6eLvoKoCrM+yJR/Vc7DCFhYXo+x8O67nnxuvzL/YWuX+njjfp9dfHq2mT63X8+AlNfWO25s1/r9B977+/l5a8P1tr132ie/sOsrV3vPlGPffck2rTurlq1aqpe/oO1Lp1m0y/NrgGwwRF+OWXX1wZB1zgkz3fasr7GzTmsbvU6rq6Wrl1n556fZHWTH5aYcFVizzu9wsXNXbuSt3QtJ4yz563++7jL/Zr5vLNmji4j1peV1e/paVr3LzVkqT/98gdrrwcwCn33ddL096YoBEjX9Ku3fs0ZHB/fbT+fTVv2UVHjx4vsH9kZLjWr3tPb/9niR59bKTax7TT//37VZ1Kz9CaNRvs9q1bt7amTB6nnTv3FOgnIKCivvnmB73z7jKtXP62y64PrsEEwiJERES4Mg64wHsbv1CfztG6p0tbSdLoR+7Urm+PaPmWvXqmX/cij3t5wVr1iGkpL08PbU08YPfd10eOqtV1dXVH+5aSpNo1gnR7TAt999Mx110IcBWefWaIFiz8QAsWLpUkPff8eHXv3lnDnhigMWMnF9j/iaH9lXw0Rc89P16SdPDgEUVHt9Rzzw6zSwY8PT313rv/p4mTpurmm29U1ar2j1R/smmrPtnEZGuUD8VOBtatW6cePXrI29tb69atu+y+vXr1uurAcHVycnN14NfjGnhXJ7v2mGYN9PWPyUUe9+GORB07malXn+yr+Wu3Ffi+9fUR2rDra3370zE1r19Hx05m6vOvD+uum1uZfAXA1fP29labNi302utv2bUnJGxXzE1tCz3mphujlZCw3a5tc8I2DXz8AVWoUMH2krZ/jn1Wp9IztPCdD3TzzTe65gLgNiw6VITevXsrLS1NISEh6t27d5H7MWegbDj9+wXl5eerepVKdu3VAwOU7lD6/8tvaemauWyzFo4dogpeXoXu0yOmhU7//ocee3m+JEO5efm6v+sNGnQXT5ig7AkOrqYKFSro5Il0u/aTJ9MVWjOk0GNCa4bo5EmH/U+ky9vbW8HB1ZSWdlLtY9rq8cceVHS7W10WO9yLYYIi5OfnF/pnZ2RlZSkrK8uuzcjOYQKaCziuDWUYBdskKS8/X3GzVujJe7oqMiy4yP72HfhZb6/brjGP3aXm9eso+USmprz/sYI/3Konet9icvSAORwnPl9pMnTB/f/XXqlSgN59598a9uT/U0bGadNjBdzB6XcTXI34+HhNnDjRrm3M4L4aO+R+d4RzTQqqXFFenp4FqgCZ5/4oUC2QpD/+zNL3v6To4G+pmrzoI0lSvmHIMAy1eXScZo9+VDc2ra+3Vm5Rzw6tbPMQrguvqT+zsvXygrUa0quzPD2devcV4BLp6ZnKzc1VaM0adu01alTXyROnCj3mRNpJhYY67B8SrJycHGVknFbTpg0VFVVXH655x/b9X//fX7zwm5o066Sff/7N3AtBqWOYoJi2bNmiLVu26OTJkwUqBQsWLLjssXFxcYqNjbVrM775yNlQUAjvChXUOLKW9nx3RF3bNrG17/nuiLq0aVxg/0r+vlr56ki7tuVbvtTeH37W1JEPqnaNIEnSxeycAktRe3le+i3LWrcOyoOcnBx99dU36ta1k9au/cTW3q1bJ61fX/hjfnu+TNSdd9qX/2/t1lmJid8oNzdXBw8eUcvW/7D7ftLE0apcqZKefW5coU8ooPxhmKAYJk6cqEmTJqlt27YKCwsr9D0Fl+Pr6ytfX1+7tosMEZiuf48OGjNnpZpE1VbLBuFatfW/Ss04q/u6tpMkzVy2WSdPn9Mrw/rK09NT14WH2h1frUqAfL0r2LV3bt1Q723cpUYRYWpev46OnsjUWyu3qHObRvKiKoAyaPrM+Xp34UwlJn6tPV8masigR1Q3vLbmzru0bsAr/3pRtWqF6fGBz0iS5s57T089+bimThmvtxcs1k03Rmvg4w/o4f7DJV0a5vz++0N25zhz5tLL2/7eHhBQUQ0aRNk+R0XWVcuWTZWZeZqEAWWOU8nAnDlz9M4776h///5mxwMT3X5Tc509f0HzPtyqU2d+V4M6oXrr+f6qFXzpt/z0M78rLeNMifoccncXechDb638VCdPn1NQlQB1btVII+7r5oIrAK7eihXrVL1akMaOeVZhYSH67vtDuqtXfyUnp0iSatYMVd3wWrb9f/31qO7q1V9Tp07Qk08+quPHT2jUs+MKrDFwJW2jW2rLpyttn9+YOkGS9O6i5Ro0+NmrvzC4VP4VFti71ngYV1pSsBDVq1fX3r17Vb9+fdMCubh3hWl9AdeKSjePcncIQJmUm53i0v4fibjHtL7e/221aX25ilN13cGDB2vJkiVmxwIAANyg2MMEf5/wl5+fr3nz5unTTz9VixYt5O1tP94/bdo08yIEAKCU8W6CIiQlJdl9btWqlSTpu+++s2sv6WRCAADKGqs9H1XsZGDrVtbYBgBYg9UeLbzqZ8GOHj2qY8d4SQ0AAOWVU8lAbm6u/vnPfyowMFCRkZGKiIhQYGCgxo4dq5ycHLNjBACgVOXLMG0rD5xaZ2DEiBFas2aNpkyZopiYGEnS7t27NWHCBKWnp2vOnDmmBgkAQGlizkAxLF26VB988IF69Ohha2vRooXq1q2rBx54gGQAAIByxKlkwM/PT5GRkQXaIyMj5ePjc7UxAQDgVkwgLIbhw4fr5ZdftnsNcVZWll555RWNGDHCtOAAAHAH4/9/a6sZW3ngVGUgKSlJW7ZsUZ06ddSyZUtJ0tdff63s7Gx17dpV99zzv2UcV68u+8swAgBgZU4lA1WrVtW9995r1xYeHm5KQAAAuFt5eQrALE4lAwsXLjQ7DgAAygyrzRlwKhn4y6lTp3To0CF5eHjo+uuvV40aNcyKCwAAlBKnJhD+8ccfGjhwoMLCwtSpUyd17NhRtWrV0qBBg3ThwgWzYwQAoFQZJv5XHjiVDMTGxmr79u1av369zpw5ozNnzmjt2rXavn27nnvuObNjBACgVLECYTGsWrVKK1euVJcuXWxtd9xxh/z9/XX//fdr9uzZZsUHAECpKy+PBJrFqcrAhQsXFBoaWqA9JCSEYQIAAMoZp5KBmJgYjR8/XhcvXrS1/fnnn5o4caLtXQUAAJRX+SZu5YFTwwQzZ87U7bffblt0yMPDQ/v375efn582bdpkdowAAJSq8jLxzyxOJQPNmjXTjz/+qPfff18HDx6UYRh64IEH9PDDD8vf39/sGAEAgAs5vc6Av7+/hgwZYmYsAACUCeXlKQCzOJ0MHDp0SP/+97914MABeXh4qFGjRhoxYoQaNWpkZnwAAJQ6niYohpUrV6pZs2ZKTExUy5Yt1aJFC3311Vdq3ry5VqxYYXaMAADAhZyqDIwePVpxcXGaNGmSXfv48eP1wgsv6L777jMlOAAA3MFqwwROVQbS0tI0YMCAAu2PPPKI0tLSrjooAADcieWIi6FLly7auXNngfbPP/9cHTt2vOqgAABwp3zDMG0rD5waJujVq5deeOEFJSYm6qabbpIk7dmzRytWrNDEiRO1bt06u30BAEDZ5WE4MWXS07N4BQUPDw/l5eUVa9+Le5l4CDiqdPMod4cAlEm52Sku7b9j7a6m9bUzZYtpfbmKU5WB/PzyssAiAAAlxwRCAABgKcWuDLz55psaOnSo/Pz89Oabb15236effvqqAwMAwF2sVhko9pyBqKgo/fe//1X16tUVFRVVdIceHvr5559LHAhzBoCCmDMAFM7VcwZuqtXFtL72HN9mWl+uUuzKwC+//FLonwEAQPlW7GQgNja2WPt5eHjojTfecDogAADczWrDBMVOBpKSkoq1n4eHh9PBAABQFpSXlQPNUuxkYOvWra6MAwAAuInTrzAGAOBaxSuMAQCwuHwZpm0lNWvWLEVFRcnPz0/R0dGFvgvoL6mpqXrooYfUsGFDeXp6atSoUU5dL8kAAAAODMMwbSuJZcuWadSoURozZoySkpLUsWNH9ejRQ8nJyYXun5WVpRo1amjMmDFq2bKl09dLMgAAQBkxbdo0DRo0SIMHD1bjxo01Y8YMhYeHa/bs2YXuHxkZqZkzZ2rAgAEKDAx0+rzMGQAAwIGZjxZmZWUpKyvLrs3X11e+vr52bdnZ2UpMTNSLL75o1969e3ft2rXLtHgKQ2UAAAAHhon/xcfHKzAw0G6Lj48vcM709HTl5eUpNDTUrj00NFRpaWkuvV4qAwAAuFBcXFyBhfscqwJ/57hej2EYLl/Dh2QAAAAH+SY+WljYkEBhgoOD5eXlVaAKcPLkyQLVArMxTAAAgAMzhwmKy8fHR9HR0UpISLBrT0hIUPv27c2+RDtUBgAAKCNiY2PVv39/tW3bVjExMZo3b56Sk5M1bNgwSZeGHFJSUrRo0SLbMfv375cknT9/XqdOndL+/fvl4+OjJk2aFPu8JAMAADgwc5igJPr166eMjAxNmjRJqampatasmTZs2KCIiAhJlxYZclxzoHXr1rY/JyYmasmSJYqIiNCvv/5a7PN6GGVkzcWLe1e4OwSgzKl08yh3hwCUSbnZKS7tv1FIO9P6Onhyn2l9uQpzBgAAsDiGCQAAcOCuYQJ3IRkAAMBBSZ4CuBaQDAAA4MBqlQHmDAAAYHFUBgAAcMAwAQAAFmcY+e4OoVQxTAAAgMVRGQAAwEE+wwQAAFhbGVmct9QwTAAAgMVRGQAAwAHDBAAAWBzDBAAAwFKoDAAA4MBqyxGTDAAA4IAVCAEAsDjmDAAAAEuhMgAAgAMeLQQAwOIYJgAAAJZCZQAAAAc8WggAgMUxTAAAACyFygAAAA54mgAAAItjmAAAAFgKlQEAABzwNAEAABbHi4oAALA4q1UGmDMAAIDFURkAAMCB1Z4mIBkAAMCB1eYMMEwAAIDFURkAAMABwwQAAFic1ZIBhgkAALA4KgMAADiwVl1A8jCsVgvBZWVlZSk+Pl5xcXHy9fV1dzhAmcB9gWsdyQDsnDt3ToGBgTp79qyqVKni7nCAMoH7Atc65gwAAGBxJAMAAFgcyQAAABZHMgA7vr6+Gj9+PJOkgL/hvsC1jgmEAABYHJUBAAAsjmQAAACLIxkAAMDiSAYswMPDQx9++KG7wwDKHO4N4BKSARfp0qWLRo0a5e4wJEmpqanq0aOHu8Mwxffff697771XkZGR8vDw0IwZM9wdEkqIe8M15s+fr44dOyooKEhBQUHq1q2b9u7d6+6wUE6QDFhAzZo1r5lHoi5cuKB69epp8uTJqlmzprvDQTl3Ld0b27Zt04MPPqitW7dq9+7dqlu3rrp3766UlBR3h4bywIDpHn30UUOXXnpl23755Rdj27ZtRrt27QwfHx+jZs2axgsvvGDk5OQYhmEY7777rlGtWjXj4sWLdn3dc889Rv/+/Q3DMIzx48cbLVu2NP7zn/8Y4eHhRkBAgDFs2DAjNzfXeO2114zQ0FCjRo0axr/+9S+7PiQZa9assZ0nICDAOHz4sO37ESNGGNddd51x/vz5K17b8ePHjTvuuMPw8/MzIiMjjcWLFxsRERHG9OnTbfucPn3aGDJkiBESEmL4+voaTZs2NdavX28YhmEsXLjQCAwMND755BOjUaNGRkBAgHHbbbcZx48fL/Hfs+N5UfZxb5TOvWEYhpGbm2tUrlzZePfdd506HtZCMuACZ86cMWJiYowhQ4YYqampRmpqqnHs2DGjYsWKxlNPPWUcOHDAWLNmjREcHGyMHz/eMAzDuHDhghEYGGgsX77c1s+pU6cMHx8f47PPPjMM49IPvEqVKhl9+/Y1vv/+e2PdunWGj4+PcdtttxkjR440Dh48aCxYsMCQZOzevdvWz99/4BmGYdx3331Gu3btjJycHGPjxo2Gt7e3sXfv3mJdW7du3YxWrVoZe/bsMRITE43OnTsb/v7+th94eXl5xk033WQ0bdrU2Lx5s/HTTz8Z69evNzZs2GAYxqUfeN7e3ka3bt2Mffv2GYmJiUbjxo2Nhx56qMR/zyQD5Q/3RuncG4ZhGOfOnTP8/PxsyQZwOSQDLtK5c2fjmWeesX1+6aWXjIYNGxr5+fm2trfeesuoVKmSkZeXZxiGYTz55JNGjx49bN/PmDHDqFevnu2Y8ePHGxUrVjTOnTtn2+e2224zIiMjbX0YhmE0bNjQiI+Pt312/IGXmZlp1KlTx3jyySeN0NDQAr8tFeXAgQOGJGPfvn22th9//NGQZPuBt2nTJsPT09M4dOhQoX0sXLjQkGQcOXLE7u8hNDS0WDH8HclA+cS94fp7wzAM46mnnjLq169v/Pnnn04dD2upUJpDElZ24MABxcTEyMPDw9bWoUMHnT9/XseOHVPdunU1ZMgQtWvXTikpKapdu7YWLlyoxx57zO6YyMhIVa5c2fY5NDRUXl5e8vT0tGs7efJkkbEEBQXpP//5j2677Ta1b99eL774YrGu4dChQ6pQoYLatGlja2vQoIGCgoJsn/fv3686dero+uuvL7KfihUrqn79+rbPYWFhl40X1zbujf8x696YMmWKli5dqm3btsnPz6/Ex8N6mEBYSgzDsPvB9VebJFt769at1bJlSy1atEhfffWVvv32Wz322GN2x3h7e9t99vDwKLQtPz//svHs2LFDXl5eOn78uP74449iX8OV2v39/a/YT2HxFtU3rn3cG/9jxr0xdepUvfrqq9q8ebNatGhRomNhXSQDLuLj46O8vDzb5yZNmmjXrl12N/auXbtUuXJl1a5d29Y2ePBgLVy4UAsWLFC3bt0UHh5uemy7du3SlClTtH79elWpUkUjR44s1nGNGjVSbm6ukpKSbG1HjhzRmTNnbJ9btGihY8eO6fDhw2aHjWsE94br7o3XX39dL7/8sj755BO1bdvWZefBtYdkwEUiIyP15Zdf6tdff1V6erqeeuopHT16VCNHjtTBgwe1du1ajR8/XrGxsXZlzIcfflgpKSmaP3++Bg4caHpcv//+u/r376+RI0eqR48eWrJkiZYvX64VK1Zc8dhGjRqpW7duGjp0qPbu3aukpCQNHTpU/v7+tt/gOnfurE6dOunee+9VQkKCfvnlF23cuFGffPKJKfFnZ2dr//792r9/v7Kzs5WSkqL9+/fryJEjpvQP1+PecM29MWXKFI0dO1YLFixQZGSk0tLSlJaWpvPnz5vSP65tJAMu8vzzz8vLy0tNmjRRjRo1lJOTow0bNmjv3r1q2bKlhg0bpkGDBmns2LF2x1WpUkX33nuvKlWqpN69e5se1zPPPKOAgAC9+uqrkqSmTZvqtdde07Bhw4r1PPKiRYsUGhqqTp06qU+fPhoyZIgqV65sNy65atUqtWvXTg8++KCaNGmi0aNH2/0meDWOHz+u1q1bq3Xr1kpNTdXUqVPVunVrDR482JT+4XrcG665N2bNmqXs7Gz17dtXYWFhtm3q1Kmm9I9rG68wLoNuvfVWNW7cWG+++aa7Q7miY8eOKTw8XJ9++qm6du3q7nBwjePeAFyDZKAMyczM1ObNm/Xwww/rhx9+UMOGDd0dUgGfffaZzp8/r+bNmys1NVWjR49WSkqKDh8+XGDyE2AW7g3AtXi0sAxp06aNTp8+rddee80tP+x27tx52XXaz58/r5ycHL300kv6+eefVblyZbVv316LFy827YddpUqVivxu48aN6tixoynnQfnCvcG9AdeiMgCbP//887Jjow0aNHB5DJebCFi7du1iPZ4FmI17A9c6kgEAACyOpwkAALA4kgEAACyOZAAAAIsjGQAAwOJIBgAAsDiSAQAALI5kAAAAiyMZAADA4v4/YDgYj30wX4gAAAAASUVORK5CYII=",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "sns.heatmap(data, annot=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The End."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.12"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/pyproject.toml b/pyproject.toml
index 20cfa9792..f24f61c82 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ classifiers = [
"Programming Language :: Python :: 3.11",
]
dependencies = [
- "click",
+ "typer",
"loguru",
"omegaconf >=2.0.0",
"tqdm",
@@ -63,9 +63,8 @@ dependencies = [
]
[project.scripts]
-graphium = "graphium.cli.main:main_cli"
- graphium-train = "graphium.cli.train_finetune:cli"
- graphium-prepare-data = "graphium.cli.prepare_data:cli"
+graphium = "graphium.cli.main:app"
+graphium-train = "graphium.cli.train_finetune:cli"
[project.urls]
Website = "https://graphium.datamol.io/"
diff --git a/requirements_ipu.txt b/requirements_ipu.txt
index 59981ff38..813471ea4 100644
--- a/requirements_ipu.txt
+++ b/requirements_ipu.txt
@@ -1,7 +1,7 @@
--find-links https://data.pyg.org/whl/torch-2.0.1+cpu.html
pip
-click
+typer
loguru
tqdm
numpy
@@ -34,7 +34,6 @@ mkdocs-material-extensions
mkdocstrings
mkdocstrings-python
mkdocs-jupyter
-mkdocs-click
markdown-include
rever >==0.4.5
omegaconf
diff --git a/tests/test_finetuning.py b/tests/test_finetuning.py
index 95ed3ddff..04ce531f0 100644
--- a/tests/test_finetuning.py
+++ b/tests/test_finetuning.py
@@ -1,31 +1,24 @@
import os
-from os.path import dirname, abspath
-
import unittest as ut
-
-import torch
from copy import deepcopy
+from os.path import abspath, dirname
+import torch
from lightning.pytorch.callbacks import Callback
-
from omegaconf import OmegaConf
-import graphium
-
-from graphium.finetuning import modify_cfg_for_finetuning
-from graphium.trainer import PredictorModule
-
-from graphium.finetuning import GraphFinetuning
+import graphium
from graphium.config._loader import (
+ load_accelerator,
+ load_architecture,
load_datamodule,
load_metrics,
- load_architecture,
load_predictor,
load_trainer,
save_params_to_wandb,
- load_accelerator,
)
-
+from graphium.finetuning import GraphFinetuning, modify_cfg_for_finetuning
+from graphium.trainer import PredictorModule
MAIN_DIR = dirname(dirname(abspath(graphium.__file__)))
CONFIG_FILE = "graphium/config/dummy_finetuning.yaml"
@@ -103,7 +96,7 @@ def test_finetuning_from_task_head(self):
# Load pretrained & replace in predictor
pretrained_model = PredictorModule.load_pretrained_models(
- cfg["finetuning"]["pretrained_model_name"], device="cpu"
+ cfg["finetuning"]["pretrained_model"], device="cpu"
).model
pretrained_model.create_module_map()