From 88a34521ec8a8f6a27604cbf29d59974304d516e Mon Sep 17 00:00:00 2001
From: Karlson Pfannschmidt <kiudee@mail.upb.de>
Date: Thu, 10 Feb 2022 22:25:26 +0100
Subject: [PATCH] Add plot_performance to the CLI plotting loop

---
 tune/__init__.py |  3 ++-
 tune/cli.py      |  1 +
 tune/local.py    | 22 +++++++++++++++++-----
 tune/plots.py    |  8 +++++++-
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/tune/__init__.py b/tune/__init__.py
index ae137ca..fc7a312 100644
--- a/tune/__init__.py
+++ b/tune/__init__.py
@@ -16,7 +16,7 @@
     reduce_ranges,
     run_match,
 )
-from tune.plots import partial_dependence, plot_objective, plot_optima
+from tune.plots import partial_dependence, plot_objective, plot_optima, plot_performance
 from tune.priors import roundflat
 from tune.utils import TimeControl, TimeControlBag, expected_ucb, parse_timecontrol
 
@@ -31,6 +31,7 @@
     "partial_dependence",
     "plot_objective",
     "plot_optima",
+    "plot_performance",
     "prob_to_elo",
     "reduce_ranges",
     "roundflat",
diff --git a/tune/cli.py b/tune/cli.py
index 9fea28d..2f76726 100644
--- a/tune/cli.py
+++ b/tune/cli.py
@@ -427,6 +427,7 @@ def local(  # noqa: C901
                     optimizer=opt,
                     result_object=result_object,
                     iterations=np.array(performance)[:, 0],
+                    elos=np.array(performance)[:, 1:],
                     optima=np.array(optima),
                     plot_path=settings.get("plot_path", plot_path),
                     parameter_names=list(param_ranges.keys()),
diff --git a/tune/local.py b/tune/local.py
index 7100ccb..20bcc3d 100644
--- a/tune/local.py
+++ b/tune/local.py
@@ -19,7 +19,7 @@
 from skopt.space import Categorical, Dimension, Integer, Real, Space
 from skopt.utils import normalize_dimensions
 
-from tune.plots import plot_objective, plot_objective_1d, plot_optima
+from tune.plots import plot_objective, plot_objective_1d, plot_optima, plot_performance
 from tune.summary import confidence_intervals
 from tune.utils import TimeControl, confidence_to_mult, expected_ucb
 
@@ -519,6 +519,7 @@ def plot_results(
     optimizer: Optimizer,
     result_object: OptimizeResult,
     iterations: np.ndarray,
+    elos: np.ndarray,
     optima: np.ndarray,
     plot_path: str,
     parameter_names: Sequence[str],
@@ -534,6 +535,8 @@ def plot_results(
         Result object containing the data and the last fitted model.
     iterations : np.ndarray
         Array containing the iterations at which optima were collected.
+    elos : np.ndarray, shape=(n_iterations, 2)
+        Array containing the estimated Elo of the optima and the standard error.
     optima : np.ndarray
         Array containing the predicted optimal parameters.
     plot_path : str
@@ -570,21 +573,30 @@ def plot_results(
         fig.patch.set_facecolor(dark_gray)
         plot_objective(result_object, dimensions=parameter_names, fig=fig, ax=ax)
     plotpath = pathlib.Path(plot_path)
-    plotpath.mkdir(parents=True, exist_ok=True)
-    full_plotpath = plotpath / f"{timestr}-{len(optimizer.Xi)}.png"
+    for subdir in ["landscapes", "elo", "optima"]:
+        (plotpath / subdir).mkdir(parents=True, exist_ok=True)
+    full_plotpath = plotpath / f"landscapes/landscape-{timestr}-{len(optimizer.Xi)}.png"
     dpi = 150 if optimizer.space.n_dims == 1 else 300
     plt.savefig(full_plotpath, dpi=dpi, facecolor=dark_gray, **save_params)
     logger.info(f"Saving a plot to {full_plotpath}.")
     plt.close(fig)
 
-    # Now plot the history of optima:
+    # Plot the history of optima:
     fig, ax = plot_optima(
         iterations=iterations,
         optima=optima,
         space=optimizer.space,
         parameter_names=parameter_names,
     )
-    full_plotpath = plotpath / f"optima-{timestr}-{len(optimizer.Xi)}.png"
+    full_plotpath = plotpath / f"optima/optima-{timestr}-{len(optimizer.Xi)}.png"
+    fig.savefig(full_plotpath, dpi=150, facecolor=dark_gray)
+    plt.close(fig)
+
+    # Plot the predicted Elo performance of the optima:
+    fig, ax = plot_performance(
+        performance=np.hstack([iterations[:, None], elos]), confidence=confidence
+    )
+    full_plotpath = plotpath / f"elo/elo-{timestr}-{len(optimizer.Xi)}.png"
     fig.savefig(full_plotpath, dpi=150, facecolor=dark_gray)
     plt.close(fig)
 
diff --git a/tune/plots.py b/tune/plots.py
index ee55c69..2094ddb 100644
--- a/tune/plots.py
+++ b/tune/plots.py
@@ -12,7 +12,13 @@
 
 from tune.utils import confidence_to_mult, expected_ucb
 
-__all__ = ["partial_dependence", "plot_objective", "plot_objective_1d", "plot_optima"]
+__all__ = [
+    "partial_dependence",
+    "plot_objective",
+    "plot_objective_1d",
+    "plot_optima",
+    "plot_performance",
+]
 
 
 def _evenly_sample(dim, n_points):