Skip to content

Commit

Permalink
Add plot_performance to the CLI plotting loop
Browse files Browse the repository at this point in the history
  • Loading branch information
kiudee committed Feb 10, 2022
1 parent fbd99df commit 88a3452
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 7 deletions.
3 changes: 2 additions & 1 deletion tune/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
reduce_ranges,
run_match,
)
from tune.plots import partial_dependence, plot_objective, plot_optima
from tune.plots import partial_dependence, plot_objective, plot_optima, plot_performance
from tune.priors import roundflat
from tune.utils import TimeControl, TimeControlBag, expected_ucb, parse_timecontrol

Expand All @@ -31,6 +31,7 @@
"partial_dependence",
"plot_objective",
"plot_optima",
"plot_performance",
"prob_to_elo",
"reduce_ranges",
"roundflat",
Expand Down
1 change: 1 addition & 0 deletions tune/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ def local( # noqa: C901
optimizer=opt,
result_object=result_object,
iterations=np.array(performance)[:, 0],
elos=np.array(performance)[:, 1:],
optima=np.array(optima),
plot_path=settings.get("plot_path", plot_path),
parameter_names=list(param_ranges.keys()),
Expand Down
22 changes: 17 additions & 5 deletions tune/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from skopt.space import Categorical, Dimension, Integer, Real, Space
from skopt.utils import normalize_dimensions

from tune.plots import plot_objective, plot_objective_1d, plot_optima
from tune.plots import plot_objective, plot_objective_1d, plot_optima, plot_performance
from tune.summary import confidence_intervals
from tune.utils import TimeControl, confidence_to_mult, expected_ucb

Expand Down Expand Up @@ -519,6 +519,7 @@ def plot_results(
optimizer: Optimizer,
result_object: OptimizeResult,
iterations: np.ndarray,
elos: np.ndarray,
optima: np.ndarray,
plot_path: str,
parameter_names: Sequence[str],
Expand All @@ -534,6 +535,8 @@ def plot_results(
Result object containing the data and the last fitted model.
iterations : np.ndarray
Array containing the iterations at which optima were collected.
elos : np.ndarray, shape=(n_iterations, 2)
Array containing the estimated Elo of the optima and the standard error.
optima : np.ndarray
Array containing the predicted optimal parameters.
plot_path : str
Expand Down Expand Up @@ -570,21 +573,30 @@ def plot_results(
fig.patch.set_facecolor(dark_gray)
plot_objective(result_object, dimensions=parameter_names, fig=fig, ax=ax)
plotpath = pathlib.Path(plot_path)
plotpath.mkdir(parents=True, exist_ok=True)
full_plotpath = plotpath / f"{timestr}-{len(optimizer.Xi)}.png"
for subdir in ["landscapes", "elo", "optima"]:
(plotpath / subdir).mkdir(parents=True, exist_ok=True)
full_plotpath = plotpath / f"landscapes/landscape-{timestr}-{len(optimizer.Xi)}.png"
dpi = 150 if optimizer.space.n_dims == 1 else 300
plt.savefig(full_plotpath, dpi=dpi, facecolor=dark_gray, **save_params)
logger.info(f"Saving a plot to {full_plotpath}.")
plt.close(fig)

# Now plot the history of optima:
# Plot the history of optima:
fig, ax = plot_optima(
iterations=iterations,
optima=optima,
space=optimizer.space,
parameter_names=parameter_names,
)
full_plotpath = plotpath / f"optima-{timestr}-{len(optimizer.Xi)}.png"
full_plotpath = plotpath / f"optima/optima-{timestr}-{len(optimizer.Xi)}.png"
fig.savefig(full_plotpath, dpi=150, facecolor=dark_gray)
plt.close(fig)

# Plot the predicted Elo performance of the optima:
fig, ax = plot_performance(
performance=np.hstack([iterations[:, None], elos]), confidence=confidence
)
full_plotpath = plotpath / f"elo/elo-{timestr}-{len(optimizer.Xi)}.png"
fig.savefig(full_plotpath, dpi=150, facecolor=dark_gray)
plt.close(fig)

Expand Down
8 changes: 7 additions & 1 deletion tune/plots.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@

from tune.utils import confidence_to_mult, expected_ucb

__all__ = ["partial_dependence", "plot_objective", "plot_objective_1d", "plot_optima"]
__all__ = [
"partial_dependence",
"plot_objective",
"plot_objective_1d",
"plot_optima",
"plot_performance",
]


def _evenly_sample(dim, n_points):
Expand Down

0 comments on commit 88a3452

Please sign in to comment.