Skip to content

Commit

Permalink
added options to turn on/off logging particular charts, dropped depre…
Browse files Browse the repository at this point in the history
…cated NeptuneMonitor and log_study (#124)
  • Loading branch information
jakubczakon authored Feb 12, 2021
1 parent 1d76f27 commit 1bca817
Showing 1 changed file with 80 additions and 36 deletions.
116 changes: 80 additions & 36 deletions neptunecontrib/monitoring/optuna.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,17 @@ class NeptuneCallback:
Args:
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_charts('bool'): Whether optuna visualization charts should be logged. By default no charts are logged.
log_study('bool'): Whether optuna study object should be pickled and logged. By default it is not.
log_charts('bool'): Deprecated argument. Whether all optuna visualizations plots should be logged.
By default they are not.
To log all the charts set log_charts=True.
If you want to log a particular chart change the argument for that chart explicitly.
For example log_charts=False and log_slice=True will log only the slice plot to Neptune.
log_optimization_history('bool'): Whether optuna optimization history chart should be logged.
By default it is not.
log_contour('bool'): Whether optuna contour plot should be logged. By default it is not.
log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. By default it is not.
log_slice('bool'): Whether optuna slice chart should be logged. By default it is not.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Expand Down Expand Up @@ -57,10 +66,33 @@ class NeptuneCallback:
neptune_callback = opt_utils.NeptuneCallback(log_charts=True, log_study=True)
"""

def __init__(self, experiment=None, log_charts=False, log_study=False, params=None): # pylint: disable=W0621
def __init__(self, experiment=None,
log_study=False,
log_charts=False,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None): # pylint: disable=W0621
self.exp = experiment if experiment else neptune
self.log_charts = log_charts
self.log_study = log_study

if log_charts:

message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)

log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True

self.log_optimization_history = log_optimization_history
self.log_contour = log_contour
self.log_parallel_coordinate = log_parallel_coordinate
self.log_slice = log_slice
self.params = params

def __call__(self, study, trial):
Expand All @@ -70,25 +102,28 @@ def __call__(self, study, trial):
self.exp.log_metric('best_so_far_run_score', study.best_value)
self.exp.log_text('run_parameters', str(trial.params))

if self.log_charts:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=self.exp)
log_chart(name='contour',
chart=vis.plot_contour(study, params=self.params),
experiment=self.exp)
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study, params=self.params),
experiment=self.exp)
log_chart(name='slice',
chart=vis.plot_slice(study, params=self.params),
experiment=self.exp)

if self.log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=self.exp)


def log_study_info(study, experiment=None, log_charts=True, params=None):
if self.log_optimization_history:
log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=self.exp)
if self.log_contour:
log_chart(name='contour', chart=vis.plot_contour(study, params=self.params), experiment=self.exp)
if self.log_parallel_coordinate:
log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=self.params),
experiment=self.exp)
if self.log_slice:
log_chart(name='slice', chart=vis.plot_slice(study, params=self.params), experiment=self.exp)


def log_study_info(study, experiment=None,
log_study=True,
log_charts=True,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
Expand All @@ -98,7 +133,16 @@ def log_study_info(study, experiment=None, log_charts=True, params=None):
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_charts('bool'): Whether optuna visualization charts should be logged. By default all charts are logged.
log_study('bool'): Whether optuna study object should be logged as pickle. Default is True.
log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged.
By default all charts are sent.
To not log any charts set log_charts=False.
If you want to log a particular chart change the argument for that chart explicitly.
For example log_charts=False and log_slice=True will log only the slice plot to Neptune.
log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True.
log_contour('bool'): Whether optuna contour plot should be logged. Default is True.
log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True.
log_slice('bool'): Whether optuna slice chart should be logged. Default is True.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Expand Down Expand Up @@ -130,23 +174,23 @@ def log_study_info(study, experiment=None, log_charts=True, params=None):
_exp.set_property('best_parameters', study.best_params)

if log_charts:
message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)

log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True

if log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
if log_optimization_history:
log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp)
if log_contour:
log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp)
if log_parallel_coordinate:
log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp)
if log_slice:
log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)

pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)


def log_study(study, experiment=None, log_charts=True, params=None):
message = """log_study was renamed to log_study_info and will be removed in future releases.
"""
warnings.warn(message)
return log_study_info(study, experiment, log_charts, params)


def NeptuneMonitor(experiment=None):
message = """NeptuneMonitor was renamed to NeptuneCallback and will be removed in future releases.
"""
warnings.warn(message)
return NeptuneCallback(experiment=experiment)

0 comments on commit 1bca817

Please sign in to comment.