diff --git a/neptunecontrib/monitoring/skopt.py b/neptunecontrib/monitoring/skopt.py index 9a29527..bb93f42 100644 --- a/neptunecontrib/monitoring/skopt.py +++ b/neptunecontrib/monitoring/skopt.py @@ -13,46 +13,58 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -import tempfile +import warnings import matplotlib.pyplot as plt import neptune import numpy as np import skopt.plots as sk_plots +from skopt.utils import dump from neptunecontrib.monitoring.utils import axes2fig -class NeptuneMonitor: +class NeptuneCallback: """Logs hyperparameter optimization process to Neptune. + Specifically using NeptuneCallback will log: run metrics and run parameters, best run metrics so far, and + the current results checkpoint. + Examples: - Initialize NeptuneMonitor:: + Initialize NeptuneCallback:: import neptune import neptunecontrib.monitoring.skopt as sk_utils - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') + neptune.init(api_token='ANONYMOUS', + project_qualified_name='shared/showroom') - monitor = sk_utils.NeptuneMonitor() + neptune.create_experiment(name='optuna sweep') - Run skopt training passing monitor as a a callback:: + neptune_callback = sk_utils.NeptuneCallback() + + Run skopt training passing neptune_callback as a callback:: ... - results = skopt.forest_minimize(objective, space, callback=[monitor], + results = skopt.forest_minimize(objective, space, callback=[neptune_callback], base_estimator='ET', n_calls=100, n_random_starts=10) + You can explore an example experiment in Neptune: + https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1065/logs """ - def __init__(self, experiment=None): + def __init__(self, experiment=None, log_checkpoint=True): self._exp = experiment if experiment else neptune + self.log_checkpoint = log_checkpoint self._iteration = 0 def __call__(self, res): self._exp.log_metric('run_score', x=self._iteration, y=res.func_vals[-1]) self._exp.log_metric('best_so_far_run_score', x=self._iteration, y=np.min(res.func_vals)) - self._exp.log_text('run_parameters', x=self._iteration, y=NeptuneMonitor._get_last_params(res)) + self._exp.log_text('run_parameters', x=self._iteration, y=NeptuneCallback._get_last_params(res)) + + if self.log_checkpoint: + self._exp.log_artifact(_export_results_object(res), 'results.pkl') self._iteration += 1 @staticmethod @@ -62,17 +74,19 @@ def _get_last_params(res): return str(named_params) -def log_results(results, experiment=None): +def log_results(results, experiment=None, log_plots=True, log_pickle=True): """Logs runs results and parameters to neptune. - Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' channel), - best parameters ('best_parameters' property), convergence plot ('diagnostics' channel), - evaluations plot ('diagnostics' channel), and objective plot ('diagnostics' channel). + Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), + best parameters ('best_parameters' property), convergence plot ('diagnostics' log), + evaluations plot ('diagnostics' log), and objective plot ('diagnostics' log). Args: results('scipy.optimize.OptimizeResult'): Results object that is typically an output of the function like `skopt.forest_minimize(...)` experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. + log_plots: ('bool'): If True skopt plots will be logged to Neptune. + log_pickle: ('bool'): if True pickled skopt results object will be logged to Neptune. Examples: Run skopt training:: @@ -81,247 +95,96 @@ def log_results(results, experiment=None): results = skopt.forest_minimize(objective, space, base_estimator='ET', n_calls=100, n_random_starts=10) - Send best parameters to neptune:: - - import neptune - import neptunecontrib.monitoring.skopt as sk_utils - - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - - sk_utils.log_results(results) - - """ - log_best_score(results, experiment) - log_best_parameters(results, experiment) - log_plot_convergence(results, experiment, channel_name='diagnostics') - log_plot_evaluations(results, experiment, channel_name='diagnostics') - log_plot_objective(results, experiment, channel_name='diagnostics') - - -def log_runs(results, experiment=None): - """Logs runs results and parameters to neptune. - - Text channel `hyperparameter_search_score` is created and a list of tuples (name, value) - of best paramters is logged to neptune. - - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. - - Examples: - Run skopt training:: - - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) - - Send best parameters to neptune:: + Initialize Neptune:: import neptune - import neptunecontrib.monitoring.skopt as sk_utils - - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - - sk_utils.log_runs(results) - - """ - _exp = experiment if experiment else neptune - - for i, (loss, params) in enumerate(zip(results.func_vals, results.x_iters)): - _exp.log_metric('run_score', x=i, y=loss) - _exp.log_metric('best_so_far_run_score', x=i, y=np.min(results.func_vals)) - - named_params = _format_to_named_params(params, results) - _exp.log_text('run_parameters', str(named_params)) - - -def log_best_parameters(results, experiment=None): - """Logs best_parameters list to neptune. - - Text channel `best_parameters` is created and a list of tuples (name, value) - of best paramters is logged to neptune. - - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. + neptune.init(api_token='ANONYMOUS', + project_qualified_name='shared/showroom') + neptune.create_experiment(name='optuna sweep') - Examples: - Run skopt training:: - - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) + Send best parameters to Neptune:: - Send best parameters to neptune:: - - import neptune - import neptunecontrib.monitoring.skopt as sk_utils - - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') + import neptunecontrib.monitoring.skopt as sk_utils - sk_utils.log_best_parameters(results) + sk_utils.log_results(results) - """ + You can explore an example experiment in Neptune: + https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1065/logs + """ _exp = experiment if experiment else neptune - named_params = _format_to_named_params(results.x, results) - _exp.set_property('best_parameters', str(named_params)) - + _log_best_score(results, _exp) + _log_best_parameters(results, _exp) -def log_best_score(results, experiment=None): - """Logs best score to neptune. + if log_plots: + _log_plot_convergence(results, _exp) + _log_plot_evaluations(results, _exp) + _log_plot_regret(results, _exp) + _log_plot_objective(results, _exp) - Metric channel `best_score` is created and the best score is logged to neptune. + if log_pickle: + _log_results_object(results, _exp) - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. - - Examples: - Run skopt training:: - - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) - - Send best parameters to neptune:: - - import neptune - import neptunecontrib.monitoring.skopt as sk_utils - - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - - sk_utils.log_best_score(results) +def NeptuneMonitor(*args, **kwargs): + message = """NeptuneMonitor was renamed to NeptuneCallback and will be removed in future releases. """ - _exp = experiment if experiment else neptune - _exp.log_metric('best_score', results.fun) - - -def log_plot_convergence(results, experiment=None, channel_name='convergence'): - """Logs skopt plot_convergence figure to neptune. + warnings.warn(message) + return NeptuneCallback(*args, **kwargs) - Image channel `convergence` is created and the output of the - plot_convergence function is first covented to `neptune.Image` and - then sent to neptune. - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. +def _log_best_parameters(results, experiment): + named_params = ([(dimension.name, param) for dimension, param in zip(results.space, results.x)]) + experiment.set_property('best_parameters', str(named_params)) - Examples: - Run skopt training:: - - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) - Send skopt plot_convergence figure to neptune:: - - import neptune - import neptunecontrib.monitoring.skopt as sk_utils +def _log_best_score(results, experiment): + experiment.log_metric('best_score', results.fun) - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - sk_utils.log_plot_convergence(results) - - """ - - _exp = experiment if experiment else neptune - - fig, ax = plt.subplots(figsize=(16, 12)) +def _log_plot_convergence(results, experiment, name='diagnostics'): + fig, ax = plt.subplots() sk_plots.plot_convergence(results, ax=ax) + experiment.log_image(name, fig) - with tempfile.NamedTemporaryFile(suffix='.png') as f: - fig.savefig(f.name) - _exp.send_image(channel_name, f.name) - - -def log_plot_evaluations(results, experiment=None, channel_name='evaluations'): - """Logs skopt plot_evaluations figure to neptune. - Image channel `evaluations` is created and the output of the - plot_evaluations function is first covented to `neptune.Image` and - then sent to neptune. - - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. - - Examples: - Run skopt training:: - - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) +def _log_plot_regret(results, experiment, name='diagnostics'): + fig, ax = plt.subplots() + sk_plots.plot_regret(results, ax=ax) + experiment.log_image(name, fig) - Send skopt plot_evaluations figure to neptune:: - - import neptune - import neptunecontrib.monitoring.skopt as sk_utils - - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - - sk_utils.log_plot_evaluations(results) - - """ - _exp = experiment if experiment else neptune +def _log_plot_evaluations(results, experiment, name='diagnostics'): fig = plt.figure(figsize=(16, 12)) fig = axes2fig(sk_plots.plot_evaluations(results, bins=10), fig=fig) + experiment.log_image(name, fig) - with tempfile.NamedTemporaryFile(suffix='.png') as f: - fig.savefig(f.name) - _exp.send_image(channel_name, f.name) - - -def log_plot_objective(results, experiment=None, channel_name='objective'): - """Logs skopt plot_objective figure to neptune. - - Image channel `objective` is created and the output of the - plot_objective function is first covented to `neptune.Image` and - then sent to neptune. - - Args: - results('scipy.optimize.OptimizeResult'): Results object that is typically an - output of the function like `skopt.forest_minimize(...)` - experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. - - Examples: - Run skopt training:: - ... - results = skopt.forest_minimize(objective, space, - base_estimator='ET', n_calls=100, n_random_starts=10) +def _log_plot_objective(results, experiment, name='diagnostics'): + try: + fig = plt.figure(figsize=(16, 12)) + fig = axes2fig(sk_plots.plot_objective(results), fig=fig) + experiment.log_image(name, fig) + except Exception as e: + print('Could not create the objective chart due to error: {}'.format(e)) - Send skopt plot_objective figure to neptune:: - import neptune - import neptunecontrib.monitoring.skopt as sk_utils +def _log_results_object(results, experiment=None): + experiment.log_artifact(_export_results_object(results), 'results.pkl') - neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') - sk_utils.log_plot_objective(results) +def _export_results_object(results): + from io import BytesIO - """ + results.specs['args'].pop('callback', None) - _exp = experiment if experiment else neptune - fig = plt.figure(figsize=(16, 12)) + buffer = BytesIO() + dump(results, buffer, store_objective=False) + buffer.seek(0) - try: - fig = axes2fig(sk_plots.plot_objective(results), fig=fig) - with tempfile.NamedTemporaryFile(suffix='.png') as f: - fig.savefig(f.name) - _exp.send_image(channel_name, f.name) - except Exception as e: - print('Could not create ans objective chart due to error: {}'.format(e)) + return buffer def _format_to_named_params(params, result): - return ([(dimension.name, param) for dimension, param in zip(result.space, params)]) + return [(dimension.name, param) for dimension, param in zip(result.space, params)] diff --git a/setup.py b/setup.py index 854acd3..4c43d9b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def main(): extras = { 'bots': ['python-telegram-bot'], 'hpo': ['scikit-optimize>=0.5.2', 'scipy'], - 'monitoring': ['scikit-optimize>=0.5.2', 'sacred>=0.7.5', 'scikit-learn>=0.21.3', + 'monitoring': ['scikit-optimize>=0.7.4', 'sacred>=0.7.5', 'scikit-learn>=0.21.3', 'scikit-plot>=0.3.7', 'seaborn>=0.8.1', 'aif360>=0.2.1', 'xgboost>=0.82', 'graphviz>=0.13.2'], 'versioning': ['boto3', 'numpy'],