Skip to content

Commit

Permalink
added prefix to binary metrics logging (#47)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakubczakon
jakubczakon authored Aug 21, 2019
1 parent 0c60f7b commit 434ecab
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 45 deletions.
2 changes: 1 addition & 1 deletion docs/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
# The short X.Y version
version = '0.12'
# The full version, including alpha/beta/rc tags
release = '0.12.1'
release = '0.12.2'

# -- General configuration ---------------------------------------------------

Expand Down
98 changes: 56 additions & 42 deletions neptunecontrib/monitoring/metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import sklearn.metrics as sk_metrics


def log_binary_classification_metrics(y_true, y_pred, threshold=0.5, experiment=None):
def log_binary_classification_metrics(y_true, y_pred, threshold=0.5, experiment=None, prefix=''):
"""Creates metric chartsa and calculates classification metrics and logs them to Neptune.
Class-based metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score',
Expand All @@ -44,6 +44,7 @@ def log_binary_classification_metrics(y_true, y_pred, threshold=0.5, experiment=
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
threshold (float): Threshold that calculates a class for class-based metrics. Default is 0.5.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -77,28 +78,29 @@ def log_binary_classification_metrics(y_true, y_pred, threshold=0.5, experiment=

_exp = experiment if experiment else neptune

log_confusion_matrix(y_true, y_pred[:, 1] > threshold, experiment=_exp)
log_classification_report(y_true, y_pred[:, 1] > threshold, experiment=_exp)
log_class_metrics(y_true, y_pred[:, 1] > threshold, experiment=_exp)
log_class_metrics_by_threshold(y_true, y_pred[:, 1], experiment=_exp)
log_roc_auc(y_true, y_pred, experiment=_exp)
log_precision_recall_auc(y_true, y_pred, experiment=_exp)
log_brier_loss(y_true, y_pred[:, 1], experiment=_exp)
log_log_loss(y_true, y_pred, experiment=_exp)
log_ks_statistic(y_true, y_pred, experiment=_exp)
log_cumulative_gain(y_true, y_pred, experiment=_exp)
log_lift_curve(y_true, y_pred, experiment=_exp)
log_prediction_distribution(y_true, y_pred[:, 1], experiment=_exp)


def log_confusion_matrix(y_true, y_pred_class, experiment=None, channel_name='metric_charts'):
log_confusion_matrix(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix)
log_classification_report(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix)
log_class_metrics(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix)
log_class_metrics_by_threshold(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix)
log_roc_auc(y_true, y_pred, experiment=_exp, prefix=prefix)
log_precision_recall_auc(y_true, y_pred, experiment=_exp, prefix=prefix)
log_brier_loss(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix)
log_log_loss(y_true, y_pred, experiment=_exp, prefix=prefix)
log_ks_statistic(y_true, y_pred, experiment=_exp, prefix=prefix)
log_cumulative_gain(y_true, y_pred, experiment=_exp, prefix=prefix)
log_lift_curve(y_true, y_pred, experiment=_exp, prefix=prefix)
log_prediction_distribution(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix)


def log_confusion_matrix(y_true, y_pred_class, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates a confusion matrix figure and logs it in Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -134,18 +136,19 @@ def log_confusion_matrix(y_true, y_pred_class, experiment=None, channel_name='me

fig, ax = plt.subplots()
_plot_confusion_matrix(y_true, y_pred_class, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_classification_report(y_true, y_pred_class, experiment=None, channel_name='metric_charts'):
def log_classification_report(y_true, y_pred_class, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates a figure with classifiction report table and logs it in Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -180,11 +183,11 @@ def log_classification_report(y_true, y_pred_class, experiment=None, channel_nam
_exp = experiment if experiment else neptune

fig = _plot_classification_report(y_true, y_pred_class)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_class_metrics(y_true, y_pred_class, experiment=None):
def log_class_metrics(y_true, y_pred_class, experiment=None, prefix=''):
"""Calculates and logs all class-based metrics to Neptune.
Metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef',
Expand All @@ -195,6 +198,7 @@ def log_class_metrics(y_true, y_pred_class, experiment=None):
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -230,10 +234,10 @@ def log_class_metrics(y_true, y_pred_class, experiment=None):

scores = _class_metrics(y_true, y_pred_class)
for metric_name, score in scores.items():
_exp.log_metric(metric_name, score)
_exp.log_metric(prefix + metric_name, score)


def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None):
def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None, channel_name='metrics_by_threshold', prefix=''):
"""Creates metric/threshold charts for each metric and logs them to Neptune.
Metrics for which charsta re created and logged are: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score',
Expand All @@ -244,6 +248,8 @@ def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None):
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metrics_by_threshold'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -280,18 +286,19 @@ def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None):
figs = _plot_class_metrics_by_threshold(y_true, y_pred_pos)

for fig in figs:
send_figure(fig, channel_name='metrics_by_threshold', experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_roc_auc(y_true, y_pred, experiment=None, channel_name='metric_charts'):
def log_roc_auc(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates and logs ROC AUC curve and ROCAUC score to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -326,22 +333,23 @@ def log_roc_auc(y_true, y_pred, experiment=None, channel_name='metric_charts'):
_exp = experiment if experiment else neptune

roc_auc = sk_metrics.roc_auc_score(y_true, y_pred[:, 1])
_exp.log_metric('roc_auc', roc_auc)
_exp.log_metric(prefix + 'roc_auc', roc_auc)

fig, ax = plt.subplots()
plt_metrics.plot_roc(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metric_charts'):
def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates and logs Precision Recall curve and Average precision score to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -376,21 +384,22 @@ def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metr
_exp = experiment if experiment else neptune

avg_precision = sk_metrics.average_precision_score(y_true, y_pred[:, 1])
_exp.log_metric('avg_precision', avg_precision)
_exp.log_metric(prefix + 'avg_precision', avg_precision)

fig, ax = plt.subplots()
plt_metrics.plot_precision_recall(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_brier_loss(y_true, y_pred_pos, experiment=None):
def log_brier_loss(y_true, y_pred_pos, experiment=None, prefix=''):
"""Calculates and logs brier loss to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -425,16 +434,17 @@ def log_brier_loss(y_true, y_pred_pos, experiment=None):
_exp = experiment if experiment else neptune

brier = sk_metrics.brier_score_loss(y_true, y_pred_pos)
_exp.log_metric('brier_loss', brier)
_exp.log_metric(prefix + 'brier_loss', brier)


def log_log_loss(y_true, y_pred, experiment=None):
def log_log_loss(y_true, y_pred, experiment=None, prefix=''):
"""Creates and logs Precision Recall curve and Average precision score to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -469,10 +479,10 @@ def log_log_loss(y_true, y_pred, experiment=None):
_exp = experiment if experiment else neptune

log_loss = sk_metrics.log_loss(y_true, y_pred)
_exp.log_metric('log_loss', log_loss)
_exp.log_metric(prefix + 'log_loss', log_loss)


def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_charts'):
def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates and logs KS statistics curve and KS statistics score to Neptune.
Kolmogorov-Smirnov statistics chart can be calculated for true positive rates (TPR) and true negative rates (TNR)
Expand All @@ -484,6 +494,7 @@ def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_chart
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -519,22 +530,23 @@ def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_chart

res = binary_ks_curve(y_true, y_pred[:, 1])
ks_stat = res[3]
_exp.log_metric('ks_statistic', ks_stat)
_exp.log_metric(prefix + 'ks_statistic', ks_stat)

fig, ax = plt.subplots()
plt_metrics.plot_ks_statistic(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_cumulative_gain(y_true, y_pred, experiment=None, channel_name='metric_charts'):
def log_cumulative_gain(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates cumulative gain chart and logs it to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -570,18 +582,19 @@ def log_cumulative_gain(y_true, y_pred, experiment=None, channel_name='metric_ch

fig, ax = plt.subplots()
plt_metrics.plot_cumulative_gain(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_lift_curve(y_true, y_pred, experiment=None, channel_name='metric_charts'):
def log_lift_curve(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''):
"""Creates cumulative gain chart and logs it to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -617,18 +630,19 @@ def log_lift_curve(y_true, y_pred, experiment=None, channel_name='metric_charts'

fig, ax = plt.subplots()
plt_metrics.plot_lift_curve(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


def log_prediction_distribution(y_true, y_pred_pos, experiment=None, channel_name='metric_charts'):
def log_prediction_distribution(y_true, y_pred_pos, experiment=None, channel_name='metric_charts', prefix=''):
"""Generates prediction distribution plot from predictions and true labels.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
Expand Down Expand Up @@ -658,7 +672,7 @@ def log_prediction_distribution(y_true, y_pred_pos, experiment=None, channel_nam

fig, ax = plt.subplots()
_plot_prediction_distribution(y_true, y_pred_pos, ax=ax)
send_figure(fig, channel_name=channel_name, experiment=_exp)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()


Expand Down
2 changes: 1 addition & 1 deletion neptunecontrib/versioning/data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def log_data_version(path, prefix='', experiment=None):
Args:
path(str): path to the file or directory,
prefix(str): Prefix that will be added before 'ata_version' and 'data_path'
prefix(str): Prefix that will be added before 'data_version' and 'data_path'
experiment(neptune.experiemnts.Experiment or None): if the data should be logged to a particular
neptune experiment it can be passed here. By default it is logged to the current experiment.
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def main():

setup(
name='neptune-contrib',
version='0.12.1',
version='0.12.2',
description='Neptune Python library contributions',
author='neptune.ml',
author_email='[email protected]',
Expand Down

0 comments on commit 434ecab

Please sign in to comment.