From 54f98fd91323ce0513f061f87f4680f2a69c32af Mon Sep 17 00:00:00 2001 From: Max Kovalenko <75629718+deepcharm@users.noreply.github.com> Date: Wed, 12 Jun 2024 19:33:18 +0300 Subject: [PATCH] Monitor was always enabled causing performance degradation (#5633) The Boolean expression for the monitor to be enabled was incorrect, as instead of using the `enabled` field, it used the comet configuration object, making the expression always True. This caused performance degradation (we've observed ~10% drop) as it erroneously invoked the events logging flow along with the expensive calculation of `loss.mean().item()`. Co-authored-by: Olatunji Ruwase Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> --- deepspeed/monitor/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/monitor/config.py b/deepspeed/monitor/config.py index d422d3b1b9bb..c4200877089a 100644 --- a/deepspeed/monitor/config.py +++ b/deepspeed/monitor/config.py @@ -140,5 +140,5 @@ class DeepSpeedMonitorConfig(DeepSpeedConfigModel): @root_validator def check_enabled(cls, values): values["enabled"] = values.get("tensorboard").enabled or values.get("wandb").enabled or values.get( - "csv_monitor").enabled or values.get("comet") + "csv_monitor").enabled or values.get("comet").enabled return values