Set tp=1 where MPU is None for bf16 optimizer

microsoft · Mar 8, 2024 · 782642d · 782642d
1 parent 283eda8
commit 782642d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/deepspeed/runtime/bf16_optimizer.py b/deepspeed/runtime/bf16_optimizer.py
@@ -463,7 +463,7 @@ def _load_hp_checkpoint_state(self, checkpoint_dir):
         else:
             tp_world_size = self.mpu.get_slice_parallel_world_size() if hasattr(self.mpu, "get_slice_parallel_world_size") \
                 else self.mpu.get_tensor_model_parallel_world_size()
-        #tp_world_size = self.mpu.get_slice_parallel_world_size()
+
 
         for i, _ in enumerate(self.optimizer.param_groups):
             for lp in self.bf16_groups[i]: