Do not use mtime for checkpoint rotation. (#28862)

Resolve #26961
huggingface · Feb 6, 2024 · ac51e59 · ac51e59
1 parent 0690116
commit ac51e59
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
@@ -2465,7 +2465,9 @@ def _save_checkpoint(self, model, trial, metrics=None):
 
             # Maybe delete some older checkpoints.
             if self.args.should_save:
-                self._rotate_checkpoints(use_mtime=True, output_dir=run_dir)
+                # Solely rely on numerical checkpoint id for rotation.
+                # mtime is not reliable especially on some fuse fs in cloud environments.
+                self._rotate_checkpoints(use_mtime=False, output_dir=run_dir)
 
         self.args.distributed_state.wait_for_everyone()