diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index bf8457309feea8..683b519a28cd1e 100755
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -349,7 +349,7 @@ def check_support_param_buffer_assignment(model_to_load, state_dict, start_prefi
         return False
 
     # Some models explicitly do not support param buffer assignment
-    if not getattr(model_to_load, "_supports_param_buffer_assignment", False):
+    if not getattr(model_to_load, "_supports_param_buffer_assignment", True):
         logger.debug(
             f"{model_to_load.__class__.__name__} does not support param buffer assignment, loading will be slower"
         )