legacy to init the slow tokenizer when converting from slow was wrong (…

…#30972)
huggingface · May 22, 2024 · 1d568df · 1d568df
1 parent 1432f64
commit 1d568df
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/src/transformers/models/llama/tokenization_llama_fast.py b/src/transformers/models/llama/tokenization_llama_fast.py
@@ -151,9 +151,6 @@ def __init__(
         self.legacy = legacy
 
         if add_prefix_space is not None:
-            logger.warning_once(
-                "You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers"
-            )
             kwargs["from_slow"] = True
 
         super().__init__(
@@ -166,6 +163,7 @@ def __init__(
             add_bos_token=add_bos_token,
             add_eos_token=add_eos_token,
             use_default_system_prompt=use_default_system_prompt,
+            legacy=legacy,
             **kwargs,
         )
         self._add_bos_token = add_bos_token