From 50522e7cbee84c68f291bbd47fe2d2a9350c917d Mon Sep 17 00:00:00 2001 From: Niels Date: Thu, 18 Apr 2024 21:24:45 +0200 Subject: [PATCH] Remove attribute --- src/transformers/models/gemma/tokenization_gemma.py | 4 ---- src/transformers/models/llama/tokenization_llama.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/transformers/models/gemma/tokenization_gemma.py b/src/transformers/models/gemma/tokenization_gemma.py index ab19ee23c794a2..0cc82d078c2844 100644 --- a/src/transformers/models/gemma/tokenization_gemma.py +++ b/src/transformers/models/gemma/tokenization_gemma.py @@ -77,8 +77,6 @@ class GemmaTokenizer(PreTrainedTokenizer): extra spaces. use_default_system_prompt (`bool`, *optional*, defaults to `False`): Whether or not the default system prompt for Gemma should be used. - spaces_between_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not to add spaces between special tokens. """ vocab_files_names = VOCAB_FILES_NAMES @@ -96,7 +94,6 @@ def __init__( add_eos_token=False, clean_up_tokenization_spaces=False, use_default_system_prompt=False, - spaces_between_special_tokens=False, **kwargs, ): self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs @@ -123,7 +120,6 @@ def __init__( sp_model_kwargs=self.sp_model_kwargs, clean_up_tokenization_spaces=clean_up_tokenization_spaces, use_default_system_prompt=use_default_system_prompt, - spaces_between_special_tokens=spaces_between_special_tokens, **kwargs, ) diff --git a/src/transformers/models/llama/tokenization_llama.py b/src/transformers/models/llama/tokenization_llama.py index 744e2e3fe2c280..3626cb44736e22 100644 --- a/src/transformers/models/llama/tokenization_llama.py +++ b/src/transformers/models/llama/tokenization_llama.py @@ -95,8 +95,6 @@ class LlamaTokenizer(PreTrainedTokenizer): extra spaces. use_default_system_prompt (`bool`, *optional*, defaults to `False`): Whether or not the default system prompt for Llama should be used. - spaces_between_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not to add spaces between special tokens. legacy (`bool`, *optional*): Whether or not the `legacy` behavior of the tokenizer should be used. Legacy is before the merge of #24622 and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple @@ -140,7 +138,6 @@ def __init__( add_eos_token=False, clean_up_tokenization_spaces=False, use_default_system_prompt=False, - spaces_between_special_tokens=False, legacy=None, add_prefix_space=True, **kwargs, @@ -179,7 +176,6 @@ def __init__( sp_model_kwargs=self.sp_model_kwargs, clean_up_tokenization_spaces=clean_up_tokenization_spaces, use_default_system_prompt=use_default_system_prompt, - spaces_between_special_tokens=spaces_between_special_tokens, legacy=legacy, add_prefix_space=add_prefix_space, **kwargs,