don't replace_additional_special_tokens

flairNLP · May 29, 2024 · 65363e8 · 65363e8
1 parent 1037fcc
commit 65363e8
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/flair/embeddings/transformer.py b/flair/embeddings/transformer.py
@@ -1139,7 +1139,9 @@ def is_supported_t5_model(config: PretrainedConfig) -> bool:
         # If we use a context separator, add a new special token
         self.use_context_separator = use_context_separator
         if use_context_separator:
-            added = self.tokenizer.add_special_tokens({"additional_special_tokens": [SENTENCE_BOUNDARY_TAG]})
+            added = self.tokenizer.add_special_tokens(
+                {"additional_special_tokens": [SENTENCE_BOUNDARY_TAG]}, replace_additional_special_tokens=False
+            )
             transformer_model.resize_token_embeddings(transformer_model.config.vocab_size + added)
 
         super().__init__(**self.to_args())