more general approach

huggingface · May 30, 2024 · 76b89ee · 76b89ee
1 parent a47a209
commit 76b89ee
Showing 1 changed file with 0 additions and 4 deletions.
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
@@ -1391,10 +1391,6 @@ def tokenizer(self, proto):
                     AddedToken(self.original_tokenizer.convert_ids_to_tokens(2), normalized=False, special=True),
                 ]
             )
-            user_defined_symbols = [
-                AddedToken(token, normalized=True, special=False) for token in proto.trainer_spec.user_defined_symbols
-            ]
-            tokenizer.add_tokens(user_defined_symbols)
         else:
             raise Exception(
                 "You're trying to run a `Unigram` model but you're file was trained with a different algorithm"