final fix

huggingface · Sep 29, 2023 · dbbdf3f · dbbdf3f
1 parent 4c23123
commit dbbdf3f
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
@@ -206,9 +206,7 @@ def __init__(
 
         # make sure that tokens made of several
         # characters are not split at tokenization
-        for token in self.encoder.keys():
-            if len(token) > 1:
-                self.add_tokens(AddedToken(token, rstrip=True, lstrip=True, normalized=False))
+        self.add_tokens([token for token in self.encoder.keys() if len(token) > 1])
 
     def set_target_lang(self, target_lang: str):
         """
@@ -227,9 +225,7 @@ def set_target_lang(self, target_lang: str):
 
         # make sure that tokens made of several
         # characters are not split at tokenization
-        for token in self.encoder.keys():
-            if len(token) > 1:
-                self.add_tokens(token)
+        self.add_tokens([token for token in self.encoder.keys() if len(token) > 1])
 
     @property
     def word_delimiter_token(self) -> str: