[Tokenizer] Fix slow and fast serialization

.circleci/create_circleci_config.py

@@ -127,6 +127,7 @@ def to_dict(self):
             },
         ]
         steps.extend([{"run": l} for l in self.install_steps])
+        steps.extend([{"run": "pip install pytest-subtests"}])
         steps.append(
             {
                 "save_cache": {

src/transformers/models/camembert/tokenization_camembert.py

@@ -145,9 +145,9 @@ def __init__(
         # In this case it is recommended to properly set the tokens by hand.
         self._added_tokens_decoder = {
             0: AddedToken("<s>NOTUSED"),
-            1: AddedToken(pad_token),
+            1: AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token,
             2: AddedToken("</s>NOTUSED"),
-            3: AddedToken(unk_token),
+            3: AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token,
             4: AddedToken("<unk>NOTUSED"),
         }
 

src/transformers/models/deberta_v2/tokenization_deberta_v2.py

@@ -138,7 +138,11 @@ def __init__(
         self._tokenizer = SPMTokenizer(
             vocab_file, None, split_by_punct=split_by_punct, sp_model_kwargs=self.sp_model_kwargs
         )
-        unk_token = AddedToken(unk_token, normalized=True, lstrip=False, rstrip=False)
+        unk_token = (
+            AddedToken(unk_token, normalized=True, lstrip=False, rstrip=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
         super().__init__(
             do_lower_case=do_lower_case,
             bos_token=bos_token,

src/transformers/models/marian/tokenization_marian.py

@@ -148,9 +148,9 @@ def __init__(
 
         self.separate_vocabs = separate_vocabs
         self.encoder = load_json(vocab)
-        if unk_token not in self.encoder:
+        if str(unk_token) not in self.encoder:
             raise KeyError("<unk> token must be in the vocab")
-        assert pad_token in self.encoder
+        assert str(pad_token) in self.encoder
 
         if separate_vocabs:
             self.target_encoder = load_json(target_vocab_file)

src/transformers/models/nllb/tokenization_nllb.py

@@ -144,7 +144,11 @@ def __init__(
         **kwargs,
     ):
         # Mask token behave like a normal word, i.e. include the space before it
-        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
+        mask_token = (
+            AddedToken(mask_token, normalized=True, lstrip=True, rstrip=False, special=True)
+            if isinstance(mask_token, str)
+            else mask_token
+        )
 
         self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
         self.legacy_behaviour = legacy_behaviour

src/transformers/models/nllb/tokenization_nllb_fast.py

@@ -155,7 +155,11 @@ def __init__(
         **kwargs,
     ):
         # Mask token behave like a normal word, i.e. include the space before it
-        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
+        mask_token = (
+            AddedToken(mask_token, normalized=True, lstrip=True, rstrip=False, special=True)
+            if isinstance(mask_token, str)
+            else mask_token
+        )
         self.legacy_behaviour = legacy_behaviour
 
         _additional_special_tokens = FAIRSEQ_LANGUAGE_CODES.copy()

src/transformers/models/t5/tokenization_t5.py

@@ -153,9 +153,9 @@ def __init__(
         legacy=None,
         **kwargs,
     ) -> None:
-        pad_token = AddedToken(pad_token, rstrip=True, lstrip=True)
-        unk_token = AddedToken(unk_token, rstrip=True, lstrip=True)
-        eos_token = AddedToken(eos_token, rstrip=True, lstrip=True)
+        pad_token = AddedToken(pad_token, rstrip=True, lstrip=True) if isinstance(pad_token, str) else pad_token
+        unk_token = AddedToken(unk_token, rstrip=True, lstrip=True) if isinstance(unk_token, str) else unk_token
+        eos_token = AddedToken(eos_token, rstrip=True, lstrip=True) if isinstance(eos_token, str) else eos_token
 
         self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
 
@@ -167,7 +167,9 @@ def __init__(
 
         if additional_special_tokens is not None:
             extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
-            if extra_ids > 0 and extra_ids != len(extra_tokens):
+            if len(extra_tokens) < 1:
+                additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
+            elif extra_ids > 0 and extra_ids != len(extra_tokens):
                 raise ValueError(
                     f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                     " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"

src/transformers/tokenization_utils.py

@@ -348,19 +348,20 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase):
 
     def __init__(self, **kwargs):
         # 1. Init the parent class
-        super().__init__(**kwargs)
+
         self.tokens_trie = Trie()
 
         # 2. init `_added_tokens_decoder` if child class did not
         if not hasattr(self, "_added_tokens_decoder"):
             self._added_tokens_decoder: Dict[int, AddedToken] = {}
-        # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
-        if "added_tokens_decoder" in kwargs:
-            # overwriting the class's added_tokens_decoder. This is the source of truth!
-            self._added_tokens_decoder.update(kwargs.get("added_tokens_decoder"))
 
+        # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
+        self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))
         self._added_tokens_encoder: Dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}
 
+        # 4 init the parent class
+        super().__init__(**kwargs)
+
         # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
         # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`
         self._add_tokens(self.all_special_tokens_extended, special_tokens=True)
@@ -459,6 +460,7 @@ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_to
         added_tokens = 0
         if new_tokens is None:
             return added_tokens
+        # TODO this is fairly slow to improve!
         current_vocab = self.get_vocab().copy()
         new_idx = len(current_vocab)  # only call this once, len gives the last index + 1
         for token in new_tokens:
@@ -467,9 +469,12 @@ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_to
             if str(token) == "":
                 continue
             if isinstance(token, str):
+                if token in self._added_tokens_encoder:
+                    continue
                 # for legacy AddedTokens strip left and right by default
                 # TODO this will be remove to have the same default behavior as rust
-                token = AddedToken(token, normalized=not special_tokens, rstrip=True, lstrip=True)
+                else:
+                    token = AddedToken(token, normalized=False, rstrip=True, lstrip=True)
             if special_tokens:
                 token.special = True
             if token in self._added_tokens_decoder:
@@ -550,7 +555,7 @@ def tokenize(self, text: TextInput, **kwargs) -> List[str]:
             logger.warning(f"Keyword arguments {kwargs} not recognized.")
 
         if hasattr(self, "do_lower_case") and self.do_lower_case:
-            # convert non-special tokens to lowercase
+            # convert non-special tokens to lowercase. Might be super slow as well?
             escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
             escaped_special_toks += [
                 re.escape(s_tok.content)
@@ -564,7 +569,7 @@ def tokenize(self, text: TextInput, **kwargs) -> List[str]:
             no_split_token = []
             tokens = [text]
         else:
-            no_split_token = set(self._added_tokens_encoder.keys())  # don't split on any of the added tokens
+            no_split_token = self._added_tokens_encoder.keys()  # don't split on any of the added tokens
             # "This is something<special_token_1>  else"
             tokens = self.tokens_trie.split(text)
 
@@ -588,7 +593,6 @@ def tokenize(self, text: TextInput, **kwargs) -> List[str]:
                     elif tok_extended.single_word and right and right[0] != " ":
                         tokens[i + 1] = token + tokens[i + 1]
                         tokens[i] = ""
-
                 else:
                     raise ValueError(
                         f"{tok_extended} cannot be tokenized because it was not properly added"