Handle when precompiled charsmap is empty (#1308)

* Handle when precompiled charsmap is empty * Black --------- Co-authored-by: Nicolas Patry <[email protected]>
huggingface · Jul 31, 2023 · efea6c7 · efea6c7
1 parent c2664ae
commit efea6c7
Showing 1 changed file with 9 additions and 6 deletions.
diff --git a/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
@@ -173,12 +173,15 @@ def from_spm(filename: str):
 
         tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback))
 
-        tokenizer.normalizer = normalizers.Sequence(
-            [
-                normalizers.Precompiled(precompiled_charsmap),
-                normalizers.Replace(Regex(" {2,}"), " "),
-            ]
-        )
+        if precompiled_charsmap:
+            tokenizer.normalizer = normalizers.Sequence(
+                [
+                    normalizers.Precompiled(precompiled_charsmap),
+                    normalizers.Replace(Regex(" {2,}"), " "),
+                ]
+            )
+        else:
+            tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
         tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
         tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)