Add fuse_unk option to SentencePieceBPETokenizer (#574)

* Add fuse_unk option to SentencePieceBPETokenizer * Fix style Co-authored-by: Anthony MOI <[email protected]>
huggingface · Jan 12, 2021 · b9c6bea · b9c6bea
1 parent 91dae1d
commit b9c6bea
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
@@ -20,9 +20,12 @@ def __init__(
         replacement: str = "▁",
         add_prefix_space: bool = True,
         dropout: Optional[float] = None,
+        fuse_unk: Optional[bool] = False,
     ):
         if vocab is not None and merges is not None:
-            tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token))
+            tokenizer = Tokenizer(
+                BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)
+            )
         else:
             tokenizer = Tokenizer(BPE())