diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs index a3e2edafa..1c2ad6e0b 100644 --- a/tokenizers/src/tokenizer/mod.rs +++ b/tokenizers/src/tokenizer/mod.rs @@ -583,7 +583,7 @@ where /// Set the decoder pub fn with_decoder(&mut self, decoder: Option>) -> &mut Self { - self.decoder = decoder.map(|dec|dec.into()); + self.decoder = decoder.map(|dec| dec.into()); self } diff --git a/tokenizers/tests/common/mod.rs b/tokenizers/tests/common/mod.rs index 444cfb269..26129699b 100644 --- a/tokenizers/tests/common/mod.rs +++ b/tokenizers/tests/common/mod.rs @@ -23,7 +23,9 @@ pub fn get_byte_level_bpe() -> BPE { pub fn get_byte_level(add_prefix_space: bool, trim_offsets: bool) -> Tokenizer { let mut tokenizer = Tokenizer::new(get_byte_level_bpe()); tokenizer - .with_pre_tokenizer(Some(ByteLevel::default().add_prefix_space(add_prefix_space))) + .with_pre_tokenizer(Some( + ByteLevel::default().add_prefix_space(add_prefix_space), + )) .with_decoder(Some(ByteLevel::default())) .with_post_processor(Some(ByteLevel::default().trim_offsets(trim_offsets))); diff --git a/tokenizers/tests/documentation.rs b/tokenizers/tests/documentation.rs index 283d53bd5..c0c471a93 100644 --- a/tokenizers/tests/documentation.rs +++ b/tokenizers/tests/documentation.rs @@ -157,15 +157,15 @@ fn quicktour() -> tokenizers::Result<()> { ("[CLS]", tokenizer.token_to_id("[CLS]").unwrap()), ("[SEP]", tokenizer.token_to_id("[SEP]").unwrap()), ]; - tokenizer.with_post_processor( - Some(TemplateProcessing::builder() + tokenizer.with_post_processor(Some( + TemplateProcessing::builder() .try_single("[CLS] $A [SEP]") .unwrap() .try_pair("[CLS] $A [SEP] $B:1 [SEP]:1") .unwrap() .special_tokens(special_tokens) - .build()?), - ); + .build()?, + )); // END quicktour_init_template_processing // START quicktour_print_special_tokens let output = tokenizer.encode("Hello, y'all! How are you 😁 ?", true)?; @@ -330,16 +330,16 @@ fn pipeline() -> tokenizers::Result<()> { // START pipeline_setup_processor use tokenizers::processors::template::TemplateProcessing; - tokenizer.with_post_processor( - Some(TemplateProcessing::builder() + tokenizer.with_post_processor(Some( + TemplateProcessing::builder() .try_single("[CLS] $A [SEP]") .unwrap() .try_pair("[CLS] $A [SEP] $B:1 [SEP]:1") .unwrap() .special_tokens(vec![("[CLS]", 1), ("[SEP]", 2)]) .build() - .unwrap()), - ); + .unwrap(), + )); // END pipeline_setup_processor // START pipeline_test_decoding let output = tokenizer.encode("Hello, y'all! How are you 😁 ?", true)?; @@ -389,16 +389,16 @@ fn train_pipeline_bert() -> tokenizers::Result<()> { // START bert_setup_processor use tokenizers::processors::template::TemplateProcessing; - bert_tokenizer.with_post_processor( - Some(TemplateProcessing::builder() + bert_tokenizer.with_post_processor(Some( + TemplateProcessing::builder() .try_single("[CLS] $A [SEP]") .unwrap() .try_pair("[CLS] $A [SEP] $B:1 [SEP]:1") .unwrap() .special_tokens(vec![("[CLS]", 1), ("[SEP]", 2)]) .build() - .unwrap()), - ); + .unwrap(), + )); // END bert_setup_processor // START bert_train_tokenizer use tokenizers::models::{wordpiece::WordPieceTrainer, TrainerWrapper};