diff --git a/tokenizers/src/processors/mod.rs b/tokenizers/src/processors/mod.rs index c7224e909..5748233d9 100644 --- a/tokenizers/src/processors/mod.rs +++ b/tokenizers/src/processors/mod.rs @@ -90,7 +90,7 @@ mod tests { #[test] fn post_processor_deserialization_no_type() { - let json = r#"{"sep":["[SEP]",102],"cls":["[CLS]",101]}}"#; + let json = r#"{"add_prefix_space": true, "trim_offsets": false, "use_regex": false}"#; let reconstructed = serde_json::from_str::(json); match reconstructed { Err(err) => assert_eq!( @@ -99,5 +99,19 @@ mod tests { ), _ => panic!("Expected an error here"), } + + let json = r#"{"sep":["[SEP]",102],"cls":["[CLS]",101]}"#; + let reconstructed = serde_json::from_str::(json); + match reconstructed { + Ok(processor) => assert!(matches!(processor, PostProcessorWrapper::Bert(_))), + Err(err) => panic!("{:?}",err) + } + + let json = r#"{"sep":["",2], "cls":["",0], "trim_offsets":true, "add_prefix_space":true}"#; + let reconstructed = serde_json::from_str::(json); + match reconstructed { + Ok(processor) => assert!(matches!(processor, PostProcessorWrapper::Roberta(_))), + Err(err) => panic!("{:?}",err) + } } }