rafiune the test

huggingface · Aug 6, 2024 · 4ecd4d0 · 4ecd4d0
1 parent f306795
commit 4ecd4d0
Showing 1 changed file with 15 additions and 1 deletion.
diff --git a/tokenizers/src/processors/mod.rs b/tokenizers/src/processors/mod.rs
@@ -90,7 +90,7 @@ mod tests {
 
     #[test]
     fn post_processor_deserialization_no_type() {
-        let json = r#"{"sep":["[SEP]",102],"cls":["[CLS]",101]}}"#;
+        let json = r#"{"add_prefix_space": true, "trim_offsets": false, "use_regex": false}"#;
         let reconstructed = serde_json::from_str::<PostProcessorWrapper>(json);
         match reconstructed {
             Err(err) => assert_eq!(
@@ -99,5 +99,19 @@ mod tests {
             ),
             _ => panic!("Expected an error here"),
         }
+
+        let json = r#"{"sep":["[SEP]",102],"cls":["[CLS]",101]}"#;
+        let reconstructed = serde_json::from_str::<PostProcessorWrapper>(json);
+        match reconstructed {
+            Ok(processor) => assert!(matches!(processor, PostProcessorWrapper::Bert(_))),
+            Err(err) => panic!("{:?}",err)
+        }
+
+        let json = r#"{"sep":["</s>",2], "cls":["<s>",0], "trim_offsets":true, "add_prefix_space":true}"#;
+        let reconstructed = serde_json::from_str::<PostProcessorWrapper>(json);
+        match reconstructed {
+            Ok(processor) => assert!(matches!(processor, PostProcessorWrapper::Roberta(_))),
+            Err(err) => panic!("{:?}",err)
+        }
     }
 }