add tests

huggingface · Aug 6, 2024 · 021eec2 · 021eec2
1 parent a010f6b
commit 021eec2
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 0 deletions.
diff --git a/tokenizers/src/decoders/mod.rs b/tokenizers/src/decoders/mod.rs
@@ -96,4 +96,11 @@ mod tests {
         let json = r#"{"type":"Sequence","decoders":[{},{"type":"Metaspace","replacement":"▁","prepend_scheme":"always"}]}"#;
         assert!(serde_json::from_str::<DecoderWrapper>(json).is_err());
     }
+
+
+    #[test]
+    fn decoder_deserialization_no_type() {
+        let json = r#"{"replacement":"▁","prepend_scheme":"always"}"#;
+        assert!(serde_json::from_str::<DecoderWrapper>(json).is_err());
+    }
 }
diff --git a/tokenizers/src/pre_tokenizers/mod.rs b/tokenizers/src/pre_tokenizers/mod.rs
@@ -144,4 +144,10 @@ mod tests {
             PreTokenizerWrapper::WhitespaceSplit(WhitespaceSplit {})
         );
     }
+
+    #[test]
+    fn pre_tokenizer_deserialization_no_type() {
+        let json = r#"{"replacement":"▁","add_prefix_space":true, "prepend_scheme":"always"}}"#;
+        assert!(serde_json::from_str::<PreTokenizerWrapper>(json).is_err());
+    }
 }
diff --git a/tokenizers/src/processors/mod.rs b/tokenizers/src/processors/mod.rs
@@ -87,4 +87,10 @@ mod tests {
             PostProcessorWrapper::Bert(bert)
         );
     }
+
+    #[test]
+    fn post_processor_deserialization_no_type() {
+        let json = r#"{"sep":["[SEP]",102],"cls":["[CLS]",101]}}"#;
+        assert!(serde_json::from_str::<PostProcessorWrapper>(json).is_err());
+    }
 }