fmt

huggingface · Aug 6, 2024 · 3b37a78 · 3b37a78
1 parent bca9bde
commit 3b37a78
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 29 deletions.
diff --git a/tokenizers/src/normalizers/byte_level.rs b/tokenizers/src/normalizers/byte_level.rs
@@ -1,7 +1,7 @@
 use crate::processors::byte_level::bytes_char;
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
-use std::collections::{HashMap, HashSet};
 use crate::utils::macro_rules_attribute;
+use std::collections::{HashMap, HashSet};
 
 #[derive(Clone, Debug)]
 #[macro_rules_attribute(impl_serde_type!)]

diff --git a/tokenizers/src/normalizers/mod.rs b/tokenizers/src/normalizers/mod.rs
@@ -78,33 +78,29 @@ impl_enum_from!(ByteLevel, NormalizerWrapper, ByteLevel);
 mod tests {
     use super::*;
     #[test]
-        fn post_processor_deserialization_no_type() {
-            let json = r#"{"strip_left":false, "strip_right":true}"#;
-            let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
-            assert!(matches!(
-                reconstructed.unwrap(),
-                NormalizerWrapper::StripNormalizer(_)
-            ));
+    fn post_processor_deserialization_no_type() {
+        let json = r#"{"strip_left":false, "strip_right":true}"#;
+        let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
+        assert!(matches!(
+            reconstructed.unwrap(),
+            NormalizerWrapper::StripNormalizer(_)
+        ));
 
-            let json =
-                r#"{"sep":["</s>",2], "cls":["<s>",0], "trim_offsets":true, "add_prefix_space":true}"#;
-            let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
-            match reconstructed {
-                Err(err) => assert_eq!(
-                    err.to_string(),
-                    "data did not match any variant of untagged enum NormalizerWrapper"
-                ),
-                _ => panic!("Expected an error here"),
-            }
-
-            let json = r#"{"type":"RobertaProcessing", "sep":["</s>",2] }"#;
-            let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
-            match reconstructed {
-                Err(err) => assert_eq!(
-                    err.to_string(),
-                    "data did not match any variant of untagged enum NormalizerWrapper"
-                ),
-                _ => panic!("Expected an error here"),
-            }
+        let json = r#"{"trim_offsets":true, "add_prefix_space":true}"#;
+        let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
+        match reconstructed {
+            Err(err) => assert_eq!(
+                err.to_string(),
+                "data did not match any variant of untagged enum NormalizerWrapper"
+            ),
+            _ => panic!("Expected an error here"),
         }
-    }
+
+        let json = r#"{"prepend":"a"}"#;
+        let reconstructed = serde_json::from_str::<NormalizerWrapper>(json);
+        assert!(matches!(
+            reconstructed.unwrap(),
+            NormalizerWrapper::Prepend(_)
+        ));
+    }
+}