diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 22eb3def4..6faeb7ad7 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -1452,6 +1452,6 @@ mod test { ]))); let output = crate::utils::serde_pyo3::to_string(&tokenizer).unwrap(); - assert_eq!(output, ""); + assert_eq!(output, "Tokenizer(version=\"1.0\", truncation=None, padding=None, added_tokens=[], normalizer=Sequence(normalizers=[NFKC(), Lowercase()]), pre_tokenizer=None, post_processor=None, decoder=None, model=BPE(dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=False, byte_fallback=False, ignore_merges=False, vocab={}, merges=[]))"); } } diff --git a/bindings/python/src/utils/serde_pyo3.rs b/bindings/python/src/utils/serde_pyo3.rs index 0dc336f2d..3969277d5 100644 --- a/bindings/python/src/utils/serde_pyo3.rs +++ b/bindings/python/src/utils/serde_pyo3.rs @@ -597,11 +597,13 @@ fn test_struct_tagged() { } let u = A { a: true, b: 1 }; - let expected = r#"A(type="A", a=True, b=1)"#; + // let expected = r#"A(type="A", a=True, b=1)"#; + // No we skip all `type` manually inserted variants. + let expected = r#"A(a=True, b=1)"#; assert_eq!(to_string(&u).unwrap(), expected); let u = E::A(A { a: true, b: 1 }); - let expected = r#"A(type="A", a=True, b=1)"#; + let expected = r#"A(a=True, b=1)"#; assert_eq!(to_string(&u).unwrap(), expected); } @@ -626,6 +628,12 @@ fn test_flatten() { d: usize, } + #[derive(Serialize)] + #[serde(transparent)] + struct D { + e: A, + } + let u = B { c: A { a: true, b: 1 }, d: 2, @@ -637,6 +645,15 @@ fn test_flatten() { c: A { a: true, b: 1 }, d: 2, }; - let expected = r#"C(a=True, b=1, d=2)"#; + // XXX This is unfortunate but true, flatten forces the serialization + // to use the serialize_map without any means for the Serializer to know about this + // flattening attempt + let expected = r#"{"a":True, "b":1, "d":2}"#; + assert_eq!(to_string(&u).unwrap(), expected); + + let u = D { + e: A { a: true, b: 1 }, + }; + let expected = r#"A(a=True, b=1)"#; assert_eq!(to_string(&u).unwrap(), expected); }