feat: support encode zero value in protobuf to adjust to go proto2

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["pilota", "pilota-build", "pilota-thrift-parser"]
+members = ["pilota", "pilota-build", "pilota-thrift-parser", "examples"]
 resolver = "2"
 
 [profile.bench]

examples/Cargo.toml

@@ -0,0 +1,58 @@
+[package]
+name = "examples"
+version = "0.1.0"
+edition = "2021"
+description = "Compile thrift and protobuf idl into rust code at compile-time."
+homepage = "https://cloudwego.io/docs/pilota/"
+repository = "https://github.com/cloudwego/pilota"
+license = "MIT OR Apache-2.0"
+authors = ["Pilota Team <[email protected]>"]
+keywords = ["serialization", "thrift", "protobuf", "volo"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[badges]
+maintenance = { status = "actively-developed" }
+
+[dependencies]
+pilota = { path = "../pilota", features = ["pb-encode-default-value"] }
+pilota-build = { path = "../pilota-build" }
+pilota-thrift-parser = { path = "../pilota-thrift-parser", version = "0.11" }
+
+ahash = "0.8"
+anyhow = "1"
+dashmap = "5"
+heck = "0.5"
+itertools = "0.13"
+normpath = "1"
+paste = "1"
+petgraph = "0.6"
+phf = { version = "0.11", features = ["macros"] }
+proc-macro2 = "1"
+quote = "1"
+rayon = "1"
+rustc-hash = "1"
+salsa = { version = "0.17.0-pre.2" }
+scoped-tls = "1"
+serde = { version = "1", features = ["derive"] }
+serde_yaml = "0.9"
+syn = "2"
+toml = "0.8"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# The official rust-protobuf parser currently has some bug.
+# We will switch to the official one when https://github.com/stepancheg/rust-protobuf/pull/646 is fixed.
+protobuf-parse = { package = "protobuf-parse2", version = "4.0.0-alpha.4" }
+protobuf = { package = "protobuf2", version = "4.0.0-alpha.2" }
+faststr = "0.2"
+
+[dev-dependencies]
+
+tokio = { version = "1", features = ["io-util"] }
+derivative = "2"
+tempfile = "3"
+diffy = "0.4"
+criterion = { version = "0.5", features = ["html_reports"] }
+rand = "0.8"
+linkedbytes = "0.1"

examples/idl/zero_value.proto

@@ -0,0 +1,5 @@
+message A {
+    map<string, string> str_map = 1;
+    required string s1 = 2;
+    optional string s2 = 3;
+}

examples/src/lib.rs

@@ -0,0 +1,33 @@
+use pilota::prost::Message as _;
+
+mod zero_value;
+
+#[test]
+fn test_pb_encode_zero_value() {
+    let test_data = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("idl")
+        .join("zero_value.proto");
+
+    let out_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("src")
+        .join("zero_value.rs");
+
+    pilota_build::Builder::protobuf()
+        .ignore_unused(false)
+        .include_dirs(vec![test_data.parent().unwrap().to_path_buf()])
+        .compile_with_config(
+            vec![pilota_build::IdlService::from_path(test_data.to_path_buf())],
+            pilota_build::Output::File(out_path.into()),
+        );
+
+    let mut a = zero_value::zero_value::A::default();
+
+    a.str_map.insert("key1".into(), "value".into());
+    a.str_map.insert("key2".into(), "".into());
+
+    let mut buf = pilota::BytesMut::new();
+    a.encode(&mut buf).unwrap();
+
+    println!("{:?}", buf);
+    println!("{:?}", buf.freeze().as_ref());
+}

examples/src/zero_value.rs

@@ -0,0 +1,102 @@
+pub mod zero_value {
+    #![allow(warnings, clippy::all)]
+    #[derive(Debug, Default, Clone, PartialEq)]
+    pub struct A {
+        pub str_map: ::pilota::AHashMap<::pilota::FastStr, ::pilota::FastStr>,
+
+        pub s1: ::pilota::FastStr,
+
+        pub s2: ::std::option::Option<::pilota::FastStr>,
+    }
+    impl ::pilota::prost::Message for A {
+        #[inline]
+        fn encoded_len(&self) -> usize {
+            0 + ::pilota::prost::encoding::hash_map::encoded_len(
+                ::pilota::prost::encoding::faststr::encoded_len,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                1,
+                &self.str_map,
+            ) + ::pilota::prost::encoding::faststr::encoded_len(2, &self.s1)
+                + self.s2.as_ref().map_or(0, |value| {
+                    ::pilota::prost::encoding::faststr::encoded_len(3, value)
+                })
+        }
+
+        #[allow(unused_variables)]
+        fn encode_raw<B>(&self, buf: &mut B)
+        where
+            B: ::pilota::prost::bytes::BufMut,
+        {
+            ::pilota::prost::encoding::hash_map::encode(
+                ::pilota::prost::encoding::faststr::encode,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                ::pilota::prost::encoding::faststr::encode,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                1,
+                &self.str_map,
+                buf,
+            );
+            ::pilota::prost::encoding::faststr::encode(2, &self.s1, buf);
+            if let Some(_pilota_inner_value) = self.s2.as_ref() {
+                ::pilota::prost::encoding::faststr::encode(3, _pilota_inner_value, buf);
+            };
+        }
+
+        #[allow(unused_variables)]
+        fn merge_field<B>(
+            &mut self,
+            tag: u32,
+            wire_type: ::pilota::prost::encoding::WireType,
+            buf: &mut B,
+            ctx: ::pilota::prost::encoding::DecodeContext,
+        ) -> ::core::result::Result<(), ::pilota::prost::DecodeError>
+        where
+            B: ::pilota::prost::bytes::Buf,
+        {
+            const STRUCT_NAME: &'static str = stringify!(A);
+            match tag {
+                1 => {
+                    let mut _inner_pilota_value = &mut self.str_map;
+                    ::pilota::prost::encoding::hash_map::merge(
+                        ::pilota::prost::encoding::faststr::merge,
+                        ::pilota::prost::encoding::faststr::merge,
+                        &mut _inner_pilota_value,
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(str_map));
+                        error
+                    })
+                }
+                2 => {
+                    let mut _inner_pilota_value = &mut self.s1;
+                    ::pilota::prost::encoding::faststr::merge(
+                        wire_type,
+                        _inner_pilota_value,
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(s1));
+                        error
+                    })
+                }
+                3 => {
+                    let mut _inner_pilota_value = &mut self.s2;
+                    ::pilota::prost::encoding::faststr::merge(
+                        wire_type,
+                        _inner_pilota_value.get_or_insert_with(::core::default::Default::default),
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(s2));
+                        error
+                    })
+                }
+                _ => ::pilota::prost::encoding::skip_field(wire_type, tag, buf, ctx),
+            }
+        }
+    }
+}

pilota/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pilota"
-version = "0.11.3"
+version = "0.11.4"
 edition = "2021"
 description = "Pilota is a thrift and protobuf implementation in pure rust with high performance and extensibility."
 documentation = "https://docs.rs/pilota"
@@ -41,6 +41,7 @@ rand = "0.8"
 
 [features]
 unstable = []
+pb-encode-default-value = []
 
 [[bench]]
 name = "faststr"

pilota/src/prost/encoding.rs

@@ -6,7 +6,7 @@
 extern crate alloc;
 
 use alloc::{collections::BTreeMap, format, string::String, vec::Vec};
 use core::{cmp::min, convert::TryFrom, mem, str, u32, usize};
 
 use ::bytes::{Buf, BufMut, Bytes};
 
@@ -185,7 +185,7 @@
 /// The context should be passed by value and can be freely cloned. When passing
 /// to a function which is decoding a nested object, then use `enter_recursion`.
 #[derive(Clone, Debug)]
 #[cfg_attr(feature = "no-recursion-limit", derive(Default))]
 pub struct DecodeContext {
     /// How many times we can recurse in the current decode stack before we hit
     /// the recursion limit.
@@ -193,11 +193,11 @@
     /// The recursion limit is defined by `RECURSION_LIMIT` and cannot be
     /// customized. The recursion limit can be ignored by building the Prost
     /// crate with the `no-recursion-limit` feature.
     #[cfg(not(feature = "no-recursion-limit"))]
     recurse_count: u32,
 }
 
 #[cfg(not(feature = "no-recursion-limit"))]
 impl Default for DecodeContext {
     #[inline]
     fn default() -> DecodeContext {
@@ -214,7 +214,7 @@
     /// `DecodeContext` to be used at the next level of recursion. Continue
     /// to use the old context
     // at the previous level of recursion.
     #[cfg(not(feature = "no-recursion-limit"))]
     #[inline]
     pub(crate) fn enter_recursion(&self) -> DecodeContext {
         DecodeContext {
@@ -222,7 +222,7 @@
         }
     }
 
     #[cfg(feature = "no-recursion-limit")]
     #[inline]
     pub(crate) fn enter_recursion(&self) -> DecodeContext {
         DecodeContext {}
@@ -233,7 +233,7 @@
     ///
     /// Returns `Ok<()>` if it is ok to continue recursing.
     /// Returns `Err<DecodeError>` if the recursion limit has been reached.
     #[cfg(not(feature = "no-recursion-limit"))]
     #[inline]
     pub(crate) fn limit_reached(&self) -> Result<(), DecodeError> {
         if self.recurse_count == 0 {
@@ -243,7 +243,7 @@
         }
     }
 
     #[cfg(feature = "no-recursion-limit")]
     #[inline]
     #[allow(clippy::unnecessary_wraps)] // needed in other features
     pub(crate) fn limit_reached(&self) -> Result<(), DecodeError> {
@@ -952,7 +952,7 @@
         // guard is used.
         unsafe {
             struct DropGuard<'a>(&'a mut Vec<u8>);
             impl<'a> Drop for DropGuard<'a> {
                 #[inline]
                 fn drop(&mut self) {
                     self.0.clear();
@@ -1579,8 +1579,14 @@
             VL: Fn(u32, &V) -> usize,
         {
             for (key, val) in values.iter() {
-                let skip_key = key == &K::default();
-                let skip_val = val == val_default;
+                let mut skip_key = key == &K::default();
+                let mut skip_val = val == val_default;
+
+                #[cfg(feature = "pb-encode-zero-value")]
+                {
+                    skip_key = false;
+                    skip_val = false;
+                }
 
                 let len = (if skip_key { 0 } else { key_encoded_len(1, key) })
                     + (if skip_val { 0 } else { val_encoded_len(2, val) });