proto: move Filter from geyser crate (#466)

.github/workflows/test.yml

@@ -11,6 +11,7 @@ on:
       - 'v1.17'
       - 'v1.18'
       - 'v2.0'
+      - 'v2.1'
   workflow_dispatch:
 
 env:
@@ -86,18 +87,21 @@ jobs:
           command: check advisories
 
       - name: cargo clippy
-        run: cargo clippy --workspace --all-targets --tests #-- --deny=warnings
+        run: cargo clippy --workspace --all-targets
 
       - name: check features in `client`
-        run: cargo check -p yellowstone-grpc-client --all-targets --tests
+        run: cargo check -p yellowstone-grpc-client --all-targets
       - name: check features in `client-simple`
-        run: cargo check -p yellowstone-grpc-client-simple --all-targets --tests
+        run: cargo check -p yellowstone-grpc-client-simple --all-targets
       - name: check features in `geyser`
-        run: cargo check -p yellowstone-grpc-geyser --all-targets --tests
+        run: cargo check -p yellowstone-grpc-geyser --all-targets
       - name: check features in `proto`
-        run: cargo check -p yellowstone-grpc-proto --all-targets --tests
+        run: cargo check -p yellowstone-grpc-proto --all-targets
       - name: check features in `proto`
-        run: cargo check -p yellowstone-grpc-proto --all-targets --tests --all-features
+        run: cargo check -p yellowstone-grpc-proto --all-targets --all-features
+
+      - name: cargo test
+        run: cargo test --all-features
 
       - name: Build
         run: ./ci/cargo-build-test.sh

.gitignore

@@ -3,6 +3,7 @@ test-ledger
 
 # Rust
 target
+yellowstone-grpc-proto/src/bin/raw.rs
 
 # Node.js
 examples/typescript/dist

CHANGELOG.md

@@ -26,6 +26,7 @@ The minor version will be incremented upon a breaking change and the patch versi
 - examples: add progress bar to client tool ([#456](https://github.com/rpcpool/yellowstone-grpc/pull/456))
 - proto: add mod `plugin` with `FilterNames` cache ([#458](https://github.com/rpcpool/yellowstone-grpc/pull/458))
 - proto: move enum Message from geyser crate ([#459](https://github.com/rpcpool/yellowstone-grpc/pull/459))
+- proto: move `Filter` from geyser crate ([#466](https://github.com/rpcpool/yellowstone-grpc/pull/466))
 
 ### Breaking
 

yellowstone-grpc-geyser/config.json

@@ -23,7 +23,7 @@
     "filter_name_size_limit": 32,
     "filter_names_size_limit": 1024,
     "filter_names_cleanup_interval": "1s",
-    "filters": {
+    "filter_limits": {
       "accounts": {
         "max": 1,
         "any": false,

yellowstone-grpc-geyser/src/config.rs

@@ -3,10 +3,10 @@ use {
         GeyserPluginError, Result as PluginResult,
     },
     serde::{de, Deserialize, Deserializer},
-    solana_sdk::pubkey::Pubkey,
-    std::{collections::HashSet, fs::read_to_string, net::SocketAddr, path::Path, time::Duration},
+    std::{fs::read_to_string, net::SocketAddr, path::Path, time::Duration},
     tokio::sync::Semaphore,
     tonic::codec::CompressionEncoding,
+    yellowstone_grpc_proto::plugin::filter::FilterLimits,
 };
 
 #[derive(Debug, Clone, Deserialize)]
@@ -103,8 +103,8 @@ pub struct ConfigGrpc {
     #[serde(default)]
     pub unary_disabled: bool,
     /// Limits for possible filters
-    #[serde(default)]
-    pub filters: ConfigGrpcFilters,
+    #[serde(default, alias = "filters")]
+    pub filter_limits: FilterLimits,
     /// x_token to enforce on connections
     pub x_token: Option<String>,
     /// Filter name size limit
@@ -210,201 +210,27 @@ impl ConfigGrpcCompression {
     }
 }
 
-#[derive(Debug, Default, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFilters {
-    pub accounts: ConfigGrpcFiltersAccounts,
-    pub slots: ConfigGrpcFiltersSlots,
-    pub transactions: ConfigGrpcFiltersTransactions,
-    pub transactions_status: ConfigGrpcFiltersTransactions,
-    pub blocks: ConfigGrpcFiltersBlocks,
-    pub blocks_meta: ConfigGrpcFiltersBlocksMeta,
-    pub entries: ConfigGrpcFiltersEntries,
-}
-
-impl ConfigGrpcFilters {
-    pub fn check_max(len: usize, max: usize) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            len <= max,
-            "Max amount of filters reached, only {} allowed",
-            max
-        );
-        Ok(())
-    }
-
-    pub fn check_any(is_empty: bool, any: bool) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            !is_empty || any,
-            "Broadcast `any` is not allowed, at least one filter required"
-        );
-        Ok(())
-    }
-
-    pub fn check_pubkey_max(len: usize, max: usize) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            len <= max,
-            "Max amount of Pubkeys reached, only {} allowed",
-            max
-        );
-        Ok(())
-    }
-
-    pub fn check_pubkey_reject(pubkey: &Pubkey, set: &HashSet<Pubkey>) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            !set.contains(pubkey),
-            "Pubkey {} in filters not allowed",
-            pubkey
-        );
-        Ok(())
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersAccounts {
-    pub max: usize,
-    pub any: bool,
-    pub account_max: usize,
-    #[serde(deserialize_with = "deserialize_pubkey_set")]
-    pub account_reject: HashSet<Pubkey>,
-    pub owner_max: usize,
-    #[serde(deserialize_with = "deserialize_pubkey_set")]
-    pub owner_reject: HashSet<Pubkey>,
-    pub data_slice_max: usize,
-}
-
-impl Default for ConfigGrpcFiltersAccounts {
-    fn default() -> Self {
-        Self {
-            max: usize::MAX,
-            any: true,
-            account_max: usize::MAX,
-            account_reject: HashSet::new(),
-            owner_max: usize::MAX,
-            owner_reject: HashSet::new(),
-            data_slice_max: usize::MAX,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersSlots {
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub max: usize,
-}
-
-impl Default for ConfigGrpcFiltersSlots {
-    fn default() -> Self {
-        Self { max: usize::MAX }
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersTransactions {
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub max: usize,
-    pub any: bool,
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub account_include_max: usize,
-    #[serde(deserialize_with = "deserialize_pubkey_set")]
-    pub account_include_reject: HashSet<Pubkey>,
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub account_exclude_max: usize,
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub account_required_max: usize,
-}
-
-impl Default for ConfigGrpcFiltersTransactions {
-    fn default() -> Self {
-        Self {
-            max: usize::MAX,
-            any: true,
-            account_include_max: usize::MAX,
-            account_include_reject: HashSet::new(),
-            account_exclude_max: usize::MAX,
-            account_required_max: usize::MAX,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersBlocks {
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub max: usize,
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub account_include_max: usize,
-    pub account_include_any: bool,
-    #[serde(deserialize_with = "deserialize_pubkey_set")]
-    pub account_include_reject: HashSet<Pubkey>,
-    pub include_transactions: bool,
-    pub include_accounts: bool,
-    pub include_entries: bool,
-}
-
-impl Default for ConfigGrpcFiltersBlocks {
-    fn default() -> Self {
-        Self {
-            max: usize::MAX,
-            account_include_max: usize::MAX,
-            account_include_any: true,
-            account_include_reject: HashSet::new(),
-            include_transactions: true,
-            include_accounts: true,
-            include_entries: true,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersBlocksMeta {
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub max: usize,
-}
-
-impl Default for ConfigGrpcFiltersBlocksMeta {
-    fn default() -> Self {
-        Self { max: usize::MAX }
-    }
-}
-
-#[derive(Debug, Clone, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-pub struct ConfigGrpcFiltersEntries {
-    #[serde(deserialize_with = "deserialize_usize_str")]
-    pub max: usize,
-}
-
-impl Default for ConfigGrpcFiltersEntries {
-    fn default() -> Self {
-        Self { max: usize::MAX }
-    }
-}
-
 #[derive(Debug, Clone, Copy, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct ConfigPrometheus {
     /// Address of Prometheus service.
     pub address: SocketAddr,
 }
 
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum ValueIntStr<'a> {
+    Int(usize),
+    Str(&'a str),
+}
+
 fn deserialize_usize_str<'de, D>(deserializer: D) -> Result<usize, D::Error>
 where
     D: Deserializer<'de>,
 {
-    #[derive(Deserialize)]
-    #[serde(untagged)]
-    enum Value {
-        Integer(usize),
-        String(String),
-    }
-
-    match Value::deserialize(deserializer)? {
-        Value::Integer(value) => Ok(value),
-        Value::String(value) => value
+    match ValueIntStr::deserialize(deserializer)? {
+        ValueIntStr::Int(value) => Ok(value),
+        ValueIntStr::Str(value) => value
             .replace('_', "")
             .parse::<usize>()
             .map_err(de::Error::custom),
@@ -415,34 +241,13 @@ fn deserialize_usize_str_maybe<'de, D>(deserializer: D) -> Result<Option<usize>,
 where
     D: Deserializer<'de>,
 {
-    #[derive(Deserialize)]
-    #[serde(untagged)]
-    enum Value {
-        Integer(usize),
-        String(String),
-    }
-
-    match Option::<Value>::deserialize(deserializer)? {
-        Some(Value::Integer(value)) => Ok(Some(value)),
-        Some(Value::String(value)) => value
+    match Option::<ValueIntStr>::deserialize(deserializer)? {
+        Some(ValueIntStr::Int(value)) => Ok(Some(value)),
+        Some(ValueIntStr::Str(value)) => value
             .replace('_', "")
             .parse::<usize>()
             .map(Some)
             .map_err(de::Error::custom),
         None => Ok(None),
     }
 }
-
-fn deserialize_pubkey_set<'de, D>(deserializer: D) -> Result<HashSet<Pubkey>, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    Vec::<&str>::deserialize(deserializer)?
-        .into_iter()
-        .map(|value| {
-            value
-                .parse()
-                .map_err(|error| de::Error::custom(format!("Invalid pubkey: {value} ({error:?})")))
-        })
-        .collect::<Result<_, _>>()
-}