From 6fe372ed88793cc0f83248a05628fd239f0057c3 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 28 Sep 2023 17:21:23 +0400 Subject: [PATCH 1/7] update to geyser block v3 --- Cargo.toml | 7 +++++++ examples/rust/src/bin/tx-blocktime.rs | 2 +- yellowstone-grpc-geyser/src/grpc.rs | 6 +++--- yellowstone-grpc-geyser/src/plugin.rs | 7 +++++-- yellowstone-grpc-kafka/Cargo.toml | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f93dd70b..eba284b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,3 +11,10 @@ members = [ debug = true lto = true codegen-units = 1 + +[patch.crates-io] +solana-geyser-plugin-interface = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } +solana-logger = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } +solana-program = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } +solana-sdk = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } +solana-transaction-status = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } diff --git a/examples/rust/src/bin/tx-blocktime.rs b/examples/rust/src/bin/tx-blocktime.rs index 0a172a4f..6dac868e 100644 --- a/examples/rust/src/bin/tx-blocktime.rs +++ b/examples/rust/src/bin/tx-blocktime.rs @@ -115,7 +115,7 @@ async fn main() -> anyhow::Result<()> { Some(UpdateOneof::Transaction(tx)) => { let entry = messages.entry(tx.slot).or_default(); let sig = Signature::try_from(tx.transaction.unwrap().signature.as_slice()) - .expect("valid signature") + .expect("valid signature from transaction") .to_string(); if let Some(timestamp) = entry.0 { info!("received txn {} at {}", sig, timestamp); diff --git a/yellowstone-grpc-geyser/src/grpc.rs b/yellowstone-grpc-geyser/src/grpc.rs index 2ce0dd4b..338a0e55 100644 --- a/yellowstone-grpc-geyser/src/grpc.rs +++ b/yellowstone-grpc-geyser/src/grpc.rs @@ -7,7 +7,7 @@ use { }, log::{error, info}, solana_geyser_plugin_interface::geyser_plugin_interface::{ - ReplicaAccountInfoV3, ReplicaBlockInfoV2, ReplicaEntryInfo, ReplicaTransactionInfoV2, + ReplicaAccountInfoV3, ReplicaBlockInfoV3, ReplicaEntryInfo, ReplicaTransactionInfoV2, SlotStatus, }, solana_sdk::{ @@ -281,8 +281,8 @@ pub struct MessageBlockMeta { pub executed_transaction_count: u64, } -impl<'a> From<&'a ReplicaBlockInfoV2<'a>> for MessageBlockMeta { - fn from(blockinfo: &'a ReplicaBlockInfoV2<'a>) -> Self { +impl<'a> From<&'a ReplicaBlockInfoV3<'a>> for MessageBlockMeta { + fn from(blockinfo: &'a ReplicaBlockInfoV3<'a>) -> Self { Self { parent_slot: blockinfo.parent_slot, slot: blockinfo.slot, diff --git a/yellowstone-grpc-geyser/src/plugin.rs b/yellowstone-grpc-geyser/src/plugin.rs index 3296f551..6eaf7df9 100644 --- a/yellowstone-grpc-geyser/src/plugin.rs +++ b/yellowstone-grpc-geyser/src/plugin.rs @@ -203,10 +203,13 @@ impl GeyserPlugin for Plugin { ReplicaBlockInfoVersions::V0_0_1(_info) => { unreachable!("ReplicaBlockInfoVersions::V0_0_1 is not supported") } - ReplicaBlockInfoVersions::V0_0_2(info) => info, + ReplicaBlockInfoVersions::V0_0_2(_info) => { + unreachable!("ReplicaBlockInfoVersions::V0_0_2 is not supported") + } + ReplicaBlockInfoVersions::V0_0_3(info) => info, }; - let message = Message::BlockMeta((blockinfo).into()); + let message = Message::BlockMeta(blockinfo.into()); inner.send_message(message); Ok(()) diff --git a/yellowstone-grpc-kafka/Cargo.toml b/yellowstone-grpc-kafka/Cargo.toml index 03e5a2c6..5aed5add 100644 --- a/yellowstone-grpc-kafka/Cargo.toml +++ b/yellowstone-grpc-kafka/Cargo.toml @@ -26,7 +26,7 @@ tonic = { version = "0.10.2", features = ["gzip", "tls", "tls-roots"] } tonic-health = "0.10.2" tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -yellowstone-grpc-proto = { path = "../yellowstone-grpc-proto" } +yellowstone-grpc-proto = { path = "../yellowstone-grpc-proto", version = "1.9.0+solana.1.16.14" } [build-dependencies] anyhow = "1.0.62" From 9eada5fa04dcbaecfb0770f4b379bd719e29a90d Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 28 Sep 2023 18:29:21 +0400 Subject: [PATCH 2/7] use entries_count --- yellowstone-grpc-geyser/src/grpc.rs | 50 ++++++++++++----------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/yellowstone-grpc-geyser/src/grpc.rs b/yellowstone-grpc-geyser/src/grpc.rs index 338a0e55..db2ea87f 100644 --- a/yellowstone-grpc-geyser/src/grpc.rs +++ b/yellowstone-grpc-geyser/src/grpc.rs @@ -279,6 +279,7 @@ pub struct MessageBlockMeta { pub block_time: Option, pub block_height: Option, pub executed_transaction_count: u64, + pub entries_count: u64, } impl<'a> From<&'a ReplicaBlockInfoV3<'a>> for MessageBlockMeta { @@ -292,6 +293,7 @@ impl<'a> From<&'a ReplicaBlockInfoV3<'a>> for MessageBlockMeta { block_time: blockinfo.block_time, block_height: blockinfo.block_height, executed_transaction_count: blockinfo.executed_transaction_count, + entries_count: blockinfo.entry_count, } } } @@ -632,8 +634,7 @@ struct SlotMessages { block_meta: Option, transactions: Vec, accounts_dedup: HashMap, // (write_version, message_index) - entries_parent_updated: bool, - entries_received: bool, + entries: Vec, sealed: bool, confirmed_at: Option, finalized_at: Option, @@ -643,17 +644,16 @@ impl SlotMessages { pub fn try_seal(&mut self) -> Option { if !self.sealed { if let Some(block_meta) = &self.block_meta { - let transactions_count = block_meta.executed_transaction_count as usize; - if transactions_count == self.transactions.len() && self.entries_received { + if self.transactions.len() == block_meta.executed_transaction_count as usize + && self.entries.len() == block_meta.entries_count as usize + { let transactions = std::mem::take(&mut self.transactions); + let entries = std::mem::take(&mut self.entries); let mut accounts = Vec::with_capacity(self.messages.len()); - let mut entries = Vec::with_capacity(self.messages.len()); for item in self.messages.iter().flatten() { - match item { - Message::Account(account) => accounts.push(account.account.clone()), - Message::Entry(entry) => entries.push(entry.clone()), - _ => {} + if let Message::Account(account) = item { + accounts.push(account.account.clone()); } } @@ -810,16 +810,17 @@ impl GrpcService { let msg_txn_count = slot_messages.transactions.len(); if block_txn_count != msg_txn_count { reasons.push("InvalidTxnCount"); - error!("failed reconstruct #{slot} -- tx count: {block_txn_count} vs {msg_txn_count}"); - } + error!("failed to reconstruct #{slot} -- tx count: {block_txn_count} vs {msg_txn_count}"); + } + let block_entries_count = block_meta.entries_count as usize; + let msg_entries_count = slot_messages.entries.len(); + if block_entries_count != msg_entries_count { + reasons.push("InvalidEntriesCount"); + error!("failed to reconstruct #{slot} -- entries count: {block_entries_count} vs {msg_entries_count}"); + } } else { reasons.push("NoBlockMeta"); } - if !slot_messages.entries_received { - reasons.push("MissedEntries"); - let msg_entries_count = slot_messages.messages.iter().filter(|x| matches!(x, Some(Message::Entry(_)))).count(); - error!("failed reconstruct #{slot} -- entries: {msg_entries_count}"); - } let reason = reasons.join(","); prom::update_invalid_blocks(format!("failed reconstruct {reason}")); @@ -895,20 +896,9 @@ impl GrpcService { slot_messages.accounts_dedup.insert(msg.account.pubkey, (write_version, msg_index)); } } - // Entries can come after block meta or last transaction - Message::Entry(msg) if !slot_messages.entries_parent_updated => { - slot_messages.entries_parent_updated = true; - - let mut iter = messages.iter_mut().peekable(); - while let Some((_slot, slot_messages)) = iter.next() { - if let Some((slot_peek, _slot_messages)) = iter.peek() { - if **slot_peek == msg.slot { - slot_messages.entries_received = true; - sealed_block_msg = slot_messages.try_seal(); - break; - } - } - } + Message::Entry(msg) => { + slot_messages.entries.push(msg.clone()); + sealed_block_msg = slot_messages.try_seal(); } _ => {} } From d580524fbc9c837cee8b4e102f7024153cde7d2a Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Thu, 28 Sep 2023 18:30:58 +0400 Subject: [PATCH 3/7] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed8308ec..c90c1f97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The minor version will be incremented upon a breaking change and the patch versi ### Fixes - geyser: add `fs` feature to `tokio` dependencies in the plugin ([#184](https://github.com/rpcpool/yellowstone-grpc/pull/184)). +- geyser: use entry_count from ReplicaBlockInfoV3 ([#185](https://github.com/rpcpool/yellowstone-grpc/pull/185)). ### Breaking From 249db0e273eb9a59e571ed6ea62008d764864a45 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 4 Oct 2023 23:36:36 +0400 Subject: [PATCH 4/7] fix deps --- Cargo.lock | 42 ++++++++++++++---------------------------- Cargo.toml | 1 + 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9277f56..822e32d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3151,8 +3151,7 @@ dependencies = [ [[package]] name = "solana-account-decoder" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0ada16ccd5ca6884ae28b716211c4f09d22225a9ebde14eccd4f605cc321e42" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "Inflector", "base64 0.21.4", @@ -3175,8 +3174,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248b435722d18100b70bb8f09fd38b8a8c46031a0de86f6b31768f64cf4092c5" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "bincode", "bytemuck", @@ -3196,8 +3194,7 @@ dependencies = [ [[package]] name = "solana-config-program" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0622d8798d060c84883572483f214b0d5c1640450e4322483cfe2e72823a6d7" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "bincode", "chrono", @@ -3210,8 +3207,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e74d294241df12a73a229e38a819e810d54234da494c3d43f8a1a828631047d" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "ahash 0.8.3", "blake3", @@ -3243,8 +3239,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dabde7fbd88a68eb083ae9d6d5f6855b7ba1bfc45d200c786b1b448ac49da5f" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "proc-macro2", "quote", @@ -3255,8 +3250,7 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-interface" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908cad0de1330b6b05d6cb6a9f5004df22a9fe46529d6bf6839835ba345ef966" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "log", "solana-sdk", @@ -3267,8 +3261,7 @@ dependencies = [ [[package]] name = "solana-logger" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24dc0037a389d782c8de3c6f509b74efa1d60523ef9e5561cefe41f1a354fee0" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -3278,8 +3271,7 @@ dependencies = [ [[package]] name = "solana-measure" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19563c27f12801e0e65b42d0f216db1d910dfeaad88d8f1335dd6369697eda60" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "log", "solana-sdk", @@ -3288,8 +3280,7 @@ dependencies = [ [[package]] name = "solana-metrics" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f74c557e821c7ff48c9b4ef4ab3a296918d7fb745e3f2e820ebdb38e8e3555b" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "crossbeam-channel", "gethostname", @@ -3302,8 +3293,7 @@ dependencies = [ [[package]] name = "solana-program" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a2faeb9c89b354b547a229e3b39fa8fd6d11c78362ba8580f0c4721739725b6" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "ark-bn254", "ark-ec", @@ -3357,8 +3347,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f65907a405764cda63be89335ffd2138ade18f065e5cae09d20adab7fb09502" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "base64 0.21.4", "bincode", @@ -3385,8 +3374,7 @@ dependencies = [ [[package]] name = "solana-sdk" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57f4046d0d487e3d79be809ff29c63c1484793956e6ccbc5d3307b0aafbf989d" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "assert_matches", "base64 0.21.4", @@ -3438,8 +3426,7 @@ dependencies = [ [[package]] name = "solana-sdk-macro" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760fdfd4b7edb02fd9173a6dcec899ffae06ac21b66b65f8c7c5f3d17b12fa64" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "bs58", "proc-macro2", @@ -3451,8 +3438,7 @@ dependencies = [ [[package]] name = "solana-transaction-status" version = "1.16.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf7379a72c051d7879f1c36cfdab5fed0653a2a613718bc5d343e44e603401" +source = "git+https://github.com/rpcpool/solana-public.git?tag=v1.16.14-geyser-block-v3#aff14b9b64bb33c63881a36a440eeb31cb21c192" dependencies = [ "Inflector", "base64 0.21.4", diff --git a/Cargo.toml b/Cargo.toml index eba284b8..835ac549 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ lto = true codegen-units = 1 [patch.crates-io] +solana-account-decoder = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } solana-geyser-plugin-interface = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } solana-logger = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } solana-program = { git = "https://github.com/rpcpool/solana-public.git", tag = "v1.16.14-geyser-block-v3" } From ca3e2049082483fde4dd5e2abfa02384ed66c28b Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 4 Oct 2023 23:42:00 +0400 Subject: [PATCH 5/7] remove version for proto dep in kafka pkg --- yellowstone-grpc-kafka/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yellowstone-grpc-kafka/Cargo.toml b/yellowstone-grpc-kafka/Cargo.toml index 5aed5add..03e5a2c6 100644 --- a/yellowstone-grpc-kafka/Cargo.toml +++ b/yellowstone-grpc-kafka/Cargo.toml @@ -26,7 +26,7 @@ tonic = { version = "0.10.2", features = ["gzip", "tls", "tls-roots"] } tonic-health = "0.10.2" tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -yellowstone-grpc-proto = { path = "../yellowstone-grpc-proto", version = "1.9.0+solana.1.16.14" } +yellowstone-grpc-proto = { path = "../yellowstone-grpc-proto" } [build-dependencies] anyhow = "1.0.62" From ad87dc5523a088ac87c6685539bd2431795866cc Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Tue, 10 Oct 2023 11:09:23 +0400 Subject: [PATCH 6/7] bump rust client --- CHANGELOG.md | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- yellowstone-grpc-client/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94111d92..2c7b0edb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The minor version will be incremented upon a breaking change and the patch versi ## 2023-10-10 -- yellowstone-grpc-client-1.11.0+solana.1.16.16 +- yellowstone-grpc-client-1.11.1+solana.1.16.16 - yellowstone-grpc-geyser-1.9.1+solana.1.16.16 - yellowstone-grpc-kafka-1.0.0-rc.3+solana.1.16.16 - yellowstone-grpc-proto-1.10.0+solana.1.16.16 diff --git a/Cargo.lock b/Cargo.lock index 9250c9ed..b7d4725b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4597,7 +4597,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.11.0+solana.1.16.16" +version = "1.11.1+solana.1.16.16" dependencies = [ "bytes", "futures", diff --git a/Cargo.toml b/Cargo.toml index 92667e6d..82bdc025 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "examples/rust", # 1.10.0+solana.1.16.16 - "yellowstone-grpc-client", # 1.11.0+solana.1.16.16 + "yellowstone-grpc-client", # 1.11.1+solana.1.16.16 "yellowstone-grpc-geyser", # 1.9.0+solana.1.16.16 "yellowstone-grpc-kafka", # 1.0.0-rc.3+solana.1.16.16 "yellowstone-grpc-proto", # 1.10.0+solana.1.16.16 diff --git a/yellowstone-grpc-client/Cargo.toml b/yellowstone-grpc-client/Cargo.toml index 17062a05..6b0daed8 100644 --- a/yellowstone-grpc-client/Cargo.toml +++ b/yellowstone-grpc-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-client" -version = "1.11.0+solana.1.16.16" +version = "1.11.1+solana.1.16.16" authors = ["Triton One"] edition = "2021" description = "Yellowstone gRPC Geyser Simple Client" From eac1590fb9eadc9d062f14df37e6662ed1d81509 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Tue, 10 Oct 2023 11:22:09 +0400 Subject: [PATCH 7/7] readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ef3b3a3f..453c0839 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ For additional documentation, please see: https://docs.triton.one/rpc-pool/grpc ### Validator +Current plugin version (`+solana.1.16.x`) use validator with backported `ReplicaBlockInfoV3` to Geyser interface — https://github.com/solana-labs/solana/pull/33359. As result it's not compatible with original validator from Solana Labs and would not work. You need to compile validator from the source code and can find patched releases in `Triton One` Solana fork: https://github.com/rpcpool/solana-public/tree/v1.16.16-geyser-block-v3. + ```bash $ solana-validator --geyser-plugin-config yellowstone-grpc-geyser/config.json ```