From 776059ffa920e7cede57abddda20c1fbf50f97a5 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 24 Dec 2023 18:22:27 -0500 Subject: [PATCH] proto: add `timestamp` field to `SubscribeUpdate` message --- .github/workflows/release.yml | 11 ++++--- .github/workflows/test.yml | 12 ++++--- CHANGELOG.md | 33 +++++++++++++++++++ Cargo.lock | 11 ++++--- Cargo.toml | 15 +++++---- examples/rust/Cargo.toml | 2 +- yellowstone-grpc-client/Cargo.toml | 2 +- yellowstone-grpc-geyser/Cargo.toml | 2 +- yellowstone-grpc-geyser/src/filters.rs | 23 ++++++++----- yellowstone-grpc-geyser/src/grpc.rs | 3 ++ yellowstone-grpc-proto/Cargo.toml | 3 +- yellowstone-grpc-proto/proto/geyser.proto | 2 ++ yellowstone-grpc-proto/src/lib.rs | 2 +- yellowstone-grpc-tools/Cargo.toml | 2 +- .../src/bin/grpc-google-pubsub.rs | 8 ++--- yellowstone-grpc-tools/src/kafka/grpc.rs | 19 +++++++---- 16 files changed, 105 insertions(+), 45 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8c58b8c..ac640bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get install -y libsasl2-dev + - name: Set rust version run: | source ci/rust-version.sh @@ -58,10 +61,10 @@ jobs: target key: ${{ matrix.os }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('**/Cargo.lock') }}-0001 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libsasl2-dev + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y libsasl2-dev - name: Check Solana version run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 600dab8..c7c7423 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ on: - 'master' - 'v1.16' - 'v1.17' + - 'v1.17-gamma' workflow_dispatch: env: @@ -23,6 +24,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get install -y libsasl2-dev + - name: Set rust version run: | source ci/rust-version.sh @@ -58,10 +62,10 @@ jobs: target key: ${{ matrix.os }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('**/Cargo.lock') }}-0001 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libsasl2-dev + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y libsasl2-dev - name: cargo tree run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a6825..743afc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,17 @@ The minor version will be incremented upon a breaking change and the patch versi ## 2024-02-06 +- yellowstone-grpc-client-1.13.0+solana.1.17.20.gamma +- yellowstone-grpc-geyser-1.12.0+solana.1.17.20.gamma +- yellowstone-grpc-proto-1.12.0+solana.1.17.20.gamma +- yellowstone-grpc-tools-1.0.0-rc.9+solana.1.17.20.gamma + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + +## 2024-02-06 + - yellowstone-grpc-client-1.13.0+solana.1.17.20 - yellowstone-grpc-geyser-1.12.0+solana.1.17.20 - yellowstone-grpc-proto-1.12.0+solana.1.17.20 @@ -214,6 +225,17 @@ The minor version will be incremented upon a breaking change and the patch versi - solana: update to 1.17.16 ([#274](https://github.com/rpcpool/yellowstone-grpc/pull/274)) +## 2024-01-09 + +- yellowstone-grpc-client-1.13.0+solana.1.17.15.gamma +- yellowstone-grpc-geyser-1.12.0+solana.1.17.15.gamma +- yellowstone-grpc-proto-1.12.0+solana.1.17.15.gamma +- yellowstone-grpc-tools-1.0.0-rc.9+solana.1.17.15.gamma + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + ## 2024-01-08 - yellowstone-grpc-client-1.13.0+solana.1.17.15 @@ -226,6 +248,17 @@ The minor version will be incremented upon a breaking change and the patch versi - proto: add more convert functions ([#264](https://github.com/rpcpool/yellowstone-grpc/pull/264)) - geyser: set plugin name to `{name}-{version}` ([#270](https://github.com/rpcpool/yellowstone-grpc/pull/270)) +## 2023-12-24 + +- yellowstone-grpc-client-1.13.0+solana.1.16.21 +- yellowstone-grpc-geyser-1.12.0+solana.1.16.21 +- yellowstone-grpc-proto-1.12.0+solana.1.16.21 +- yellowstone-grpc-tools-1.0.0-rc.10+solana.1.16.21 + +### Features + +- proto: add `timestamp` field to `SubscribeUpdate` message + ## 2023-12-22 - yellowstone-grpc-client-1.12.0+solana.1.17.12 diff --git a/Cargo.lock b/Cargo.lock index 726d693..3308d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5174,7 +5174,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.15.0+solana.1.17.33" +version = "1.15.0+solana.1.17.33.gamma" dependencies = [ "bytes", "futures", @@ -5187,7 +5187,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client-simple" -version = "1.13.0+solana.1.17.33" +version = "1.13.0+solana.1.17.33.gamma" dependencies = [ "anyhow", "backoff", @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-geyser" -version = "1.14.2+solana.1.17.33" +version = "1.14.2+solana.1.17.33.gamma" dependencies = [ "anyhow", "base64 0.21.7", @@ -5243,11 +5243,12 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "1.14.0+solana.1.17.33" +version = "1.14.0+solana.1.17.33.gamma" dependencies = [ "anyhow", "bincode", "prost 0.12.3", + "prost-types 0.12.3", "protobuf-src", "solana-account-decoder", "solana-sdk", @@ -5258,7 +5259,7 @@ dependencies = [ [[package]] name = "yellowstone-grpc-tools" -version = "1.0.0-rc.11+solana.1.17.33" +version = "1.0.0-rc.11+solana.1.17.33.gamma" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 224fdfd..fa07e47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [workspace] resolver = "2" members = [ - "examples/rust", # 1.13.0+solana.1.17.33 - "yellowstone-grpc-client", # 1.15.0+solana.1.17.33 - "yellowstone-grpc-geyser", # 1.14.2+solana.1.17.33 - "yellowstone-grpc-proto", # 1.14.0+solana.1.17.33 - "yellowstone-grpc-tools", # 1.0.0-rc.11+solana.1.17.33 + "examples/rust", # 1.13.0+solana.1.17.33.gamma + "yellowstone-grpc-client", # 1.15.0+solana.1.17.33.gamma + "yellowstone-grpc-geyser", # 1.14.2+solana.1.17.33.gamma + "yellowstone-grpc-proto", # 1.14.0+solana.1.17.33.gamma + "yellowstone-grpc-tools", # 1.0.0-rc.11+solana.1.17.33.gamma ] [workspace.package] @@ -45,6 +45,7 @@ log = "0.4.17" maplit = "1.0.2" prometheus = "0.13.2" prost = "0.12.1" +prost-types = "0.12.1" protobuf-src = "1.1.0" rdkafka = "0.34.0" serde = "1.0.145" @@ -66,8 +67,8 @@ tonic-health = "0.10.2" tracing = "0.1.37" tracing-subscriber = "0.3.17" vergen = "8.2.1" -yellowstone-grpc-client = { path = "yellowstone-grpc-client", version = "=1.15.0+solana.1.17.33" } -yellowstone-grpc-proto = { path = "yellowstone-grpc-proto", version = "=1.14.0+solana.1.17.33" } +yellowstone-grpc-client = { path = "yellowstone-grpc-client", version = "=1.15.0+solana.1.17.33.gamma" } +yellowstone-grpc-proto = { path = "yellowstone-grpc-proto", version = "=1.14.0+solana.1.17.33.gamma" } [profile.release] debug = true diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index e0916f0..7825311 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-client-simple" -version = "1.13.0+solana.1.17.33" +version = "1.13.0+solana.1.17.33.gamma" authors = { workspace = true } edition = { workspace = true } homepage = { workspace = true } diff --git a/yellowstone-grpc-client/Cargo.toml b/yellowstone-grpc-client/Cargo.toml index 4ae1cb0..4ef0986 100644 --- a/yellowstone-grpc-client/Cargo.toml +++ b/yellowstone-grpc-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-client" -version = "1.15.0+solana.1.17.33" +version = "1.15.0+solana.1.17.33.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Simple Client" diff --git a/yellowstone-grpc-geyser/Cargo.toml b/yellowstone-grpc-geyser/Cargo.toml index f2c399c..ac0997c 100644 --- a/yellowstone-grpc-geyser/Cargo.toml +++ b/yellowstone-grpc-geyser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-geyser" -version = "1.14.2+solana.1.17.33" +version = "1.14.2+solana.1.17.33.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Plugin" diff --git a/yellowstone-grpc-geyser/src/filters.rs b/yellowstone-grpc-geyser/src/filters.rs index 62ea1bd..d5dc204 100644 --- a/yellowstone-grpc-geyser/src/filters.rs +++ b/yellowstone-grpc-geyser/src/filters.rs @@ -16,15 +16,20 @@ use { std::{ collections::{HashMap, HashSet}, str::FromStr, + time::SystemTime, }, - yellowstone_grpc_proto::prelude::{ - subscribe_request_filter_accounts_filter::Filter as AccountsFilterDataOneof, - subscribe_request_filter_accounts_filter_memcmp::Data as AccountsFilterMemcmpOneof, - subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, - SubscribeRequestAccountsDataSlice, SubscribeRequestFilterAccounts, - SubscribeRequestFilterAccountsFilter, SubscribeRequestFilterBlocks, - SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterEntry, SubscribeRequestFilterSlots, - SubscribeRequestFilterTransactions, SubscribeUpdate, SubscribeUpdatePong, + yellowstone_grpc_proto::{ + prelude::{ + subscribe_request_filter_accounts_filter::Filter as AccountsFilterDataOneof, + subscribe_request_filter_accounts_filter_memcmp::Data as AccountsFilterMemcmpOneof, + subscribe_update::UpdateOneof, CommitmentLevel, SubscribeRequest, + SubscribeRequestAccountsDataSlice, SubscribeRequestFilterAccounts, + SubscribeRequestFilterAccountsFilter, SubscribeRequestFilterBlocks, + SubscribeRequestFilterBlocksMeta, SubscribeRequestFilterEntry, + SubscribeRequestFilterSlots, SubscribeRequestFilterTransactions, SubscribeUpdate, + SubscribeUpdatePong, + }, + prost_types::Timestamp, }, }; @@ -133,6 +138,7 @@ impl Filter { Some(SubscribeUpdate { filters, update_oneof: Some(message.to_proto(&self.accounts_data_slice)), + timestamp: Some(Timestamp::from(SystemTime::now())), }) } }), @@ -143,6 +149,7 @@ impl Filter { self.ping.map(|id| SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Pong(SubscribeUpdatePong { id })), + timestamp: Some(Timestamp::from(SystemTime::now())), }) } } diff --git a/yellowstone-grpc-geyser/src/grpc.rs b/yellowstone-grpc-geyser/src/grpc.rs index eb644d7..4cc5827 100644 --- a/yellowstone-grpc-geyser/src/grpc.rs +++ b/yellowstone-grpc-geyser/src/grpc.rs @@ -24,6 +24,7 @@ use { atomic::{AtomicUsize, Ordering}, Arc, }, + time::SystemTime, }, tokio::{ fs, @@ -56,6 +57,7 @@ use { SubscribeUpdateSlot, SubscribeUpdateTransaction, SubscribeUpdateTransactionInfo, SubscribeUpdateTransactionStatus, TransactionError as SubscribeUpdateTransactionError, }, + prost_types::Timestamp, }, }; @@ -1287,6 +1289,7 @@ impl Geyser for GrpcService { let ping_msg = SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Ping(SubscribeUpdatePing {})), + timestamp: Some(Timestamp::from(SystemTime::now())), }; loop { diff --git a/yellowstone-grpc-proto/Cargo.toml b/yellowstone-grpc-proto/Cargo.toml index c41c008..1267325 100644 --- a/yellowstone-grpc-proto/Cargo.toml +++ b/yellowstone-grpc-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-proto" -version = "1.14.0+solana.1.17.33" +version = "1.14.0+solana.1.17.33.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Geyser Protobuf Definitions" @@ -13,6 +13,7 @@ publish = true [dependencies] bincode = { workspace = true } prost = { workspace = true } +prost-types = { workspace = true } solana-account-decoder = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } diff --git a/yellowstone-grpc-proto/proto/geyser.proto b/yellowstone-grpc-proto/proto/geyser.proto index dbfdb22..db7cc83 100644 --- a/yellowstone-grpc-proto/proto/geyser.proto +++ b/yellowstone-grpc-proto/proto/geyser.proto @@ -1,5 +1,6 @@ syntax = "proto3"; +import "google/protobuf/timestamp.proto"; import public "solana-storage.proto"; option go_package = "github.com/rpcpool/solana-geyser-grpc/golang/proto"; @@ -104,6 +105,7 @@ message SubscribeUpdate { SubscribeUpdateBlockMeta block_meta = 7; SubscribeUpdateEntry entry = 8; } + google.protobuf.Timestamp timestamp = 11; } message SubscribeUpdateAccount { diff --git a/yellowstone-grpc-proto/src/lib.rs b/yellowstone-grpc-proto/src/lib.rs index 8760c7c..60ebf3a 100644 --- a/yellowstone-grpc-proto/src/lib.rs +++ b/yellowstone-grpc-proto/src/lib.rs @@ -16,7 +16,7 @@ pub mod prelude { pub use super::{geyser::*, solana::storage::confirmed_block::*}; } -pub use {prost, tonic}; +pub use {prost, prost_types, tonic}; pub mod convert_to { use { diff --git a/yellowstone-grpc-tools/Cargo.toml b/yellowstone-grpc-tools/Cargo.toml index d457aa6..5cdf522 100644 --- a/yellowstone-grpc-tools/Cargo.toml +++ b/yellowstone-grpc-tools/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "yellowstone-grpc-tools" -version = "1.0.0-rc.11+solana.1.17.33" +version = "1.0.0-rc.11+solana.1.17.33.gamma" authors = { workspace = true } edition = { workspace = true } description = "Yellowstone gRPC Tools" diff --git a/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs b/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs index c29538e..31724ba 100644 --- a/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs +++ b/yellowstone-grpc-tools/src/bin/grpc-google-pubsub.rs @@ -177,9 +177,9 @@ impl ArgsAction { .context("failed to get message from gRPC")?; match &message { - SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Ping(_)) } => prom::recv_inc(GprcMessageKind::Ping), - SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Pong(_)) } => prom::recv_inc(GprcMessageKind::Pong), - SubscribeUpdate { filters: _, update_oneof: Some(value) } => { + SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Ping(_)), timestamp: _ } => prom::recv_inc(GprcMessageKind::Ping), + SubscribeUpdate { filters: _, update_oneof: Some(UpdateOneof::Pong(_)), timestamp: _ } => prom::recv_inc(GprcMessageKind::Pong), + SubscribeUpdate { filters: _, update_oneof: Some(value), timestamp: _ } => { if let UpdateOneof::Slot(slot) = value { prom::set_slot_tip( CommitmentLevel::try_from(slot.status).expect("valid commitment"), @@ -196,7 +196,7 @@ impl ArgsAction { prom::recv_inc(prom_kind); }, - SubscribeUpdate { filters: _, update_oneof: None } => anyhow::bail!("received empty updat emessage"), + SubscribeUpdate { filters: _, update_oneof: None, timestamp: _ } => anyhow::bail!("received empty updat emessage"), }; } }; diff --git a/yellowstone-grpc-tools/src/kafka/grpc.rs b/yellowstone-grpc-tools/src/kafka/grpc.rs index bb9997c..f2c703a 100644 --- a/yellowstone-grpc-tools/src/kafka/grpc.rs +++ b/yellowstone-grpc-tools/src/kafka/grpc.rs @@ -7,6 +7,7 @@ use { atomic::{AtomicUsize, Ordering}, Arc, }, + time::SystemTime, }, tokio::{ sync::{broadcast, mpsc, Notify}, @@ -24,13 +25,16 @@ use { }, tonic_health::server::health_reporter, tracing::{error, info}, - yellowstone_grpc_proto::prelude::{ - geyser_server::{Geyser, GeyserServer}, - subscribe_update::UpdateOneof, - GetBlockHeightRequest, GetBlockHeightResponse, GetLatestBlockhashRequest, - GetLatestBlockhashResponse, GetSlotRequest, GetSlotResponse, GetVersionRequest, - GetVersionResponse, IsBlockhashValidRequest, IsBlockhashValidResponse, PingRequest, - PongResponse, SubscribeRequest, SubscribeUpdate, SubscribeUpdatePing, + yellowstone_grpc_proto::{ + prelude::{ + geyser_server::{Geyser, GeyserServer}, + subscribe_update::UpdateOneof, + GetBlockHeightRequest, GetBlockHeightResponse, GetLatestBlockhashRequest, + GetLatestBlockhashResponse, GetSlotRequest, GetSlotResponse, GetVersionRequest, + GetVersionResponse, IsBlockhashValidRequest, IsBlockhashValidResponse, PingRequest, + PongResponse, SubscribeRequest, SubscribeUpdate, SubscribeUpdatePing, + }, + prost_types::Timestamp, }, }; @@ -119,6 +123,7 @@ impl Geyser for GrpcService { let ping_msg = SubscribeUpdate { filters: vec![], update_oneof: Some(UpdateOneof::Ping(SubscribeUpdatePing {})), + timestamp: Some(Timestamp::from(SystemTime::now())), }; loop {