tools: scylladb cosumer and refactor scylla sink

Cargo.toml

@@ -48,7 +48,7 @@ prometheus = "0.13.2"
 prost = "0.12.1"
 protobuf-src = "1.1.0"
 rdkafka = "0.34.0"
-scylla = "0.12.0"
+scylla = "0.13.0"
 serde = "1.0.145"
 serde_json = "1.0.86"
 serde_with = "3.7.0"
@@ -68,6 +68,7 @@ tonic-build = "0.10.2"
 tonic-health = "0.10.2"
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
+uuid = "1.8.0"
 vergen = "8.2.1"
 yellowstone-grpc-client = { path = "yellowstone-grpc-client", version = "=1.15.0+solana.1.18.12" }
 yellowstone-grpc-proto = { path = "yellowstone-grpc-proto", version = "=1.14.0+solana.1.18.12" }

yellowstone-grpc-proto/build.rs

@@ -1,5 +1,5 @@
 fn main() -> anyhow::Result<()> {
     std::env::set_var("PROTOC", protobuf_src::protoc());
-    tonic_build::compile_protos("proto/geyser.proto")?;
+    tonic_build::compile_protos("proto/yellowstone-log.proto")?;
     Ok(())
 }

yellowstone-grpc-proto/proto/yellowstone-log.proto

@@ -0,0 +1,43 @@
+syntax = "proto3";
+
+import public "geyser.proto";
+
+
+option go_package = "github.com/rpcpool/solana-geyser-grpc/golang/proto";
+
+package yellowstone.log;
+
+service YellowstoneLog {
+  rpc Consume(ConsumeRequest) returns (stream geyser.SubscribeUpdate) {}
+}
+
+
+enum InitialOffsetPolicy {
+  EARLIEST = 0;
+  LATEST = 1;
+  SLOT = 2;
+}
+
+enum EventSubscriptionPolicy {
+  ACCOUNT_UPDATE_ONLY = 0;
+  TRANSACTION_ONLY = 1;
+  BOTH = 2;
+}
+
+message ConsumeRequest {
+  optional string consumer_id = 1;
+  InitialOffsetPolicy initial_offset_policy = 2;
+  optional int64 at_slot = 3;
+  EventSubscriptionPolicy event_subscription_policy = 4;
+  optional AccountUpdateEventFilter account_update_event_filter = 5;
+  optional TransactionEventFilter tx_event_filter = 6;
+}
+
+message AccountUpdateEventFilter {
+  repeated bytes pubkeys = 1;
+  repeated bytes owners = 2;
+}
+
+message TransactionEventFilter {
+  repeated bytes account_keys = 1;
+}

yellowstone-grpc-proto/src/lib.rs

@@ -1,5 +1,11 @@
 #![allow(clippy::large_enum_variant)]
 
+pub mod yellowstone {
+    pub mod log {
+        tonic::include_proto!("yellowstone.log");
+    }
+}
+
 pub mod geyser {
     tonic::include_proto!("geyser");
 }

yellowstone-grpc-tools/Cargo.toml

@@ -48,9 +48,11 @@ tonic = { workspace = true, features = ["gzip"] }
 tonic-health = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
+uuid = { workspace = true, optional = true }
 yellowstone-grpc-client = { workspace = true }
 yellowstone-grpc-proto = { workspace = true }
 
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["test-util"] }
 
@@ -61,6 +63,7 @@ rdkafka = { workspace = true, features = ["sasl", "ssl"], optional = true }
 rdkafka = { workspace = true, features = ["sasl", "ssl-vendored"], optional = true }
 
 [build-dependencies]
+project-root = "0.2.2"
 anyhow = { workspace = true }
 cargo-lock = { workspace = true }
 git-version = { workspace = true }
@@ -70,4 +73,4 @@ vergen = { workspace = true, features = ["build", "rustc"] }
 default = ["google-pubsub", "kafka"]
 google-pubsub = ["google-cloud-googleapis", "google-cloud-pubsub"]
 kafka = ["const-hex", "rdkafka", "sha2"]
-scylladb = ["scylla", "serde_with", "deepsize"]
+scylladb = ["scylla", "serde_with", "deepsize", "uuid"]