diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..0770125 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ +## Overview + +- Summary of changes + +## Testing + +- Testing performed to validate the changes diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..9da3a1d --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,54 @@ +name: CI + +on: + pull_request: + branches: [main] + +jobs: + + changes: + runs-on: ubuntu-latest + outputs: + fee-estimator: ${{ steps.filter.outputs.fee-estimator}} + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2.2.1 + id: filter + with: + filters: | + fee-estimator: + - '*/**' + + fee-estimator-tests: + needs: changes + if: ${{ needs.changes.outputs.fee-estimator == 'true' }} + runs-on: ubuntu-latest + env: + RUST_TOOLCHAIN: stable + CARGO_INCREMENTAL: 0 + SCCACHE_GHA_ENABLED: 'true' + RUSTC_WRAPPER: 'sccache' + ENV: 'test' + steps: + - uses: actions/checkout@v4 + - name: Cleanup Apt + run: sudo find /var/lib/apt/lists/ -type 'f' -name 'archive*' -delete && sudo apt-get update + - uses: rui314/setup-mold@v1 + with: + mold-version: 1.1.1 + make-default: true + - name: Ensure we use Mold + run: sudo ln -s /usr/local/bin/ld.mold /usr/bin/ld.lld + - name: Install rust + caching + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: '-C link-arg=-fuse-ld=lld' + components: rustfmt, clippy + - name: Set up sccache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Clear sccache stats + run: sccache --zero-stats + - name: Run Tests + run: | + cargo test -- --nocapture + diff --git a/.gitignore b/.gitignore index 575a1af..6969b2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,72 @@ -.vscode/ -target/ \ No newline at end of file +# Javascript/Typescript +node_modules/ +dist/ +build/ +coverage/ +vendor/ + +# Node Logs +logs +*.log +npm-debug.log* +AWSCLIV2.pkg + +# Mac +**/.DS_Store + +# IDEs +**/.vscode +**/.idea/ + +# Services +ts-services/parser-lambda/parser/test/data/local/* +ts-services/parser-lambda/parser/test/output/* +ts-services/parser-lambda/parser/playground.ts + +# Secrets +ts-services/dev-api-lambda/crypto-billing/*_secrets.ts +infra/ansible/vars/**/*-secrets.yaml +infra/ansible/vars/**/*-secrets.yml +infra/ansible/photon/vars/*-secrets.yaml +infra/ansible/photon/vars/*-secrets.yml +infra/ansible/rpc/templates/bt-creds*.json + +# Rust +**/target/ + +# CPU profiles +*.cpuprofile + +# Built helm charts +infra/**/*.tgz + +.gradle +.m2 +bin + +# Build output directies +/target +*/target +/build +*/build + +# Local Redis +dump.rdb + +infra/ansible/roles/Datadog.datadog +infra/ansible/photon/roles/datadog.datadog + + +scripts/data + +# Jupyter notebooks +*.ipynb + +# Personal Playground +personal_playground + + +# Used for local deployments of RPC operator +/rpc-operator/ +**/systest/*.json +**/router-benchmarks/*.json diff --git a/src/grpc_geyser.rs b/src/grpc_geyser.rs index 9d3fed1..01de114 100644 --- a/src/grpc_geyser.rs +++ b/src/grpc_geyser.rs @@ -1,10 +1,8 @@ -use std::sync::atomic::Ordering; use std::sync::Arc; -use std::time::Instant; use std::{collections::HashMap, time::Duration}; use cadence_macros::statsd_count; -use futures::{future::TryFutureExt, sink::SinkExt, stream::StreamExt}; +use futures::{sink::SinkExt, stream::StreamExt}; use rand::distributions::Alphanumeric; use rand::Rng; use tokio::time::sleep; diff --git a/src/priority_fee.rs b/src/priority_fee.rs index 8de8429..b74fe4a 100644 --- a/src/priority_fee.rs +++ b/src/priority_fee.rs @@ -13,12 +13,11 @@ use solana_sdk::{pubkey::Pubkey, slot_history::Slot}; use tracing::error; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; use yellowstone_grpc_proto::geyser::SubscribeUpdate; - use crate::grpc_consumer::GrpcConsumer; use crate::rpc_server::get_recommended_fee; use crate::slot_cache::SlotCache; -#[derive(Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub enum PriorityLevel { Min, // 0th percentile Low, // 25th percentile @@ -244,7 +243,7 @@ impl PriorityFeeTracker { }); } - fn push_priority_fee_for_txn( + pub fn push_priority_fee_for_txn( &self, slot: Slot, accounts: Vec, @@ -279,7 +278,7 @@ impl PriorityFeeTracker { &self, accounts: Vec, include_vote: bool, - lookback_period: Option, + lookback_period: Option, ) -> MicroLamportPriorityFeeEstimates { let start = std::time::Instant::now(); let mut account_fees = vec![]; @@ -403,7 +402,7 @@ mod tests { set_global_default(client) } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn test_specific_fee_estimates() { init_metrics(); let tracker = PriorityFeeTracker::new(10); @@ -442,7 +441,7 @@ mod tests { assert_eq!(estimates.unsafe_max, expected_max_fee); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn test_with_many_slots() { init_metrics(); let tracker = PriorityFeeTracker::new(101); @@ -482,7 +481,7 @@ mod tests { assert_eq!(estimates.unsafe_max, expected_max_fee); } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn test_with_many_slots_broken() { // same test as above but with an extra slot to throw off the value init_metrics(); diff --git a/src/rpc_server.rs b/src/rpc_server.rs index 43db208..270d509 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - env, fmt, + fmt, str::FromStr, sync::Arc, time::Instant, @@ -28,7 +28,7 @@ use tracing::info; pub struct AtlasPriorityFeeEstimator { pub priority_fee_tracker: Arc, - pub rpc_client: RpcClient, + pub rpc_client: Option, pub max_lookback_slots: usize, } @@ -41,29 +41,36 @@ impl fmt::Debug for AtlasPriorityFeeEstimator { } } -#[derive(Deserialize)] -#[serde(rename_all(serialize = "camelCase", deserialize = "camelCase"))] +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +#[serde( + rename_all(serialize = "camelCase", deserialize = "camelCase"), + deny_unknown_fields +)] pub struct GetPriorityFeeEstimateRequest { pub transaction: Option, // estimate fee for a txn pub account_keys: Option>, // estimate fee for a list of accounts pub options: Option, } -#[derive(Deserialize, Clone)] -#[serde(rename_all(serialize = "camelCase", deserialize = "camelCase"))] +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +#[serde( + rename_all(serialize = "camelCase", deserialize = "camelCase"), + deny_unknown_fields +)] pub struct GetPriorityFeeEstimateOptions { // controls input txn encoding pub transaction_encoding: Option, // controls custom priority fee level response pub priority_level: Option, // Default to MEDIUM pub include_all_priority_fee_levels: Option, // Include all priority level estimates in the response - pub lookback_slots: Option, // how many slots to look back on, default 50, min 1, max 300 - pub include_vote: Option, // include vote txns in the estimate + #[serde()] + pub lookback_slots: Option, // how many slots to look back on, default 50, min 1, max 300 + pub include_vote: Option, // include vote txns in the estimate // returns recommended fee, incompatible with custom controls. Currently the recommended fee is the median fee excluding vote txns pub recommended: Option, // return the recommended fee (median fee excluding vote txns) } -#[derive(Serialize, Clone)] +#[derive(Serialize, Clone, Debug, Default)] #[serde(rename_all(serialize = "camelCase", deserialize = "camelCase"))] pub struct GetPriorityFeeEstimateResponse { #[serde(skip_serializing_if = "Option::is_none")] @@ -113,7 +120,7 @@ fn validate_get_priority_fee_estimate_request( None } -/// returns account keys from transcation +/// returns account keys from transaction fn get_from_account_keys(transaction: &VersionedTransaction) -> Vec { transaction .message @@ -198,33 +205,35 @@ fn get_from_address_lookup_tables( } fn get_accounts( - rpc_client: &RpcClient, + rpc_client: &Option, get_priority_fee_estimate_request: GetPriorityFeeEstimateRequest, ) -> RpcResult> { if let Some(account_keys) = get_priority_fee_estimate_request.account_keys { return Ok(account_keys); } - if let Some(transaction) = get_priority_fee_estimate_request.transaction { - let tx_encoding = if let Some(options) = get_priority_fee_estimate_request.options { - options - .transaction_encoding - .unwrap_or(UiTransactionEncoding::Base58) - } else { - UiTransactionEncoding::Base58 - }; - let binary_encoding = tx_encoding.into_binary_encoding().ok_or_else(|| { - invalid_request(&format!( - "unsupported encoding: {tx_encoding}. Supported encodings: base58, base64" - )) - })?; - let (_, transaction) = - decode_and_deserialize::(transaction, binary_encoding)?; - let account_keys: Vec = vec![ - get_from_account_keys(&transaction), - get_from_address_lookup_tables(rpc_client, &transaction), - ] - .concat(); - return Ok(account_keys); + if let Some(rpc_client) = rpc_client { + if let Some(transaction) = get_priority_fee_estimate_request.transaction { + let tx_encoding = if let Some(options) = get_priority_fee_estimate_request.options { + options + .transaction_encoding + .unwrap_or(UiTransactionEncoding::Base58) + } else { + UiTransactionEncoding::Base58 + }; + let binary_encoding = tx_encoding.into_binary_encoding().ok_or_else(|| { + invalid_request(&format!( + "unsupported encoding: {tx_encoding}. Supported encodings: base58, base64" + )) + })?; + let (_, transaction) = + decode_and_deserialize::(transaction, binary_encoding)?; + let account_keys: Vec = vec![ + get_from_account_keys(&transaction), + get_from_address_lookup_tables(rpc_client, &transaction), + ] + .concat(); + return Ok(account_keys); + } } Ok(vec![]) } @@ -254,7 +263,7 @@ impl AtlasPriorityFeeEstimatorRpcServer for AtlasPriorityFeeEstimator { .collect(); let lookback_slots = options.clone().map(|o| o.lookback_slots).flatten(); if let Some(lookback_slots) = lookback_slots { - if lookback_slots < 1 || lookback_slots > self.max_lookback_slots { + if lookback_slots < 1 || lookback_slots as usize > self.max_lookback_slots { return Err(invalid_request("lookback_slots must be between 1 and 150")); } } @@ -310,7 +319,7 @@ impl AtlasPriorityFeeEstimator { ) -> Self { let server = AtlasPriorityFeeEstimator { priority_fee_tracker, - rpc_client: RpcClient::new(rpc_url), + rpc_client: Some(RpcClient::new(rpc_url)), max_lookback_slots, }; server @@ -339,3 +348,175 @@ pub fn get_recommended_fee(priority_fee_levels: MicroLamportPriorityFeeEstimates let recommended_with_buffer = recommended * (1.0 + RECOMMENDED_FEE_SAFETY_BUFFER); recommended_with_buffer.ceil() } + +#[cfg(test)] +mod tests { + use crate::priority_fee::PriorityFeeTracker; + use crate::rpc_server::{ + AtlasPriorityFeeEstimator, AtlasPriorityFeeEstimatorRpcServer, + GetPriorityFeeEstimateOptions, GetPriorityFeeEstimateRequest, + }; + use cadence::{NopMetricSink, StatsdClient}; + use jsonrpsee::core::Cow; + use jsonrpsee::core::__reexports::serde_json; + use jsonrpsee::core::__reexports::serde_json::value::RawValue; + use jsonrpsee::types::{Id, Request, TwoPointZero}; + use solana_sdk::clock::Slot; + use solana_sdk::pubkey::Pubkey; + use std::sync::Arc; + + #[tokio::test(flavor = "current_thread")] + async fn test_calculating_fees_with_all_options_none() { + prep_statsd(); + + let acc1 = Pubkey::new_unique(); + let acc2 = Pubkey::new_unique(); + let tracker = PriorityFeeTracker::new(150); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc1, acc2], 100u64, false); + + let server = AtlasPriorityFeeEstimator { + priority_fee_tracker: Arc::new(tracker), + rpc_client: None, + max_lookback_slots: 150, + }; + + let result = server.get_priority_fee_estimate(GetPriorityFeeEstimateRequest { + account_keys: Some(vec![acc1.to_string(), acc2.to_string()]), + options: Some(GetPriorityFeeEstimateOptions::default()), + ..Default::default() + }); + let resp = result.unwrap(); + assert_eq!(resp.priority_fee_estimate, Some(100.0)); + assert!(resp.priority_fee_levels.is_none()); + } + + #[tokio::test(flavor = "current_thread")] + async fn test_calculating_fees_with_no_options() { + prep_statsd(); + + let acc1 = Pubkey::new_unique(); + let acc2 = Pubkey::new_unique(); + let tracker = PriorityFeeTracker::new(150); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc1, acc2], 100u64, false); + + let server = AtlasPriorityFeeEstimator { + priority_fee_tracker: Arc::new(tracker), + rpc_client: None, + max_lookback_slots: 150, + }; + + let result = server.get_priority_fee_estimate(GetPriorityFeeEstimateRequest { + account_keys: Some(vec![acc1.to_string(), acc2.to_string()]), + ..Default::default() + }); + let resp = result.unwrap(); + assert_eq!(resp.priority_fee_estimate, Some(100.0)); + assert!(resp.priority_fee_levels.is_none()); + } + + #[tokio::test(flavor = "current_thread")] + async fn test_calculating_all_fees() { + prep_statsd(); + + let acc1 = Pubkey::new_unique(); + let acc2 = Pubkey::new_unique(); + let tracker = PriorityFeeTracker::new(150); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc1], 100u64, false); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc2], 200u64, false); + + let server = AtlasPriorityFeeEstimator { + priority_fee_tracker: Arc::new(tracker), + rpc_client: None, + max_lookback_slots: 150, + }; + + let result = server.get_priority_fee_estimate(GetPriorityFeeEstimateRequest { + account_keys: Some(vec![acc1.to_string(), acc2.to_string()]), + options: Some(GetPriorityFeeEstimateOptions { + include_all_priority_fee_levels: Some(true), + ..Default::default() + }), + ..Default::default() + }); + let resp = result.unwrap(); + let levels = resp.priority_fee_levels.unwrap(); + assert_eq!(levels.min, 100.0); + assert_eq!(levels.low, 125.0); + assert_eq!(levels.medium, 150.0); + assert_eq!(levels.high, 175.0); + assert_eq!(levels.very_high, 195.0); + assert_eq!(levels.unsafe_max, 200.0); + assert!(resp.priority_fee_estimate.is_none()); + } + #[tokio::test(flavor = "current_thread")] + async fn test_calculating_recommended_given_very_low_calculated_fee() { + prep_statsd(); + + let acc1 = Pubkey::new_unique(); + let acc2 = Pubkey::new_unique(); + let tracker = PriorityFeeTracker::new(150); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc1], 100u64, false); + tracker.push_priority_fee_for_txn(1 as Slot, vec![acc2], 200u64, false); + + let server = AtlasPriorityFeeEstimator { + priority_fee_tracker: Arc::new(tracker), + rpc_client: None, + max_lookback_slots: 150, + }; + + let result = server.get_priority_fee_estimate(GetPriorityFeeEstimateRequest { + account_keys: Some(vec![acc1.to_string(), acc2.to_string()]), + options: Some(GetPriorityFeeEstimateOptions { + recommended: Some(true), + ..Default::default() + }), + ..Default::default() + }); + let resp = result.unwrap(); + assert!(resp.priority_fee_levels.is_none()); + assert_eq!(resp.priority_fee_estimate, Some(10500.0)); + } + + #[test] + fn test_parsing_wrong_fields() { + for (param, error) in bad_params() { + let json_val = format!("{{\"jsonrpc\": \"2.0\",\"id\": \"1\", \"method\": \"getPriorityFeeEstimate\", \"params\": {param} }}"); + let res = serde_json::from_str::(json_val.as_str()); + let res = res.unwrap(); + assert_request(&res, Id::Str(Cow::const_str("1")), "getPriorityFeeEstimate"); + + let params: serde_json::error::Result = + serde_json::from_str(res.params.map(RawValue::get).unwrap()); + assert!(params.is_err()); + assert_eq!(params.err().unwrap().to_string(), error, "testing {param}"); + } + } + + fn prep_statsd() { + let systemd_client = StatsdClient::builder("test", NopMetricSink) + .with_error_handler(|e| eprintln!("metric error: {}", e)) + .build(); + cadence_macros::set_global_default(systemd_client); + } + + fn assert_request<'a>(request: &Request<'a>, id: Id<'a>, method: &str) { + assert_eq!(request.jsonrpc, TwoPointZero); + assert_eq!(request.id, id); + assert_eq!(request.method, method); + } + + fn bad_params<'a>() -> Vec<(&'a str, &'a str)> { + vec![ + (r#"{"transactions": null}"#,"unknown field `transactions`, expected one of `transaction`, `accountKeys`, `options` at line 1 column 15"), + (r#"{"account_keys": null}"#,"unknown field `account_keys`, expected one of `transaction`, `accountKeys`, `options` at line 1 column 15"), + (r#"{"accountkeys": null}"#,"unknown field `accountkeys`, expected one of `transaction`, `accountKeys`, `options` at line 1 column 14"), + (r#"{"accountKeys": [1, 2]}"#, "invalid type: integer `1`, expected a string at line 1 column 18"), + (r#"{"option": null}"#, "unknown field `option`, expected one of `transaction`, `accountKeys`, `options` at line 1 column 9"), + (r#"{"options": {"transaction_encoding":null}}"#, "unknown field `transaction_encoding`, expected one of `transactionEncoding`, `priorityLevel`, `includeAllPriorityFeeLevels`, `lookbackSlots`, `includeVote`, `recommended` at line 1 column 35"), + (r#"{"options": {"priorityLevel":"HIGH"}}"#, "unknown variant `HIGH`, expected one of `Min`, `Low`, `Medium`, `High`, `VeryHigh`, `UnsafeMax`, `Default` at line 1 column 35"), + (r#"{"options": {"includeAllPriorityFeeLevels":"no"}}"#, "invalid type: string \"no\", expected a boolean at line 1 column 47"), + (r#"{"options": {"lookbackSlots":"no"}}"#, "invalid type: string \"no\", expected u32 at line 1 column 33"), + (r#"{"options": {"lookbackSlots":"-1"}}"#, "invalid type: string \"-1\", expected u32 at line 1 column 33"), + ] + } +}