diff --git a/src/priority_fee.rs b/src/priority_fee.rs index b96a19e..b9cc959 100644 --- a/src/priority_fee.rs +++ b/src/priority_fee.rs @@ -12,6 +12,7 @@ use solana_program_runtime::prioritization_fee::PrioritizationFeeDetails; use solana_sdk::instruction::CompiledInstruction; use solana_sdk::transaction::TransactionError; use solana_sdk::{pubkey::Pubkey, slot_history::Slot}; +use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::mpsc::{channel, Receiver, Sender}; @@ -100,7 +101,7 @@ pub struct PriorityFeeTracker { priority_fees: Arc, compute_budget: ComputeBudget, slot_cache: SlotCache, - sampling_sender: Sender<(Vec, bool, Option)>, + sampling_sender: Sender<(Vec, bool, bool, Option)>, } #[derive(Debug, Copy, Clone, PartialEq)] @@ -298,7 +299,7 @@ impl GrpcConsumer for PriorityFeeTracker { impl PriorityFeeTracker { pub fn new(slot_cache_length: usize) -> Self { - let (sampling_txn, sampling_rxn) = channel::<(Vec, bool, Option)>(100); + let (sampling_txn, sampling_rxn) = channel::<(Vec, bool, bool, Option)>(100); let tracker = Self { priority_fees: Arc::new(DashMap::new()), @@ -310,7 +311,7 @@ impl PriorityFeeTracker { tracker } - fn poll_fees(&self, mut sampling_rxn: Receiver<(Vec, bool, Option)>) { + fn poll_fees(&self, mut sampling_rxn: Receiver<(Vec, bool, bool, Option)>) { { let priority_fee_tracker = self.clone(); // task to run global fee comparison every 1 second @@ -328,8 +329,8 @@ impl PriorityFeeTracker { tokio::spawn(async move { loop { match sampling_rxn.recv().await { - Some((accounts, include_vote, lookback_period)) => priority_fee_tracker - .record_specific_fees(accounts, include_vote, lookback_period), + Some((accounts, include_vote, include_empty_slots, lookback_period)) => priority_fee_tracker + .record_specific_fees(accounts, include_vote, include_empty_slots, lookback_period), _ => {} } } @@ -338,7 +339,7 @@ impl PriorityFeeTracker { } fn record_general_fees(&self) { - let global_fees = self.calculation1(&vec![], false, &None); + let global_fees = self.calculation1(&vec![], false, false, &None); statsd_gauge!( "min_priority_fee", global_fees.min as u64, @@ -376,11 +377,12 @@ impl PriorityFeeTracker { &self, accounts: Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: Option, ) { - let old_fee = self.calculation1(&accounts, include_vote, &lookback_period); - let new_fee = self.calculation2(&accounts, include_vote, &lookback_period); - let new_fee_last = self.calculation2(&accounts, include_vote, &Some(1)); + let old_fee = self.calculation1(&accounts, include_vote, include_empty_slots, &lookback_period); + let new_fee = self.calculation2(&accounts, include_vote, include_empty_slots, &lookback_period); + let new_fee_last = self.calculation2(&accounts, include_vote, include_empty_slots, &Some(1)); statsd_gauge!( "min_priority_fee", @@ -529,14 +531,15 @@ impl PriorityFeeTracker { &self, accounts: Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: Option, calculation1: bool, ) -> MicroLamportPriorityFeeEstimates { let start = Instant::now(); let micro_lamport_priority_fee_estimates = if calculation1 { - self.calculation1(&accounts, include_vote, &lookback_period) + self.calculation1(&accounts, include_vote, include_empty_slots, &lookback_period) } else { - self.calculation2(&accounts, include_vote, &lookback_period) + self.calculation2(&accounts, include_vote, include_empty_slots, &lookback_period) }; let version = if calculation1 { "v1" } else { "v2" }; @@ -553,6 +556,7 @@ impl PriorityFeeTracker { if let Err(e) = self.sampling_sender.try_send(( accounts.to_owned(), include_vote, + include_empty_slots, lookback_period.to_owned(), )) { debug!("Did not add sample for calculation, {:?}", e); @@ -572,6 +576,7 @@ impl PriorityFeeTracker { &self, accounts: &Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: &Option, ) -> MicroLamportPriorityFeeEstimates { let mut account_fees = vec![]; @@ -596,6 +601,11 @@ impl PriorityFeeTracker { } } } + if include_empty_slots { + // if there are less data than number of slots - append 0s for up to number of slots so we don't overestimate the values + account_fees.resize(150, 0f64); + } + let micro_lamport_priority_fee_estimates = MicroLamportPriorityFeeEstimates { min: max_percentile(&mut account_fees, &mut transaction_fees, 0), low: max_percentile(&mut account_fees, &mut transaction_fees, 25), @@ -604,6 +614,7 @@ impl PriorityFeeTracker { very_high: max_percentile(&mut account_fees, &mut transaction_fees, 95), unsafe_max: max_percentile(&mut account_fees, &mut transaction_fees, 100), }; + micro_lamport_priority_fee_estimates } @@ -617,6 +628,7 @@ impl PriorityFeeTracker { &self, accounts: &Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: &Option, ) -> MicroLamportPriorityFeeEstimates { let mut slots_vec = Vec::with_capacity(self.slot_cache.len()); @@ -624,9 +636,11 @@ impl PriorityFeeTracker { slots_vec.sort(); slots_vec.reverse(); + let slots_vec = slots_vec; + let lookback = calculate_lookback_size(&lookback_period, slots_vec.len()); - let mut fees = vec![]; + let mut fees = Vec::with_capacity(slots_vec.len()); let mut micro_lamport_priority_fee_estimates = MicroLamportPriorityFeeEstimates::default(); for slot in &slots_vec[..lookback] { @@ -654,6 +668,10 @@ impl PriorityFeeTracker { } } } + if include_empty_slots { + // if there are less data than number of slots - append 0s for up to number of slots so we don't overestimate the values + fees.resize(slots_vec.len(), 0f64); + } micro_lamport_priority_fee_estimates = estimate_max_values(&mut fees, micro_lamport_priority_fee_estimates); } @@ -665,12 +683,41 @@ fn estimate_max_values( mut fees: &mut Vec, mut estimates: MicroLamportPriorityFeeEstimates, ) -> MicroLamportPriorityFeeEstimates { - estimates.min = percentile(&mut fees, 0).max(estimates.min); - estimates.low = percentile(&mut fees, 25).max(estimates.low); - estimates.medium = percentile(&mut fees, 50).max(estimates.medium); - estimates.high = percentile(&mut fees, 75).max(estimates.high); - estimates.very_high = percentile(&mut fees, 95).max(estimates.very_high); - estimates.unsafe_max = percentile(&mut fees, 100).max(estimates.unsafe_max); + let vals: HashMap = percentile( + &mut fees, + &[ + PriorityLevel::Min.into(), + PriorityLevel::Low.into(), + PriorityLevel::Medium.into(), + PriorityLevel::High.into(), + PriorityLevel::VeryHigh.into(), + PriorityLevel::UnsafeMax.into(), + ], + ); + estimates.min = vals + .get(&PriorityLevel::Min.into()) + .unwrap_or(&estimates.min) + .max(estimates.min); + estimates.low = vals + .get(&PriorityLevel::Low.into()) + .unwrap_or(&estimates.low) + .max(estimates.low); + estimates.medium = vals + .get(&PriorityLevel::Medium.into()) + .unwrap_or(&estimates.medium) + .max(estimates.medium); + estimates.high = vals + .get(&PriorityLevel::High.into()) + .unwrap_or(&estimates.high) + .max(estimates.high); + estimates.very_high = vals + .get(&PriorityLevel::VeryHigh.into()) + .unwrap_or(&estimates.very_high) + .max(estimates.very_high); + estimates.unsafe_max = vals + .get(&PriorityLevel::UnsafeMax.into()) + .unwrap_or(&estimates.unsafe_max) + .max(estimates.unsafe_max); estimates } @@ -725,30 +772,40 @@ fn max_percentile( transaction_fees: &mut Vec, percentile_value: Percentile, ) -> f64 { + let results1 = percentile(account_fees, &[percentile_value]); + let results2 = percentile(transaction_fees, &[percentile_value]); max( - percentile(account_fees, percentile_value), - percentile(transaction_fees, percentile_value), + *results1.get(&percentile_value).unwrap_or(&0f64), + *results2.get(&percentile_value).unwrap_or(&0f64), ) } // pulled from here - https://www.calculatorsoup.com/calculators/statistics/percentile-calculator.php // couldn't find any good libraries that worked -fn percentile(values: &mut Vec, percentile: Percentile) -> f64 { +fn percentile(values: &mut Vec, percentiles: &[Percentile]) -> HashMap { if values.is_empty() { - return 0.0; + return HashMap::with_capacity(0); } + values.sort_by(|a, b| a.partial_cmp(b).unwrap()); let n = values.len() as f64; - let r = (percentile as f64 / 100.0) * (n - 1.0) + 1.0; - let val = if r.fract() == 0.0 { - values[r as usize - 1] - } else { - let ri = r.trunc() as usize - 1; - let rf = r.fract(); - values[ri] + rf * (values[ri + 1] - values[ri]) - }; - val + percentiles.into_iter().fold( + HashMap::with_capacity(percentiles.len()), + |mut data, &percentile| { + let r = (percentile as f64 / 100.0) * (n - 1.0) + 1.0; + + let val = if r.fract() == 0.0 { + values[r as usize - 1] + } else { + let ri = r.trunc() as usize - 1; + let rf = r.fract(); + values[ri] + rf * (values[ri + 1] - values[ri]) + }; + data.insert(percentile, val); + data + }, + ) } #[cfg(test)] @@ -793,10 +850,10 @@ mod tests { let estimates = tracker.calculation1(&vec![accounts.get(0).unwrap().clone()], false, &None); // Since the fixed fees are evenly distributed, the 50th percentile should be the middle value let expected_min_fee = 0.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -809,10 +866,10 @@ mod tests { let estimates = tracker.calculation2(&vec![accounts.get(0).unwrap().clone()], false, &None); // Since the fixed fees are evenly distributed, the 50th percentile should be the middle value let expected_min_fee = 0.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -847,11 +904,11 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation1(&vec![accounts.get(0).unwrap().clone()], false, &None); - let expected_min_fee = 1.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_min_fee = 0.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -862,11 +919,11 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation2(&vec![accounts.get(0).unwrap().clone()], false, &None); - let expected_min_fee = 1.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_min_fee = 0.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -969,10 +1026,10 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation1(&vec![account_1, account_4], false, &None); let expected_min_fee = 0.0; - let expected_low_fee = 24.75; - let expected_medium_fee = 49.5; - let expected_high_fee = 86.75; - let expected_very_high_fee = 96.55; + let expected_low_fee = 0.0; + let expected_medium_fee = 24.5; + let expected_high_fee = 61.75; + let expected_very_high_fee = 91.54999999999998; let expected_max_fee = 99.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -983,11 +1040,11 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation2(&vec![account_1, account_4], false, &None); - let expected_min_fee = 75.0; - let expected_low_fee = 81.0; - let expected_medium_fee = 87.0; - let expected_high_fee = 93.0; - let expected_very_high_fee = 97.8; + let expected_min_fee = 0.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 24.5; + let expected_high_fee = 61.75; + let expected_very_high_fee = 91.54999999999998; let expected_max_fee = 99.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -1045,10 +1102,10 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation1(&vec![account_1, account_4], false, &None); let expected_min_fee = 0.0; - let expected_low_fee = 24.75; - let expected_medium_fee = 49.5; - let expected_high_fee = 74.25; - let expected_very_high_fee = 94.05; + let expected_low_fee = 0.0; + let expected_medium_fee = 24.5; + let expected_high_fee = 61.75; + let expected_very_high_fee = 91.54999999999998; let expected_max_fee = 99.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -1059,11 +1116,11 @@ mod tests { // Now test the fee estimates for a known priority level, let's say medium (50th percentile) let estimates = tracker.calculation2(&vec![account_1, account_4], false, &None); - let expected_min_fee = 3.0; - let expected_low_fee = 27.0; - let expected_medium_fee = 51.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 94.19999999999999; + let expected_min_fee = 0.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 24.5; + let expected_high_fee = 61.75; + let expected_very_high_fee = 91.54999999999998; let expected_max_fee = 99.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -1097,13 +1154,14 @@ mod tests { } // Now test the fee estimates for a known priority level, let's say medium (50th percentile) - let estimates = tracker.calculation1(&vec![accounts.get(0).unwrap().clone()], false, &None); + let v = vec![accounts.get(0).unwrap().clone()]; + let estimates = tracker.calculation1(&v, false, &None); // Since the fixed fees are evenly distributed, the 50th percentile should be the middle value let expected_min_fee = 0.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); @@ -1116,10 +1174,10 @@ mod tests { let estimates = tracker.calculation2(&vec![accounts.get(0).unwrap().clone()], false, &None); // Since the fixed fees are evenly distributed, the 50th percentile should be the middle value let expected_min_fee = 0.0; - let expected_low_fee = 25.0; - let expected_medium_fee = 50.0; - let expected_high_fee = 75.0; - let expected_very_high_fee = 95.0; + let expected_low_fee = 0.0; + let expected_medium_fee = 25.5; + let expected_high_fee = 62.75; + let expected_very_high_fee = 92.54999999999998; let expected_max_fee = 100.0; assert_eq!(estimates.min, expected_min_fee); assert_eq!(estimates.low, expected_low_fee); diff --git a/src/rpc_server.rs b/src/rpc_server.rs index ea0672b..892c7c2 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -70,6 +70,7 @@ pub struct GetPriorityFeeEstimateOptionsLight { pub include_vote: Option, // include vote txns in the estimate // returns recommended fee, incompatible with custom controls. Currently the recommended fee is the median fee excluding vote txns pub recommended: Option, // return the recommended fee (median fee excluding vote txns) + pub include_empty_slots: Option, // if true than slots with no transactions will be treated as 0 } impl Into for GetPriorityFeeEstimateRequestLight { @@ -84,6 +85,7 @@ impl Into for GetPriorityFeeEstimateRequestLight lookback_slots: o.lookback_slots, include_vote: o.include_vote, recommended: o.recommended, + include_empty_slots: o.include_empty_slots, } }); @@ -123,6 +125,7 @@ pub struct GetPriorityFeeEstimateOptions { pub include_vote: Option, // include vote txns in the estimate // returns recommended fee, incompatible with custom controls. Currently the recommended fee is the median fee excluding vote txns pub recommended: Option, // return the recommended fee (median fee excluding vote txns) + pub include_empty_slots: Option, // if true than slots with no transactions will be treated as 0 } #[derive(Serialize, Clone, Debug, Default)] @@ -146,6 +149,7 @@ impl Into for GetPriorityFeeEstimateRequest lookback_slots: o.lookback_slots, include_vote: o.include_vote, recommended: o.recommended, + include_empty_slots: o.include_empty_slots, } }); @@ -376,11 +380,13 @@ impl AtlasPriorityFeeEstimatorRpcServer for AtlasPriorityFeeEstimator { ) -> RpcResult { let algo_run_fn = |accounts: Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: Option| -> MicroLamportPriorityFeeEstimates { self.priority_fee_tracker.get_priority_fee_estimates( accounts, include_vote, + include_empty_slots, lookback_period, true, ) @@ -395,11 +401,13 @@ impl AtlasPriorityFeeEstimatorRpcServer for AtlasPriorityFeeEstimator { ) -> RpcResult { let algo_run_fn = |accounts: Vec, include_vote: bool, + include_empty_slots: bool, lookback_period: Option| -> MicroLamportPriorityFeeEstimates { self.priority_fee_tracker.get_priority_fee_estimates( accounts, include_vote, + include_empty_slots, lookback_period, false, ) @@ -425,7 +433,7 @@ impl AtlasPriorityFeeEstimator { fn execute_priority_fee_estimate_coordinator( &self, get_priority_fee_estimate_request: GetPriorityFeeEstimateRequest, - priority_fee_calc_fn: impl FnOnce(Vec, bool, Option) -> MicroLamportPriorityFeeEstimates, + priority_fee_calc_fn: impl FnOnce(Vec, bool, bool, Option) -> MicroLamportPriorityFeeEstimates, ) -> RpcResult { let options = get_priority_fee_estimate_request.options.clone(); @@ -449,7 +457,11 @@ impl AtlasPriorityFeeEstimator { } } let include_vote = should_include_vote(&options); - let priority_fee_levels = priority_fee_calc_fn(accounts, include_vote, lookback_slots); + let include_empty_slots = should_include_empty_slots(&options); + let priority_fee_levels = priority_fee_calc_fn(accounts, + include_vote, + include_empty_slots, + lookback_slots); if let Some(options) = options.clone() { if options.include_all_priority_fee_levels == Some(true) { return Ok(GetPriorityFeeEstimateResponse { @@ -496,6 +508,13 @@ fn should_include_vote(options: &Option) -> bool true } +fn should_include_empty_slots(options: &Option) -> bool { + if let Some(options) = options { + return options.include_empty_slots.unwrap_or(false); + } + false +} + const MIN_RECOMMENDED_PRIORITY_FEE: f64 = 10_000.0; pub fn get_recommended_fee(priority_fee_levels: MicroLamportPriorityFeeEstimates) -> f64 {