From 62077ec21f92830e060cefbcf1effdf553f03d93 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 11:28:07 +0100 Subject: [PATCH 01/12] add logging to server --- Cargo.lock | 96 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 10 ++++ src/bin/serve/config.rs | 5 +- src/bin/serve/handlers.rs | 82 +++++++++++++++++++++++++-------- src/bin/serve/main.rs | 8 ++-- src/bin/serve/server.rs | 3 +- 6 files changed, 178 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51b4e90..711097c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1160,6 +1160,9 @@ dependencies = [ "serde", "serde_json", "tokio", + "tracing", + "tracing-subscriber", + "uuid", ] [[package]] @@ -1863,6 +1866,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -1962,6 +1975,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parity-scale-codec" version = "3.6.9" @@ -2573,6 +2592,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -2744,6 +2772,16 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "threadpool" version = "1.8.1" @@ -2918,6 +2956,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -2994,6 +3058,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom", + "rand", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3112,6 +3186,28 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 737131e..296c15f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,16 @@ jsonrpc = { version = "0.18.0", optional = true } axum = { version = "0.6.20", features = ["headers"] } serde = "1.0.188" dotenv = "0.15.0" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" + +[dependencies.uuid] +version = "1.8.0" +features = [ + "v4", + "fast-rng", +] + [dev-dependencies] reqwest = { version = "0.12.4", features = ["json"] } diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index 2f54e94..2725188 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -3,7 +3,7 @@ use eyre::Result; pub const DEFAULT_HOST: &str = "127.0.0.1"; -const DEFAULT_PORT: u32 = 3001; +const DEFAULT_PORT: u32 = 3000; pub struct Config { pub server_addr: String, @@ -20,8 +20,7 @@ pub enum RpcUrl { Primary(String), Fallack(String), } -// Server running on: 127.0.0.1:8080 -// Server running on: 127.0.0.1:8080 + impl Config { pub fn from_env() -> Result { diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index 0d68fc8..e065ce4 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -3,14 +3,15 @@ use serde::{Serialize, Deserialize}; use std::time::Instant; use axum::{ response::{Json, IntoResponse, Response as AxumResponse}, - http::StatusCode, extract::{Path, State}, + http::StatusCode, }; +use tracing::info; use super::state::{Chain, AppState}; #[derive(Debug, Serialize)] -struct Response { +pub struct Response { success: bool, #[serde(skip_serializing_if = "Option::is_none")] msg: Option, @@ -18,6 +19,26 @@ struct Response { error: Option, } +impl From for Response { + fn from(msg: SearchResponse) -> Self { + Self { + success: true, + msg: Some(msg), + error: None, + } + } +} + +impl From<&AppError> for Response { + fn from(err: &AppError) -> Self { + Self { + success: false, + msg: None, + error: Some(err.0.to_string()), + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct SearchResponse { token: Address, @@ -28,17 +49,14 @@ pub struct SearchResponse { lang: String, } +#[derive(Debug)] pub struct AppError(eyre::Error); impl IntoResponse for AppError { fn into_response(self) -> AxumResponse { ( StatusCode::INTERNAL_SERVER_ERROR, - serde_json::to_string(&Response { - success: false, - msg: None, - error: Some(self.0.to_string()), - }).unwrap(), + serde_json::to_string(&Response::from(&self)).unwrap(), ) .into_response() } @@ -53,28 +71,56 @@ where } } +// todo: split expected and unexpected errors pub async fn search_handler( State(app_state): State>, Path((chain_str, token_str)): Path<(String, String)> -) -> Result, AppError> +) -> Result, AppError> + where T: Sync + Send + Clone + 'static +{ + let rtime0 = Instant::now(); + let request_id = uuid::Uuid::new_v4().as_u128().to_string(); + info!("{}", serde_json::json!({ + "msg": "new_request", + "handler": "search_handler", + "id": request_id, + "args": serde_json::json!({ + "chain": chain_str, + "token": token_str, + }), + })); + let res = _search_handler(State(app_state), Path((chain_str, token_str))).await; + + let res_str = match &res { + Ok(Json(res)) => serde_json::to_string(res)?, + Err(ref err) => serde_json::to_string(&Response::from(err))?, + }; + let duration_ms = rtime0.elapsed().as_millis(); + info!("{}", serde_json::json!({ + "msg": "new_response", + "handler": "search_handler", + "id": request_id, + "response": res_str, + "duration": duration_ms, + })); + res +} + +async fn _search_handler( + State(app_state): State>, + Path((chain_str, token_str)): Path<(String, String)> +) -> Result, AppError> where T: Sync + Send + Clone + 'static { - // todo: logging + clean let chain = chain_str.parse::()?; let token: Address = token_str.parse().map_err(|_| eyre::eyre!("Invalid token"))?; - println!("Searching for token: {token:?} on chain {chain:?}"); - let now = Instant::now(); - let endpoint = &app_state.providers .get(&chain) .ok_or_else(|| eyre::eyre!(format!("Can't find provider for chain: {chain:?}")))? .endpoint; - let response = erc20_topup::find_slot(&endpoint, token, None).await?; - let t2 = now.elapsed(); - println!("{:?}", response); let response = SearchResponse { token: token, contract: response.0, @@ -82,8 +128,6 @@ pub async fn search_handler( update_ratio: response.2, lang: response.3, }; - println!("{:?}", response); - println!("Time taken: {:?}", t2); - Ok(Json(response)) -} + Ok(Json(response.into())) +} \ No newline at end of file diff --git a/src/bin/serve/main.rs b/src/bin/serve/main.rs index 6853357..236c18c 100644 --- a/src/bin/serve/main.rs +++ b/src/bin/serve/main.rs @@ -5,6 +5,7 @@ mod state; mod config; use std::sync::Arc; +use tracing::info; use eyre::Result; use config::{Config, RpcUrl}; @@ -13,10 +14,11 @@ use config::{Config, RpcUrl}; async fn main() -> Result<()> { let config = Config::from_env()?; if config.chain_configs.is_empty() { - eprintln!("No endpoint for any chain"); - return Ok(()); + return Err(eyre::eyre!("No chain configs found")); } + tracing_subscriber::fmt::init(); // todo: only if logging is on + let mut app_state = state::AppProviders::new(); for chain_config in config.chain_configs { let (endpoint, handler) = match chain_config.rpc_url { @@ -26,7 +28,7 @@ async fn main() -> Result<()> { (anvil.endpoint(), Some(Arc::new(anvil))) }, }; - println!("Added provider for chain: {:?} with endpoint {endpoint:?}", chain_config.chain); + info!("Added provider for chain: {:?} with endpoint {endpoint:?}", chain_config.chain); app_state.set_provider(chain_config.chain, endpoint, handler); } diff --git a/src/bin/serve/server.rs b/src/bin/serve/server.rs index 13f2e24..ba619d5 100644 --- a/src/bin/serve/server.rs +++ b/src/bin/serve/server.rs @@ -1,4 +1,5 @@ use axum::{Router, routing::get}; +use tracing::info; use eyre::Result; use crate::handlers::search_handler; @@ -12,7 +13,7 @@ pub async fn run(addr: &str, state: AppState) -> Result<()> .route("/:chain/:token", get(search_handler)) .with_state(state); - println!("Server running on: {addr}"); + info!("Server running on: {addr}"); axum::Server::bind(&addr.parse().unwrap()) .serve(app.into_make_service()) From 606a9fda5c40c5f93dc3c8543863527d5fdc7a03 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 13:21:43 +0100 Subject: [PATCH 02/12] redis db --- Cargo.lock | 32 ++++++++++++++++++++++++++++ Cargo.toml | 1 + src/bin/serve/.env.sample | 10 ++++++++- src/bin/serve/config.rs | 41 ++++++++++++++++++++++++++++++++++-- src/bin/serve/db.rs | 44 +++++++++++++++++++++++++++++++++++++++ src/bin/serve/handlers.rs | 16 +++++++++++++- src/bin/serve/main.rs | 25 +++++++++++++++------- src/bin/serve/state.rs | 18 +++++++++++++--- 8 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 src/bin/serve/db.rs diff --git a/Cargo.lock b/Cargo.lock index 711097c..ab212f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -931,6 +931,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-hex" version = "1.11.3" @@ -1156,6 +1166,7 @@ dependencies = [ "hex", "jsonrpc", "rand", + "redis", "reqwest", "serde", "serde_json", @@ -2241,6 +2252,21 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6472825949c09872e8f2c50bde59fcefc17748b6be5c90fd67cd8b4daca73bfd" +dependencies = [ + "combine", + "itoa", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2", + "url", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2571,6 +2597,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.8" diff --git a/Cargo.toml b/Cargo.toml index 296c15f..e15612d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ serde = "1.0.188" dotenv = "0.15.0" tracing = "0.1.40" tracing-subscriber = "0.3.18" +redis = "0.25.3" [dependencies.uuid] version = "1.8.0" diff --git a/src/bin/serve/.env.sample b/src/bin/serve/.env.sample index cb48667..d8da079 100644 --- a/src/bin/serve/.env.sample +++ b/src/bin/serve/.env.sample @@ -1,5 +1,6 @@ SERVER_PORT=8080 SERVER_HOST=localhost +LOGGING_ENABLED=1 # RPCs with debug api and struct trace enabled (can be omitted) ETHEREUM_RPC= @@ -12,4 +13,11 @@ AVALANCHE_RPC= ETHEREUM_FORK_RPC=https://eth.llamarpc.com ARBITRUM_FORK_RPC=https://arb1.arbitrum.io/rpc OPTIMISM_FORK_RPC=https://optimism.llamarpc.com -AVALANCHE_FORK_RPC=https://avalanche.public-rpc.com \ No newline at end of file +AVALANCHE_FORK_RPC=https://avalanche.public-rpc.com + +# Redis (optional) +REDIS_ENABLED=1 +REDIS_HOST=localhost +REDIS_PORT=9090 +REDIS_PASSWORD= +REDIS_IS_TLS=1 \ No newline at end of file diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index 2725188..bf8189c 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -8,6 +8,8 @@ const DEFAULT_PORT: u32 = 3000; pub struct Config { pub server_addr: String, pub chain_configs: Vec, + pub redis_config: Option, + pub logging_enabled: bool, } pub struct ChainConfig { @@ -15,19 +17,26 @@ pub struct ChainConfig { pub rpc_url: RpcUrl, } -#[derive(Debug)] pub enum RpcUrl { Primary(String), Fallack(String), } +#[derive(Default)] +pub struct RedisConfig { + pub addr: String, + pub password: Option, + pub is_tls: bool, +} + impl Config { pub fn from_env() -> Result { let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push("./src/bin/serve/.env"); + path.push("./src/bin/serve/.env"); // todo: make this more robust dotenv::from_path(path).ok(); + // Server config let host = std::env::var("SERVER_HOST").ok() .and_then(|s| (!s.trim().is_empty()).then(|| s)) .unwrap_or(DEFAULT_HOST.to_string()); @@ -36,6 +45,7 @@ impl Config { .unwrap_or(DEFAULT_PORT); let server_addr = format!("{}:{}", host, port); + // Chain & RPCs config let available_chains = vec![ Chain::Ethereum, Chain::Arbitrum, @@ -56,8 +66,35 @@ impl Config { .map(|url| ChainConfig { chain, rpc_url: url }) }).collect::>(); + // Redis config + let redis_enabled = std::env::var("REDIS_ENABLED").ok() + .map(|s| s == "1").unwrap_or(false); + let redis_config = + if redis_enabled { + let mut redis_config = RedisConfig::default(); + let redis_host = std::env::var("REDIS_HOST").ok() + .and_then(|s| (!s.trim().is_empty()).then(|| s)) + .unwrap_or("localhost".to_string()); + let redis_port = std::env::var("REDIS_PORT").ok() + .and_then(|p_str| p_str.parse::().ok()) + .unwrap_or(6379); + redis_config.addr = format!("{}:{}", redis_host, redis_port); + redis_config.password = std::env::var("REDIS_PASSWORD").ok(); + redis_config.is_tls = std::env::var("REDIS_IS_TLS").ok() + .map(|s| s == "1").unwrap_or(false); + Some(redis_config) + } else { + None + }; + + // Logging config + let logging_enabled = std::env::var("LOGGING_ENABLED").ok() + .map(|s| s == "1").unwrap_or(false); + Ok(Self { + logging_enabled, chain_configs, + redis_config, server_addr, }) } diff --git a/src/bin/serve/db.rs b/src/bin/serve/db.rs new file mode 100644 index 0000000..7b0ac60 --- /dev/null +++ b/src/bin/serve/db.rs @@ -0,0 +1,44 @@ +use redis::{Commands, Connection}; +use alloy::primitives::Address; +use eyre::Result; +use crate::handlers::SearchResponse; +use crate::config::RedisConfig; +use crate::state::Chain; + + +pub struct RedisConnection(Connection); + +impl RedisConnection { + + pub fn connect(RedisConfig { addr, password, is_tls }: RedisConfig) -> Result { + println!("{:?}", (addr.clone(), password.clone(), is_tls)); + let password = password.unwrap_or_default(); + let uri_scheme = match is_tls { + true => "rediss", + false => "redis", + }; + let redis_conn_url = format!("{uri_scheme}://:{password}@{addr}"); + let conn = redis::Client::open(redis_conn_url)?.get_connection()?; + Ok(Self(conn)) + } + + pub fn store_entry(&mut self, address: &Address, chain_id: &Chain, result: &SearchResponse) -> Result<()> { + let key = make_key(address, chain_id); + let val_str = serde_json::to_string(result)?; + self.0.set(&key, val_str)?; + Ok(()) + } + + pub fn get_entry(&mut self, address: &Address, chain_id: &Chain) -> Result> { + let key = make_key(address, chain_id); + let val_str: Option = self.0.get(&key)?; + let val = val_str.map(|val| serde_json::from_str(&val)).transpose()?; + Ok(val) + } + +} + +#[inline] +fn make_key(address: &Address, chain_id: &Chain) -> String { + format!("{:?}:{}", address, chain_id) +} diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index e065ce4..21968ec 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -39,7 +39,7 @@ impl From<&AppError> for Response { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct SearchResponse { token: Address, contract: Address, @@ -115,10 +115,19 @@ async fn _search_handler( let chain = chain_str.parse::()?; let token: Address = token_str.parse().map_err(|_| eyre::eyre!("Invalid token"))?; + if let Some(db_conn) = &app_state.db_connection { + let mut db_conn = db_conn.lock().unwrap(); + let entry = db_conn.get_entry(&token, &chain)?; + if let Some(entry) = entry { + return Ok(Json(Response::from(entry))); + } + } + let endpoint = &app_state.providers .get(&chain) .ok_or_else(|| eyre::eyre!(format!("Can't find provider for chain: {chain:?}")))? .endpoint; + // todo: someone could spam tokens that don't exist or take a long time to process (set a limit + store err in db) let response = erc20_topup::find_slot(&endpoint, token, None).await?; let response = SearchResponse { @@ -129,5 +138,10 @@ async fn _search_handler( lang: response.3, }; + if let Some(db_conn) = app_state.db_connection { + let mut db_conn = db_conn.lock().unwrap(); + db_conn.store_entry(&token, &chain, &response)?; // todo this err should not be propagated to the user + } + Ok(Json(response.into())) } \ No newline at end of file diff --git a/src/bin/serve/main.rs b/src/bin/serve/main.rs index 236c18c..9ba0fce 100644 --- a/src/bin/serve/main.rs +++ b/src/bin/serve/main.rs @@ -3,6 +3,7 @@ mod server; mod utils; mod state; mod config; +mod db; use std::sync::Arc; use tracing::info; @@ -12,15 +13,17 @@ use config::{Config, RpcUrl}; #[tokio::main] async fn main() -> Result<()> { - let config = Config::from_env()?; - if config.chain_configs.is_empty() { + let configs = Config::from_env()?; + + if configs.chain_configs.is_empty() { return Err(eyre::eyre!("No chain configs found")); } + if configs.logging_enabled { + tracing_subscriber::fmt::init(); + } - tracing_subscriber::fmt::init(); // todo: only if logging is on - - let mut app_state = state::AppProviders::new(); - for chain_config in config.chain_configs { + let mut app_providers = state::AppProviders::new(); + for chain_config in configs.chain_configs { let (endpoint, handler) = match chain_config.rpc_url { RpcUrl::Primary(url) => (url, None), RpcUrl::Fallack(url) => { @@ -29,10 +32,16 @@ async fn main() -> Result<()> { }, }; info!("Added provider for chain: {:?} with endpoint {endpoint:?}", chain_config.chain); - app_state.set_provider(chain_config.chain, endpoint, handler); + app_providers.set_provider(chain_config.chain, endpoint, handler); + } + + let mut app_state: state::AppState<_> = app_providers.into(); + if let Some(redis_config) = configs.redis_config { + let redis_connection = db::RedisConnection::connect(redis_config)?; + app_state.set_db_connection(redis_connection); } - server::run(&config.server_addr, app_state.into()).await?; + server::run(&configs.server_addr, app_state).await?; Ok(()) diff --git a/src/bin/serve/state.rs b/src/bin/serve/state.rs index 0ba1902..c7db6e3 100644 --- a/src/bin/serve/state.rs +++ b/src/bin/serve/state.rs @@ -2,14 +2,26 @@ use std::collections::HashMap; use std::str::FromStr; use std::hash::Hash; -use std::sync::Arc; - +use std::sync::{Arc, Mutex}; +use crate::db::RedisConnection; #[derive(Clone)] pub struct AppState where T: Sync + Send + Clone + 'static { pub providers: Arc>>, + pub db_connection: Option>>, +} + +impl AppState + where T: Sync + Send + Clone + 'static +{ + pub fn set_db_connection( + &mut self, + db_connection: RedisConnection + ) -> () { + self.db_connection = Some(Arc::new(Mutex::new(db_connection))); + } } pub struct AppProviders(HashMap>) @@ -41,7 +53,7 @@ impl Into> for AppProviders where T: Sync + Send + Clone + 'static { fn into(self) -> AppState { - AppState { providers: Arc::new(self.build()) } + AppState { providers: Arc::new(self.build()), db_connection: None } } } From 6cd61e586d6e8a3c5098615136a347c9b55b51c5 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 14:53:19 +0100 Subject: [PATCH 03/12] add timeout --- src/bin/serve/.env.sample | 6 +++++ src/bin/serve/config.rs | 6 +++++ src/bin/serve/db.rs | 11 ++++++-- src/bin/serve/handlers.rs | 12 ++++++--- src/bin/serve/main.rs | 11 ++++---- src/bin/serve/state.rs | 53 +++++++++++++++++++++++++++++++-------- 6 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/bin/serve/.env.sample b/src/bin/serve/.env.sample index d8da079..01398d0 100644 --- a/src/bin/serve/.env.sample +++ b/src/bin/serve/.env.sample @@ -1,7 +1,13 @@ +# Server configuration SERVER_PORT=8080 SERVER_HOST=localhost + +# Logging to stdout LOGGING_ENABLED=1 +# Timeout for the RPC requests in milliseconds +TIMEOUT_MS= + # RPCs with debug api and struct trace enabled (can be omitted) ETHEREUM_RPC= ARBITRUM_RPC= diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index bf8189c..559328d 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -2,6 +2,7 @@ use super::state::Chain; use eyre::Result; +pub const DEFAULT_TIMEOUT_MS: u64 = 5000; pub const DEFAULT_HOST: &str = "127.0.0.1"; const DEFAULT_PORT: u32 = 3000; @@ -10,6 +11,7 @@ pub struct Config { pub chain_configs: Vec, pub redis_config: Option, pub logging_enabled: bool, + pub timeout_ms: u64, } pub struct ChainConfig { @@ -90,12 +92,16 @@ impl Config { // Logging config let logging_enabled = std::env::var("LOGGING_ENABLED").ok() .map(|s| s == "1").unwrap_or(false); + let timeout_ms = std::env::var("TIMEOUT_MS").ok() + .and_then(|t_str| t_str.parse::().ok()) + .unwrap_or(DEFAULT_TIMEOUT_MS); Ok(Self { logging_enabled, chain_configs, redis_config, server_addr, + timeout_ms, }) } } \ No newline at end of file diff --git a/src/bin/serve/db.rs b/src/bin/serve/db.rs index 7b0ac60..657a905 100644 --- a/src/bin/serve/db.rs +++ b/src/bin/serve/db.rs @@ -10,8 +10,7 @@ pub struct RedisConnection(Connection); impl RedisConnection { - pub fn connect(RedisConfig { addr, password, is_tls }: RedisConfig) -> Result { - println!("{:?}", (addr.clone(), password.clone(), is_tls)); + fn connect(RedisConfig { addr, password, is_tls }: RedisConfig) -> Result { let password = password.unwrap_or_default(); let uri_scheme = match is_tls { true => "rediss", @@ -38,6 +37,14 @@ impl RedisConnection { } +impl TryFrom for RedisConnection { + type Error = eyre::Error; + + fn try_from(config: RedisConfig) -> Result { + Self::connect(config) + } +} + #[inline] fn make_key(address: &Address, chain_id: &Chain) -> String { format!("{:?}:{}", address, chain_id) diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index 21968ec..4b3f50b 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -1,6 +1,7 @@ use alloy::primitives::{Address, U256}; use serde::{Serialize, Deserialize}; use std::time::Instant; +use tokio::time::{timeout, Duration}; use axum::{ response::{Json, IntoResponse, Response as AxumResponse}, extract::{Path, State}, @@ -71,7 +72,7 @@ where } } -// todo: split expected and unexpected errors +// todo: split into user vs internal errors pub async fn search_handler( State(app_state): State>, Path((chain_str, token_str)): Path<(String, String)> @@ -89,7 +90,12 @@ pub async fn search_handler( "token": token_str, }), })); - let res = _search_handler(State(app_state), Path((chain_str, token_str))).await; + let tm_out = app_state.timeout_ms; + let fut = _search_handler(State(app_state), Path((chain_str, token_str))); + let res = match timeout(Duration::from_millis(tm_out), fut).await { + Ok(res) => res, + Err(_) => Err(AppError(eyre::eyre!("Timeout"))), + }; let res_str = match &res { Ok(Json(res)) => serde_json::to_string(res)?, @@ -127,7 +133,7 @@ async fn _search_handler( .get(&chain) .ok_or_else(|| eyre::eyre!(format!("Can't find provider for chain: {chain:?}")))? .endpoint; - // todo: someone could spam tokens that don't exist or take a long time to process (set a limit + store err in db) + // todo: store "slot not found response in db" let response = erc20_topup::find_slot(&endpoint, token, None).await?; let response = SearchResponse { diff --git a/src/bin/serve/main.rs b/src/bin/serve/main.rs index 9ba0fce..a9f2365 100644 --- a/src/bin/serve/main.rs +++ b/src/bin/serve/main.rs @@ -35,11 +35,12 @@ async fn main() -> Result<()> { app_providers.set_provider(chain_config.chain, endpoint, handler); } - let mut app_state: state::AppState<_> = app_providers.into(); - if let Some(redis_config) = configs.redis_config { - let redis_connection = db::RedisConnection::connect(redis_config)?; - app_state.set_db_connection(redis_connection); - } + let redis_conn = configs.redis_config.map(|c| c.try_into()).transpose()?; + let app_state = state::AppState::new( + app_providers.build(), + redis_conn, + configs.timeout_ms, + ); server::run(&configs.server_addr, app_state).await?; diff --git a/src/bin/serve/state.rs b/src/bin/serve/state.rs index c7db6e3..72cea21 100644 --- a/src/bin/serve/state.rs +++ b/src/bin/serve/state.rs @@ -1,9 +1,15 @@ -use std::collections::HashMap; -use std::str::FromStr; -use std::hash::Hash; -use std::sync::{Arc, Mutex}; -use crate::db::RedisConnection; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + str::FromStr, + hash::Hash, +}; +use crate::{ + config::DEFAULT_TIMEOUT_MS, + db::RedisConnection, +}; + #[derive(Clone)] pub struct AppState @@ -11,17 +17,38 @@ pub struct AppState { pub providers: Arc>>, pub db_connection: Option>>, + pub timeout_ms: u64, } impl AppState where T: Sync + Send + Clone + 'static { - pub fn set_db_connection( - &mut self, - db_connection: RedisConnection - ) -> () { - self.db_connection = Some(Arc::new(Mutex::new(db_connection))); + + pub fn new( + providers: HashMap>, + db_connection: Option, + timeout_ms: u64, + ) -> Self { + Self { + providers: Arc::new(providers), + db_connection: db_connection.map(|conn| Arc::new(Mutex::new(conn))), + timeout_ms, + } } + + // pub fn set_db_connection( + // &mut self, + // db_connection: RedisConnection + // ) -> () { + // self.db_connection = Some(Arc::new(Mutex::new(db_connection))); + // } + + // pub fn set_timeout_ms( + // &mut self, + // timeout_ms: u64 + // ) -> () { + // self.timeout_ms = timeout_ms; + // } } pub struct AppProviders(HashMap>) @@ -53,7 +80,11 @@ impl Into> for AppProviders where T: Sync + Send + Clone + 'static { fn into(self) -> AppState { - AppState { providers: Arc::new(self.build()), db_connection: None } + AppState { + providers: Arc::new(self.build()), + timeout_ms: DEFAULT_TIMEOUT_MS, + db_connection: None + } } } From 53be5123590e9e87741981d2d1b35ae11baaece1 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 15:17:35 +0100 Subject: [PATCH 04/12] add anvil config --- src/bin/serve/.env.sample | 7 ++++++- src/bin/serve/config.rs | 20 +++++++++++++++++++- src/bin/serve/db.rs | 2 ++ src/bin/serve/main.rs | 2 +- src/bin/serve/utils.rs | 28 +++++++++++++++++++++++++--- 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/bin/serve/.env.sample b/src/bin/serve/.env.sample index 01398d0..168a528 100644 --- a/src/bin/serve/.env.sample +++ b/src/bin/serve/.env.sample @@ -26,4 +26,9 @@ REDIS_ENABLED=1 REDIS_HOST=localhost REDIS_PORT=9090 REDIS_PASSWORD= -REDIS_IS_TLS=1 \ No newline at end of file +REDIS_IS_TLS=1 + +# Anvil +ANVIL_TIMEOUT_MS= +ANVIL_MEMORY_LIMIT= +ANVIL_CPU_PER_SEC= diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index 559328d..eff7c02 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -12,6 +12,7 @@ pub struct Config { pub redis_config: Option, pub logging_enabled: bool, pub timeout_ms: u64, + pub anvil_config: AnvilConfig, } pub struct ChainConfig { @@ -31,6 +32,13 @@ pub struct RedisConfig { pub is_tls: bool, } +#[derive(Default)] +pub struct AnvilConfig { + pub cpu_per_sec: Option, + pub memory_limit: Option, + pub timeout: Option, +} + impl Config { pub fn from_env() -> Result { @@ -88,8 +96,17 @@ impl Config { } else { None }; + + // Anvil config + let mut anvil_config = AnvilConfig::default(); + anvil_config.cpu_per_sec = std::env::var("ANVIL_CPU_PER_SEC").ok() + .and_then(|s| s.parse::().ok()); + anvil_config.memory_limit = std::env::var("ANVIL_MEMORY_LIMIT").ok() + .and_then(|s| s.parse::().ok()); + anvil_config.timeout = std::env::var("ANVIL_TIMEOUT_MS").ok() + .and_then(|s| s.parse::().ok()); + - // Logging config let logging_enabled = std::env::var("LOGGING_ENABLED").ok() .map(|s| s == "1").unwrap_or(false); let timeout_ms = std::env::var("TIMEOUT_MS").ok() @@ -99,6 +116,7 @@ impl Config { Ok(Self { logging_enabled, chain_configs, + anvil_config, redis_config, server_addr, timeout_ms, diff --git a/src/bin/serve/db.rs b/src/bin/serve/db.rs index 657a905..fbaf435 100644 --- a/src/bin/serve/db.rs +++ b/src/bin/serve/db.rs @@ -4,6 +4,7 @@ use eyre::Result; use crate::handlers::SearchResponse; use crate::config::RedisConfig; use crate::state::Chain; +use tracing::info; pub struct RedisConnection(Connection); @@ -18,6 +19,7 @@ impl RedisConnection { }; let redis_conn_url = format!("{uri_scheme}://:{password}@{addr}"); let conn = redis::Client::open(redis_conn_url)?.get_connection()?; + info!("Connected to Redis at: {}", addr); Ok(Self(conn)) } diff --git a/src/bin/serve/main.rs b/src/bin/serve/main.rs index a9f2365..2cf9d7b 100644 --- a/src/bin/serve/main.rs +++ b/src/bin/serve/main.rs @@ -27,7 +27,7 @@ async fn main() -> Result<()> { let (endpoint, handler) = match chain_config.rpc_url { RpcUrl::Primary(url) => (url, None), RpcUrl::Fallack(url) => { - let anvil = utils::spawn_anvil(Some(&url)); + let anvil = utils::spawn_anvil(Some(&url), Some(&configs.anvil_config)); (anvil.endpoint(), Some(Arc::new(anvil))) }, }; diff --git a/src/bin/serve/utils.rs b/src/bin/serve/utils.rs index e117b03..b93bf7c 100644 --- a/src/bin/serve/utils.rs +++ b/src/bin/serve/utils.rs @@ -1,9 +1,31 @@ use alloy::node_bindings::{Anvil, AnvilInstance}; +use crate::config::AnvilConfig; -pub fn spawn_anvil(fork_url: Option<&str>) -> AnvilInstance { - (match fork_url { +pub fn spawn_anvil(fork_url: Option<&str>, config: Option<&AnvilConfig>) -> AnvilInstance { + let mut anvil = match fork_url { Some(url) => Anvil::new().fork(url), None => Anvil::new(), - }).spawn() + }; + if let Some(config) = config { + if let Some(cpu_per_sec) = config.cpu_per_sec { + anvil = anvil.args(vec![ + "--compute-units-per-second", + &cpu_per_sec.to_string(), + ]); + } + if let Some(memory_limit) = config.memory_limit { + anvil = anvil.args(vec![ + "--memory-limit", + &memory_limit.to_string(), + ]); + } + if let Some(timeout) = config.timeout { + anvil = anvil.args(vec![ + "--timeout", + &timeout.to_string(), + ]); + } + } + anvil.spawn() } From 8d28ebdb681481098aeee47d90c54da95d0c7f0c Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 15:19:58 +0100 Subject: [PATCH 05/12] fallback -> fork --- src/bin/serve/config.rs | 4 ++-- src/bin/serve/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index eff7c02..6c9a577 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -22,7 +22,7 @@ pub struct ChainConfig { pub enum RpcUrl { Primary(String), - Fallack(String), + Fork(String), } #[derive(Default)] @@ -71,7 +71,7 @@ impl Config { let fallback_key = format!("{}_FORK_RPC", chain.to_string().to_uppercase()); std::env::var(fallback_key).ok() .and_then(|s| (!s.trim().is_empty()).then(|| s)) - .map(RpcUrl::Fallack) + .map(RpcUrl::Fork) }) .map(|url| ChainConfig { chain, rpc_url: url }) }).collect::>(); diff --git a/src/bin/serve/main.rs b/src/bin/serve/main.rs index 2cf9d7b..12a18aa 100644 --- a/src/bin/serve/main.rs +++ b/src/bin/serve/main.rs @@ -26,7 +26,7 @@ async fn main() -> Result<()> { for chain_config in configs.chain_configs { let (endpoint, handler) = match chain_config.rpc_url { RpcUrl::Primary(url) => (url, None), - RpcUrl::Fallack(url) => { + RpcUrl::Fork(url) => { let anvil = utils::spawn_anvil(Some(&url), Some(&configs.anvil_config)); (anvil.endpoint(), Some(Arc::new(anvil))) }, From 1e546fd684265378bfe02fe916a954209ee72fa1 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 15:59:33 +0100 Subject: [PATCH 06/12] user vs internal err --- src/bin/serve/handlers.rs | 97 +++++++++++++++++++++++++++++++-------- src/bin/serve/state.rs | 14 ------ 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index 4b3f50b..d35a42e 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -7,7 +7,7 @@ use axum::{ extract::{Path, State}, http::StatusCode, }; -use tracing::info; +use tracing::{info, error}; use super::state::{Chain, AppState}; @@ -30,16 +30,36 @@ impl From for Response { } } -impl From<&AppError> for Response { - fn from(err: &AppError) -> Self { +impl From<&UserError> for Response { + fn from(err: &UserError) -> Self { Self { success: false, msg: None, - error: Some(err.0.to_string()), + error: Some(format!("{err:?}")), } } } +impl From<&eyre::Report> for Response { + fn from(err: &eyre::Report) -> Self { + Self { + success: false, + msg: None, + error: Some(format!("{err:#}")), + } + } +} + +impl From<&AppError> for Response { + fn from(err: &AppError) -> Self { + match err { + AppError::UserError(err) => Self::from(err), + AppError::InternalError(err) => Self::from(err), + } + } + +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SearchResponse { token: Address, @@ -51,15 +71,48 @@ pub struct SearchResponse { } #[derive(Debug)] -pub struct AppError(eyre::Error); +pub enum AppError { + UserError(UserError), + InternalError(eyre::Error), +} + +#[derive(Debug)] +pub enum UserError { + InvalidToken, + ChainNotFound, + ProviderNotFound, + Timeout, + SlotNotFound, +} impl IntoResponse for AppError { fn into_response(self) -> AxumResponse { - ( - StatusCode::INTERNAL_SERVER_ERROR, - serde_json::to_string(&Response::from(&self)).unwrap(), - ) - .into_response() + match self { + AppError::UserError(err) => match err { + UserError::InvalidToken | UserError::ChainNotFound | UserError::ProviderNotFound => ( + StatusCode::BAD_REQUEST, + serde_json::to_string(&Response::from(&err)).unwrap(), + ) + .into_response(), + UserError::Timeout => ( + StatusCode::GATEWAY_TIMEOUT, + serde_json::to_string(&Response::from(&err)).unwrap(), + ) + .into_response(), + UserError::SlotNotFound => ( + StatusCode::NOT_FOUND, + serde_json::to_string(&Response::from(&err)).unwrap(), + ) + .into_response(), + }, + AppError::InternalError(err) => { + error!("{:#}", err); + ( + StatusCode::INTERNAL_SERVER_ERROR, + serde_json::to_string(&Response::from(&eyre::eyre!("InternalError"))).unwrap(), + ).into_response() + } + } } } @@ -68,11 +121,10 @@ where E: Into, { fn from(err: E) -> Self { - Self(err.into()) + Self::InternalError(err.into()) } } -// todo: split into user vs internal errors pub async fn search_handler( State(app_state): State>, Path((chain_str, token_str)): Path<(String, String)> @@ -94,7 +146,7 @@ pub async fn search_handler( let fut = _search_handler(State(app_state), Path((chain_str, token_str))); let res = match timeout(Duration::from_millis(tm_out), fut).await { Ok(res) => res, - Err(_) => Err(AppError(eyre::eyre!("Timeout"))), + Err(_) => Err(AppError::UserError(UserError::Timeout)), }; let res_str = match &res { @@ -118,8 +170,10 @@ async fn _search_handler( ) -> Result, AppError> where T: Sync + Send + Clone + 'static { - let chain = chain_str.parse::()?; - let token: Address = token_str.parse().map_err(|_| eyre::eyre!("Invalid token"))?; + let chain = chain_str.parse::() + .map_err(|_| AppError::UserError(UserError::ChainNotFound))?; + let token: Address = token_str.parse() + .map_err(|_| AppError::UserError(UserError::InvalidToken))?; if let Some(db_conn) = &app_state.db_connection { let mut db_conn = db_conn.lock().unwrap(); @@ -131,10 +185,17 @@ async fn _search_handler( let endpoint = &app_state.providers .get(&chain) - .ok_or_else(|| eyre::eyre!(format!("Can't find provider for chain: {chain:?}")))? + .ok_or(AppError::UserError(UserError::ProviderNotFound))? .endpoint; // todo: store "slot not found response in db" - let response = erc20_topup::find_slot(&endpoint, token, None).await?; + let response = erc20_topup::find_slot(&endpoint, token, None).await + .map_err(|err| { + if err.to_string().contains("No valid slots found") { + AppError::UserError(UserError::SlotNotFound) + } else { + AppError::InternalError(err) + } + })?; let response = SearchResponse { token: token, @@ -146,7 +207,7 @@ async fn _search_handler( if let Some(db_conn) = app_state.db_connection { let mut db_conn = db_conn.lock().unwrap(); - db_conn.store_entry(&token, &chain, &response)?; // todo this err should not be propagated to the user + db_conn.store_entry(&token, &chain, &response)?; } Ok(Json(response.into())) diff --git a/src/bin/serve/state.rs b/src/bin/serve/state.rs index 72cea21..fc7746a 100644 --- a/src/bin/serve/state.rs +++ b/src/bin/serve/state.rs @@ -35,20 +35,6 @@ impl AppState timeout_ms, } } - - // pub fn set_db_connection( - // &mut self, - // db_connection: RedisConnection - // ) -> () { - // self.db_connection = Some(Arc::new(Mutex::new(db_connection))); - // } - - // pub fn set_timeout_ms( - // &mut self, - // timeout_ms: u64 - // ) -> () { - // self.timeout_ms = timeout_ms; - // } } pub struct AppProviders(HashMap>) From e821ebf3e632ba58684a70a18a6159dd7bf3f78b Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 16:23:23 +0100 Subject: [PATCH 07/12] store 'slot not found' instances --- src/bin/serve/db.rs | 6 +++--- src/bin/serve/handlers.rs | 30 +++++++++++++++++++++++++----- src/bin/serve/utils.rs | 1 + 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/bin/serve/db.rs b/src/bin/serve/db.rs index fbaf435..16b3216 100644 --- a/src/bin/serve/db.rs +++ b/src/bin/serve/db.rs @@ -1,7 +1,7 @@ use redis::{Commands, Connection}; use alloy::primitives::Address; use eyre::Result; -use crate::handlers::SearchResponse; +use crate::handlers::SearchResponseWrapper; use crate::config::RedisConfig; use crate::state::Chain; use tracing::info; @@ -23,14 +23,14 @@ impl RedisConnection { Ok(Self(conn)) } - pub fn store_entry(&mut self, address: &Address, chain_id: &Chain, result: &SearchResponse) -> Result<()> { + pub fn store_search_response(&mut self, address: &Address, chain_id: &Chain, result: &SearchResponseWrapper) -> Result<()> { let key = make_key(address, chain_id); let val_str = serde_json::to_string(result)?; self.0.set(&key, val_str)?; Ok(()) } - pub fn get_entry(&mut self, address: &Address, chain_id: &Chain) -> Result> { + pub fn get_search_response(&mut self, address: &Address, chain_id: &Chain) -> Result> { let key = make_key(address, chain_id); let val_str: Option = self.0.get(&key)?; let val = val_str.map(|val| serde_json::from_str(&val)).transpose()?; diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index d35a42e..ca83f70 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -60,6 +60,18 @@ impl From<&AppError> for Response { } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum SearchResponseWrapper { + Found(SearchResponse), + NotFound, +} + +impl From for SearchResponseWrapper { + fn from(msg: SearchResponse) -> Self { + Self::Found(msg) + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SearchResponse { token: Address, @@ -177,9 +189,12 @@ async fn _search_handler( if let Some(db_conn) = &app_state.db_connection { let mut db_conn = db_conn.lock().unwrap(); - let entry = db_conn.get_entry(&token, &chain)?; - if let Some(entry) = entry { - return Ok(Json(Response::from(entry))); + let response = db_conn.get_search_response(&token, &chain)?; + if let Some(response) = response { + return match response { + SearchResponseWrapper::NotFound => Err(AppError::UserError(UserError::SlotNotFound)), + SearchResponseWrapper::Found(entry) => Ok(Json(Response::from(entry))), + }; } } @@ -187,10 +202,14 @@ async fn _search_handler( .get(&chain) .ok_or(AppError::UserError(UserError::ProviderNotFound))? .endpoint; - // todo: store "slot not found response in db" let response = erc20_topup::find_slot(&endpoint, token, None).await .map_err(|err| { if err.to_string().contains("No valid slots found") { + if let Some(db_conn) = &app_state.db_connection { + let mut db_conn = db_conn.lock().unwrap(); + let response = SearchResponseWrapper::NotFound; + db_conn.store_search_response(&token, &chain, &response).unwrap(); // todo dont unwrap! + } AppError::UserError(UserError::SlotNotFound) } else { AppError::InternalError(err) @@ -207,7 +226,8 @@ async fn _search_handler( if let Some(db_conn) = app_state.db_connection { let mut db_conn = db_conn.lock().unwrap(); - db_conn.store_entry(&token, &chain, &response)?; + let response = response.clone().into(); + db_conn.store_search_response(&token, &chain, &response)?; } Ok(Json(response.into())) diff --git a/src/bin/serve/utils.rs b/src/bin/serve/utils.rs index b93bf7c..7e114e8 100644 --- a/src/bin/serve/utils.rs +++ b/src/bin/serve/utils.rs @@ -27,5 +27,6 @@ pub fn spawn_anvil(fork_url: Option<&str>, config: Option<&AnvilConfig>) -> Anvi ]); } } + anvil = anvil.arg("--no-storage-caching"); anvil.spawn() } From 1ae5ccac38a3cd3d816cf511607ec1f2aaf88dfc Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 16:33:42 +0100 Subject: [PATCH 08/12] log info source --- src/bin/serve/handlers.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index ca83f70..e4fed4f 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -137,6 +137,12 @@ where } } +#[derive(Debug, Serialize)] +enum InfoSource { + Provider, + Database, +} + pub async fn search_handler( State(app_state): State>, Path((chain_str, token_str)): Path<(String, String)> @@ -161,25 +167,27 @@ pub async fn search_handler( Err(_) => Err(AppError::UserError(UserError::Timeout)), }; - let res_str = match &res { - Ok(Json(res)) => serde_json::to_string(res)?, - Err(ref err) => serde_json::to_string(&Response::from(err))?, + let (res_str, source) = match &res { + Ok((Json(res), source)) => (serde_json::to_string(res)?, Some(source)), + Err(ref err) => (serde_json::to_string(&Response::from(err))?, None), }; let duration_ms = rtime0.elapsed().as_millis(); + let source = source.map(|s| format!("{s:?}")).unwrap_or("null".to_string()); info!("{}", serde_json::json!({ "msg": "new_response", "handler": "search_handler", "id": request_id, "response": res_str, "duration": duration_ms, + "source": source, })); - res + res.map(|(res, _)| res) } async fn _search_handler( State(app_state): State>, Path((chain_str, token_str)): Path<(String, String)> -) -> Result, AppError> +) -> Result<(Json, InfoSource), AppError> where T: Sync + Send + Clone + 'static { let chain = chain_str.parse::() @@ -193,7 +201,7 @@ async fn _search_handler( if let Some(response) = response { return match response { SearchResponseWrapper::NotFound => Err(AppError::UserError(UserError::SlotNotFound)), - SearchResponseWrapper::Found(entry) => Ok(Json(Response::from(entry))), + SearchResponseWrapper::Found(entry) => Ok((Json(Response::from(entry)), InfoSource::Database)), }; } } @@ -230,5 +238,5 @@ async fn _search_handler( db_conn.store_search_response(&token, &chain, &response)?; } - Ok(Json(response.into())) + Ok((Json(response.into()), InfoSource::Provider)) } \ No newline at end of file From 7aa296c184f8eba1957bd20390882cd614d90317 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 18:22:32 +0100 Subject: [PATCH 09/12] make clippy happy --- src/bin/serve/config.rs | 25 ++++++++++++------------- src/bin/serve/handlers.rs | 4 ++-- src/bin/serve/state.rs | 8 ++++---- src/lib.rs | 2 ++ src/slot_finder/lang.rs | 22 ++++++++++++++-------- src/slot_finder/mod.rs | 1 + src/slot_finder/ops/trace.rs | 2 +- src/slot_finder/trace_parser.rs | 6 +++++- src/slot_finder/utils.rs | 14 ++++++-------- 9 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/bin/serve/config.rs b/src/bin/serve/config.rs index 6c9a577..0029d46 100644 --- a/src/bin/serve/config.rs +++ b/src/bin/serve/config.rs @@ -48,7 +48,7 @@ impl Config { // Server config let host = std::env::var("SERVER_HOST").ok() - .and_then(|s| (!s.trim().is_empty()).then(|| s)) + .and_then(|s| (!s.trim().is_empty()).then_some(s)) .unwrap_or(DEFAULT_HOST.to_string()); let port = std::env::var("SERVER_PORT").ok() .and_then(|p_str| p_str.parse::().ok()) @@ -65,12 +65,12 @@ impl Config { let chain_configs = available_chains.into_iter().filter_map(|chain| { let primary_key = format!("{}_RPC", chain.to_string().to_uppercase()); std::env::var(primary_key).ok() - .and_then(|s| (!s.trim().is_empty()).then(|| s)) + .and_then(|s| (!s.trim().is_empty()).then_some(s)) .map(RpcUrl::Primary) .or_else(|| { let fallback_key = format!("{}_FORK_RPC", chain.to_string().to_uppercase()); std::env::var(fallback_key).ok() - .and_then(|s| (!s.trim().is_empty()).then(|| s)) + .and_then(|s| (!s.trim().is_empty()).then_some(s)) .map(RpcUrl::Fork) }) .map(|url| ChainConfig { chain, rpc_url: url }) @@ -83,7 +83,7 @@ impl Config { if redis_enabled { let mut redis_config = RedisConfig::default(); let redis_host = std::env::var("REDIS_HOST").ok() - .and_then(|s| (!s.trim().is_empty()).then(|| s)) + .and_then(|s| (!s.trim().is_empty()).then_some(s)) .unwrap_or("localhost".to_string()); let redis_port = std::env::var("REDIS_PORT").ok() .and_then(|p_str| p_str.parse::().ok()) @@ -97,15 +97,14 @@ impl Config { None }; - // Anvil config - let mut anvil_config = AnvilConfig::default(); - anvil_config.cpu_per_sec = std::env::var("ANVIL_CPU_PER_SEC").ok() - .and_then(|s| s.parse::().ok()); - anvil_config.memory_limit = std::env::var("ANVIL_MEMORY_LIMIT").ok() - .and_then(|s| s.parse::().ok()); - anvil_config.timeout = std::env::var("ANVIL_TIMEOUT_MS").ok() - .and_then(|s| s.parse::().ok()); - + let anvil_config = AnvilConfig { + cpu_per_sec: std::env::var("ANVIL_CPU_PER_SEC").ok() + .and_then(|s| s.parse::().ok()), + memory_limit: std::env::var("ANVIL_MEMORY_LIMIT").ok() + .and_then(|s| s.parse::().ok()), + timeout: std::env::var("ANVIL_TIMEOUT_MS").ok() + .and_then(|s| s.parse::().ok()), + }; let logging_enabled = std::env::var("LOGGING_ENABLED").ok() .map(|s| s == "1").unwrap_or(false); diff --git a/src/bin/serve/handlers.rs b/src/bin/serve/handlers.rs index e4fed4f..e3195c6 100644 --- a/src/bin/serve/handlers.rs +++ b/src/bin/serve/handlers.rs @@ -210,7 +210,7 @@ async fn _search_handler( .get(&chain) .ok_or(AppError::UserError(UserError::ProviderNotFound))? .endpoint; - let response = erc20_topup::find_slot(&endpoint, token, None).await + let response = erc20_topup::find_slot(endpoint, token, None).await .map_err(|err| { if err.to_string().contains("No valid slots found") { if let Some(db_conn) = &app_state.db_connection { @@ -225,7 +225,7 @@ async fn _search_handler( })?; let response = SearchResponse { - token: token, + token, contract: response.0, slot: response.1.into(), update_ratio: response.2, diff --git a/src/bin/serve/state.rs b/src/bin/serve/state.rs index fc7746a..48b1555 100644 --- a/src/bin/serve/state.rs +++ b/src/bin/serve/state.rs @@ -62,12 +62,12 @@ impl AppProviders } } -impl Into> for AppProviders +impl From> for AppState where T: Sync + Send + Clone + 'static { - fn into(self) -> AppState { - AppState { - providers: Arc::new(self.build()), + fn from(providers: AppProviders) -> Self { + Self { + providers: Arc::new(providers.build()), timeout_ms: DEFAULT_TIMEOUT_MS, db_connection: None } diff --git a/src/lib.rs b/src/lib.rs index 1cdf557..258c641 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![allow(clippy::module_inception)] + mod common; mod slot_finder; diff --git a/src/slot_finder/lang.rs b/src/slot_finder/lang.rs index 8bfdcf3..154a669 100644 --- a/src/slot_finder/lang.rs +++ b/src/slot_finder/lang.rs @@ -26,23 +26,29 @@ impl EvmLanguage { } fn mapping_loc_from_tokens(token_0: &FixedBytes<32>, token_1: &FixedBytes<32>) -> B256 { - let hashable = vec![token_0.0.to_vec(), token_1.0.to_vec()].concat(); - alloy_utils::keccak256(&hashable).into() + let hashable = [token_0.0.to_vec(), token_1.0.to_vec()].concat(); + alloy_utils::keccak256(hashable) } - pub fn to_string(&self) -> String { - match &self { - EvmLanguage::Solidity => String::from("solidity"), - EvmLanguage::Vyper => String::from("vyper"), +} + +impl std::fmt::Display for EvmLanguage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EvmLanguage::Solidity => write!(f, "solidity"), + EvmLanguage::Vyper => write!(f, "vyper"), } } +} - pub fn from_str(s: &str) -> Result { +impl std::str::FromStr for EvmLanguage { + type Err = eyre::Error; + + fn from_str(s: &str) -> Result { match s.to_lowercase().as_str() { "solidity" => Ok(EvmLanguage::Solidity), "vyper" => Ok(EvmLanguage::Vyper), _ => Err(eyre::eyre!("Invalid language")), } } - } \ No newline at end of file diff --git a/src/slot_finder/mod.rs b/src/slot_finder/mod.rs index 44a4739..d11bc3c 100644 --- a/src/slot_finder/mod.rs +++ b/src/slot_finder/mod.rs @@ -1,3 +1,4 @@ +#[warn(clippy::module_inception)] mod trace_parser; mod slot_finder; mod lang; diff --git a/src/slot_finder/ops/trace.rs b/src/slot_finder/ops/trace.rs index 62df0f4..380ae85 100644 --- a/src/slot_finder/ops/trace.rs +++ b/src/slot_finder/ops/trace.rs @@ -36,7 +36,7 @@ pub async fn default_trace_call( let block = block.unwrap_or(BlockNumberOrTag::Latest); let response = provider.debug_trace_call( call_request, - block.into(), + block, tracing_options, ).await?; diff --git a/src/slot_finder/trace_parser.rs b/src/slot_finder/trace_parser.rs index 831f873..67018b3 100644 --- a/src/slot_finder/trace_parser.rs +++ b/src/slot_finder/trace_parser.rs @@ -16,7 +16,7 @@ impl TraceParser { pub fn parse(struct_logs: Vec, token: Address, holder: Address) -> Result> { let mut parser = TraceParser::default(); - parser.holder = holder; + parser.set_holder(holder); parser.depth_to_address.insert(1, token); parser.parse_logs(struct_logs)?; Ok(parser.results.into_iter().collect()) @@ -92,4 +92,8 @@ impl TraceParser { Ok(()) } + fn set_holder(&mut self, holder: Address) { + self.holder = holder; + } + } diff --git a/src/slot_finder/utils.rs b/src/slot_finder/utils.rs index 874ea52..179fe03 100644 --- a/src/slot_finder/utils.rs +++ b/src/slot_finder/utils.rs @@ -9,18 +9,16 @@ pub fn ratio_f64(val1: U256, val2: U256, precision_mul: Option) -> f64 { } let p_mul = precision_mul.unwrap_or(DEFAULT_PRECISION_MUL); let ur_bn = U512::from(val1) * U512::from(p_mul) / U512::from(val2); - let update_ratio = - if ur_bn <= U512::from(U128::MAX) { - ur_bn.to::() as f64 / p_mul as f64 - } else { - f64::INFINITY - }; - update_ratio + if ur_bn <= U512::from(U128::MAX) { + ur_bn.to::() as f64 / p_mul as f64 + } else { + f64::INFINITY + } } pub fn bytes_to_u256(val: Bytes) -> U256 { let bytes = val.to_vec(); - if bytes.len() == 0 { + if bytes.is_empty() { U256::ZERO } else { B256::from_slice(&bytes[..32]).into() From 0ea71f59c656e4c96521f11548bed2d81d1e15df Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 18:33:27 +0100 Subject: [PATCH 10/12] add ci --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..dad03cd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: Cargo Build & Test + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + ETH_RPC_URL: https://eth.drpc.org + +jobs: + build_and_test: + name: Rust project - latest + runs-on: ubuntu-latest + strategy: + matrix: + toolchain: + - stable + - beta + - nightly + steps: + - uses: actions/checkout@v3 + - name: Update + run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose + - name: Run Clippy + run: cargo clippy \ No newline at end of file From fde30aaf4df18fb726f267ade38da952c6d68768 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Thu, 2 May 2024 18:33:57 +0100 Subject: [PATCH 11/12] update env var rpc --- examples/eth_token_support.rs | 2 +- src/slot_finder/slot_finder.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/eth_token_support.rs b/examples/eth_token_support.rs index c5651be..1441b75 100644 --- a/examples/eth_token_support.rs +++ b/examples/eth_token_support.rs @@ -18,7 +18,7 @@ struct TokenInfo { #[tokio::main] async fn main() -> Result<()> { let ethereum_tokens = coingecko_all_tokens("ethereum".to_string()).await?; - let rpc_endpoint = env_var("RPC_URL")?; + let rpc_endpoint = env_var("ETH_RPC_URL")?; let anvil = spawn_anvil(Some(&rpc_endpoint)); for (symbol, token) in ethereum_tokens { diff --git a/src/slot_finder/slot_finder.rs b/src/slot_finder/slot_finder.rs index 3ed8e2d..5f437e2 100644 --- a/src/slot_finder/slot_finder.rs +++ b/src/slot_finder/slot_finder.rs @@ -111,7 +111,7 @@ mod tests { } fn rpc_endpoint() -> Result { - env_var("RPC_URL") // todo: rename this to test-rpc or emphasize that it must be eth based + env_var("ETH_RPC_URL") } #[tokio::test] From 564481cc6adcdb5990dcc5146c1f154409550495 Mon Sep 17 00:00:00 2001 From: halo3mic <46010359+halo3mic@users.noreply.github.com> Date: Fri, 3 May 2024 13:10:29 +0100 Subject: [PATCH 12/12] add foundry to ci --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dad03cd..7fe87f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,15 +16,19 @@ jobs: matrix: toolchain: - stable - - beta - - nightly steps: - uses: actions/checkout@v3 + - name: foundryup + run: | + curl -L https://foundry.paradigm.xyz | bash + /home/runner/.config/.foundry/bin/foundryup - name: Update run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} - name: Build run: cargo build --verbose - name: Run tests - run: cargo test --verbose + run: | + export PATH="$PATH:/home/runner/.config/.foundry/bin" + cargo test --verbose -- --nocapture - name: Run Clippy run: cargo clippy \ No newline at end of file