diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index 004a3c71..3a032edb 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -106,8 +106,11 @@ jobs: MAIN_VAULT_ADDRESS: ${{ secrets.MAIN_VAULT_ADDRESS }} RUST_BACKTRACE: 1 + - name: RGB Test Init + run: cargo test --locked --features server --test _init -- _init --nocapture --test-threads 1 + - name: RGB Tests - run: cargo test --locked --features server --test rgb -- rgb --nocapture --test-threads 1 + run: cargo test --locked --features server --test rgb -- rgb --nocapture --test-threads 1 env: TEST_WALLET_SEED: ${{ secrets.TEST_WALLET_SEED }} RUST_BACKTRACE: 1 diff --git a/Cargo.lock b/Cargo.lock index 4da41365..f9597f0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,7 +111,7 @@ dependencies = [ "serde_json", "serde_yaml", "stringly_conversions", - "toml 0.8.8", + "toml", "wasm-bindgen", ] @@ -719,11 +719,12 @@ dependencies = [ "serde-encrypt", "serde-wasm-bindgen", "serde_json", + "sled", "strict_encoding", "strict_types", "thiserror", "tokio", - "toml 0.7.8", + "toml", "tower-http", "walkdir", "wasm-bindgen", @@ -1114,6 +1115,27 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" +[[package]] +name = "crossbeam-epoch" +version = "0.9.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -1301,7 +1323,7 @@ dependencies = [ "hkdf", "libsecp256k1", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "rand_core 0.6.4", "sha2 0.10.8", "typenum", @@ -1437,6 +1459,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.29" @@ -2183,6 +2215,15 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2418,6 +2459,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2425,7 +2477,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -2436,7 +2502,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -2684,6 +2750,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3380,6 +3455,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "slip132" version = "0.10.1" @@ -3515,7 +3606,7 @@ dependencies = [ "serde_yaml", "sha2 0.10.8", "strict_encoding", - "toml 0.8.8", + "toml", ] [[package]] @@ -3597,7 +3688,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.4.1", "rustix", "windows-sys 0.48.0", ] @@ -3686,7 +3777,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.5.5", @@ -3766,29 +3857,17 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", -] - [[package]] name = "toml" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ + "indexmap 2.1.0", "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit", ] [[package]] @@ -3800,19 +3879,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "toml_edit" version = "0.21.0" diff --git a/Cargo.toml b/Cargo.toml index 6d0905c7..e532d38a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,7 @@ esplora_block = { version = "0.5.0", package = "esplora-client", default-feature "blocking", ] } inflate = "0.4.5" +sled = "0.34.7" tower-http = { version = "0.4.4", features = ["cors"], optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] @@ -134,7 +135,8 @@ anyhow = "1.0.71" blake3 = "1.4.1" rgb-std = { version = "0.10.2" } serde = "1.0.189" -toml = { version = "0.7.8", features = ["preserve_order"] } +serde_json = "1.0.107" +toml = { version = "0.8.0", features = ["preserve_order"] } [patch.crates-io] # Remove after merge and release https://github.com/BP-WG/bitcoin_foundation/pull/20 diff --git a/build.rs b/build.rs index 5b5cfa4e..5b61dad4 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fs}; +use std::{collections::BTreeMap, env, fs, path}; use anyhow::Result; use rgbstd::{ @@ -54,6 +54,30 @@ const MARKETPLACE_OFFERS: &str = "bitmask-marketplace_public_offers.c15"; const MARKETPLACE_BIDS: &str = "bitmask-marketplace_public_bids.c15"; const NETWORK: &str = "bitcoin"; // Only mainnet is tracked, no monetary incentive to upgrade testnet assets +#[derive(Serialize, Deserialize, Default)] +pub struct MetricsData { + bytes: u64, + bytes_by_day: BTreeMap, + bitcoin_wallets_by_day: BTreeMap, + signet_wallets_by_day: BTreeMap, + testnet_wallets_by_day: BTreeMap, + regtest_wallets_by_day: BTreeMap, + wallets_by_network: BTreeMap, +} + +pub fn init_fs() -> Result<()> { + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let dir = path::Path::new(&dir); + + fs::create_dir_all(dir)?; + fs::write( + dir.join("metrics.json"), + serde_json::to_string_pretty(&MetricsData::default())?, + )?; + + Ok(()) +} + fn main() -> Result<()> { // lib ids const BMC_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -158,5 +182,7 @@ fn main() -> Result<()> { fs::write(FILE_HASHES_FILE, toml)?; + init_fs()?; + Ok(()) } diff --git a/src/bin/bitmaskd.rs b/src/bin/bitmaskd.rs index 658ce315..66308004 100644 --- a/src/bin/bitmaskd.rs +++ b/src/bin/bitmaskd.rs @@ -724,7 +724,8 @@ async fn send_coins( } async fn json_metrics() -> Result { - let metrics_json = metrics::json().await?; + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let metrics_json = fs::read_to_string(&format!("{dir}/metrics.json")).await?; Ok(( StatusCode::OK, @@ -734,14 +735,16 @@ async fn json_metrics() -> Result { } async fn csv_metrics() -> Result { - let metrics_csv = metrics::csv().await; + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let metrics_csv = fs::read_to_string(&format!("{dir}/metrics.csv")).await?; Ok((StatusCode::OK, [("content-type", "text/csv")], metrics_csv)) } async fn init_metrics() -> Result<()> { - let path = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); - let dir = path::Path::new(&path); + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let dir = path::Path::new(&dir); + fs::create_dir_all(dir).await?; info!("Starting metrics collection..."); let duration = Instant::now(); @@ -819,14 +822,14 @@ async fn main() -> Result<()> { app = app .route("/regtest/block", get(new_block)) .route("/regtest/send/:address/:amount", get(send_coins)); - } else { - tokio::spawn(async { - if let Err(e) = init_metrics().await { - error!("Error in periodic metrics: {e}"); - } - }); } + tokio::spawn(async { + if let Err(e) = init_metrics().await { + error!("Error in init metrics: {e}"); + } + }); + let app = app.layer(CorsLayer::permissive()); let addr = SocketAddr::from(([0, 0, 0, 0], 7070)); diff --git a/src/carbonado/metrics.rs b/src/carbonado/metrics.rs index aa813db0..3a620200 100644 --- a/src/carbonado/metrics.rs +++ b/src/carbonado/metrics.rs @@ -1,18 +1,18 @@ #![cfg(not(target_arch = "wasm32"))] use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, env, path::{Path, PathBuf}, - sync::Arc, time::SystemTime, }; use anyhow::{anyhow, Result}; use chrono::{DateTime, Duration, NaiveDate, Utc}; -use log::debug; +use log::{debug, error}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -use tokio::{fs, sync::RwLock}; +use sled::{Config, Db, Mode}; +use tokio::fs; use walkdir::WalkDir; #[derive(Serialize, Deserialize, Default)] @@ -47,12 +47,34 @@ const NETWORK_TOTAL: &str = "total"; const NETWORK_RGB_STOCKS: &str = "rgb_stocks"; const NETWORK_RGB_TRANSFER_FILES: &str = "rgb_transfer_files"; -static METRICS_DATA: Lazy>> = Lazy::new(Default::default); -static METRICS_SET: Lazy>>> = Lazy::new(Default::default); +const DB_PATHS: &str = "PATHS"; +const DB_DAYS: &str = "DAYS"; + +static METRICS_DB: Lazy = Lazy::new(|| { + Config::default() + .path( + PathBuf::from( + env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()), + ) + .join("metrics_sled_kv"), + ) + .mode(Mode::HighThroughput) // Since this uses Tor, disk IO will not be a bottleneck + .compression_factor(19) + .open() + .unwrap_or_else(|e| { + error!( + "Trouble opening Sled keystore: {}. Using a temporary in-memory database.", + e + ); + Config::default() + .temporary(true) + .open() + .expect("temporary sled db") + }) +}); pub async fn init(dir: &Path) -> Result<()> { - let mut metrics = METRICS_DATA.write().await; - let mut dataset = METRICS_SET.write().await; + let mut metrics = MetricsData::default(); metrics .wallets_by_network @@ -85,7 +107,9 @@ pub async fn init(dir: &Path) -> Result<()> { let day_created = metadata.created()?; let day = round_datetime_to_day(day_created.into()); - dataset.insert(entry.path().to_path_buf()); + METRICS_DB + .open_tree(DB_PATHS)? + .insert(entry.path().to_str().unwrap_or("ERROR").as_bytes(), &[1])?; if metadata.is_file() { metrics.bytes += metadata.len(); @@ -264,20 +288,27 @@ pub async fn init(dir: &Path) -> Result<()> { } } + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + + fs::write(&format!("{dir}/metrics.json"), &json(&metrics).await?).await?; + fs::write(&format!("{dir}/metrics.csv"), &csv(&metrics).await).await?; + Ok(()) } pub async fn update(path: &Path) -> Result<()> { debug!("Updating metrics with {path:?}"); - let mut metrics = METRICS_DATA.write().await; - let mut dataset = METRICS_SET.write().await; - - if dataset.get(path).is_some() { + if METRICS_DB + .open_tree(DB_PATHS)? + .contains_key(path.to_str().unwrap_or("ERROR").as_bytes())? + { debug!("Path already present"); return Ok(()); } else { - dataset.insert(path.to_path_buf()); + METRICS_DB + .open_tree(DB_PATHS)? + .insert(path.to_str().unwrap_or("ERROR").as_bytes(), &[1])?; } let filename = path @@ -287,9 +318,102 @@ pub async fn update(path: &Path) -> Result<()> { .to_string(); let metadata = path.metadata()?; let day_created = metadata.created()?; + let day_prior = day_created + .checked_sub(Duration::days(1).to_std()?) + .expect("day exists"); + let day_prior = round_datetime_to_day(day_prior.into()); let day = round_datetime_to_day(day_created.into()); + let first_of_day = if METRICS_DB + .open_tree(DB_DAYS)? + .contains_key(day.as_bytes())? + { + debug!("Day already present"); + false + } else { + METRICS_DB + .open_tree(DB_DAYS)? + .insert(day.as_bytes(), &[1])?; + true + }; + + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let mut metrics: MetricsData = + serde_json::from_str(&fs::read_to_string(format!("{dir}/metrics.json")).await?)?; + if metadata.is_file() { + if first_of_day { + let bytes_day_prior = { + metrics + .bytes_by_day + .get(&day_prior) + .unwrap_or(&0) + .to_owned() + }; + + metrics + .bytes_by_day + .entry(day.clone()) + .and_modify(|b| *b += bytes_day_prior) + .or_insert(bytes_day_prior); + + let bitcoin_wallets_day_prior = { + metrics + .bitcoin_wallets_by_day + .get(&day_prior) + .unwrap_or(&0) + .to_owned() + }; + + metrics + .bitcoin_wallets_by_day + .entry(day.clone()) + .and_modify(|w| *w += bitcoin_wallets_day_prior) + .or_insert(bitcoin_wallets_day_prior); + + let testnet_wallets_day_prior = { + metrics + .testnet_wallets_by_day + .get(&day_prior) + .unwrap_or(&0) + .to_owned() + }; + + metrics + .testnet_wallets_by_day + .entry(day.clone()) + .and_modify(|w| *w += testnet_wallets_day_prior) + .or_insert(testnet_wallets_day_prior); + + let signet_wallets_day_prior = { + metrics + .signet_wallets_by_day + .get(&day_prior) + .unwrap_or(&0) + .to_owned() + }; + + metrics + .signet_wallets_by_day + .entry(day.clone()) + .and_modify(|w| *w += signet_wallets_day_prior) + .or_insert(signet_wallets_day_prior); + + let regtest_wallets_day_prior = { + metrics + .regtest_wallets_by_day + .get(&day_prior) + .unwrap_or(&0) + .to_owned() + }; + + metrics + .regtest_wallets_by_day + .entry(day.clone()) + .and_modify(|w| *w += regtest_wallets_day_prior) + .or_insert(regtest_wallets_day_prior); + } + metrics.bytes += metadata.len(); *metrics.bytes_by_day.entry(day.clone()).or_insert(0) += metadata.len(); @@ -370,16 +494,14 @@ pub async fn update(path: &Path) -> Result<()> { } } - let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); - // Write metrics to disk as a backup - fs::write(&format!("{dir}/metrics.csv"), &csv().await).await?; - fs::write(&format!("{dir}/metrics.json"), &json().await?).await?; + fs::write(&format!("{dir}/metrics.json"), &json(&metrics).await?).await?; + fs::write(&format!("{dir}/metrics.csv"), &csv(&metrics).await).await?; Ok(()) } -pub async fn csv() -> String { +pub async fn csv(metrics: &MetricsData) -> String { let mut lines = vec![vec![ "Wallet".to_owned(), "Wallet Count".to_owned(), @@ -392,8 +514,6 @@ pub async fn csv() -> String { "Bytes by Day".to_owned(), ]]; - let metrics = METRICS_DATA.read().await; - for (day, bitcoin_wallets) in metrics.bitcoin_wallets_by_day.iter() { let mut line = vec![]; @@ -403,7 +523,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_BITCOIN) - .expect("network is defined") + .unwrap_or(&0) .to_string(), ); line.push(metrics.bytes.to_string()); @@ -415,7 +535,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_TESTNET) - .expect("network is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -427,7 +547,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_SIGNET) - .expect("network is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -439,7 +559,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_REGTEST) - .expect("network is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -451,7 +571,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_TOTAL) - .expect("total is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -463,7 +583,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_RGB_STOCKS) - .expect("rgb_stocks is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -475,7 +595,7 @@ pub async fn csv() -> String { metrics .wallets_by_network .get(NETWORK_RGB_TRANSFER_FILES) - .expect("rgb_transfer_files is defined") + .unwrap_or(&0) .to_string(), ); line.push("".to_owned()); @@ -519,10 +639,8 @@ pub async fn csv() -> String { lines.join("\n") } -pub async fn json() -> Result { - let metrics = METRICS_DATA.read().await; - - Ok(serde_json::to_string_pretty(&*metrics)?) +pub async fn json(metrics: &MetricsData) -> Result { + Ok(serde_json::to_string_pretty(metrics)?) } fn round_datetime_to_day(datetime: DateTime) -> String { diff --git a/src/regtest.rs b/src/regtest.rs index 4edbdcd8..fd59b01d 100644 --- a/src/regtest.rs +++ b/src/regtest.rs @@ -1,6 +1,25 @@ #![cfg(not(target_arch = "wasm32"))] -use std::env; -use std::process::{Command, Stdio}; +use std::{ + env, fs, path, + process::{Command, Stdio}, +}; + +use anyhow::Result; + +use crate::carbonado::metrics::MetricsData; + +pub fn init_fs() -> Result<()> { + let dir = env::var("CARBONADO_DIR").unwrap_or("/tmp/bitmaskd/carbonado".to_owned()); + let dir = path::Path::new(&dir); + + fs::create_dir_all(dir)?; + fs::write( + dir.join("metrics.json"), + serde_json::to_string_pretty(&MetricsData::default())?, + )?; + + Ok(()) +} pub fn send_coins(address: &str, amount: &str) { let path = env::current_dir().expect("oh no!"); diff --git a/tests/_init.rs b/tests/_init.rs new file mode 100644 index 00000000..04f403d3 --- /dev/null +++ b/tests/_init.rs @@ -0,0 +1,11 @@ +#![cfg(not(target_arch = "wasm32"))] + +use anyhow::Result; +use bitmask_core::regtest::init_fs; + +#[test] +pub fn _init() -> Result<()> { + init_fs()?; + + Ok(()) +}