From dfa911b2cfb4f784c507d7cfa9998a1d18577509 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Jul 2022 14:33:24 -0700 Subject: [PATCH 01/80] wip: monitor --- agents/monitor/Cargo.toml | 15 +++++++ agents/monitor/src/main.rs | 90 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 agents/monitor/Cargo.toml create mode 100644 agents/monitor/src/main.rs diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml new file mode 100644 index 00000000..8de310e9 --- /dev/null +++ b/agents/monitor/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "monitor" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "1.0.1", features = ["rt", "macros"] } +prometheus = "0.13.1" +tracing = "0.1.35" + +ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false, features = ['legacy'] } + +nomad-core = { path = "../../nomad-core", default-features = false } \ No newline at end of file diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs new file mode 100644 index 00000000..4862ef6e --- /dev/null +++ b/agents/monitor/src/main.rs @@ -0,0 +1,90 @@ +use tokio::task::JoinHandle; +use tracing::{info_span, instrument::Instrumented, Instrument}; + +#[tokio::main] +async fn main() { + println!("Hello, world!"); +} + +/// Has timestamp +pub trait NomadEvent: Send + Sync { + /// Get the timestamp + fn timestamp(&self) -> u32; + + /// block number + fn block_number(&self) -> u64; + + /// tx hash + fn tx_hash(&self) -> ethers::types::H256; +} + +// Simple struct +pub struct BetweenEvents { + incoming: tokio::sync::mpsc::Receiver, + outgoing: tokio::sync::mpsc::Sender, + count: prometheus::IntCounter, + wallclock_latency: prometheus::Histogram, + timestamp_latency: prometheus::Histogram, + network: String, +} + +/// Track latency between blockchain events +impl BetweenEvents +where + T: NomadEvent + 'static, +{ + pub fn new( + incoming: tokio::sync::mpsc::Receiver, + outgoing: tokio::sync::mpsc::Sender, + count: prometheus::IntCounter, + wallclock_latency: prometheus::Histogram, + timestamp_latency: prometheus::Histogram, + network: String, + ) -> Self { + Self { + incoming, + outgoing, + count, + wallclock_latency, + timestamp_latency, + network, + } + } + + pub fn spawn(mut self) -> Instrumented> { + let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); + + tokio::spawn(async move { + let mut last_timestamp = 0; + let mut wallclock_latency = self.wallclock_latency.start_timer(); + + loop { + // get the next event from the channel + let incoming = self.incoming.recv().await; + if incoming.is_none() { + break; + } + let incoming = incoming.unwrap(); + + // calculate the blockchain-reported latency in seconds + let timestamp = incoming.timestamp(); + let event_latency = timestamp.saturating_sub(last_timestamp); + last_timestamp = timestamp; + + // update our metrics + self.count.inc(); + self.timestamp_latency.observe(event_latency as f64); + wallclock_latency.observe_duration(); + + // send the next event out + if self.outgoing.send(incoming).await.is_err() { + break; + } + + // restart the timer + wallclock_latency = self.wallclock_latency.start_timer(); + } + }) + .instrument(span) + } +} From 60bcc688f380cf9f2e53cf234ae2eafe02ea5bd5 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Jul 2022 18:21:48 -0700 Subject: [PATCH 02/80] feat: basic init logic --- Cargo.lock | 1063 ++++++----------- agents/monitor/Cargo.toml | 5 +- agents/monitor/src/between.rs | 76 ++ agents/monitor/src/init.rs | 85 ++ agents/monitor/src/main.rs | 91 +- chains/nomad-ethereum/src/bindings/home.rs | 257 +++- chains/nomad-ethereum/src/bindings/replica.rs | 241 +++- .../src/bindings/xappconnectionmanager.rs | 107 +- configuration/src/builtin.rs | 2 +- 9 files changed, 1099 insertions(+), 828 deletions(-) create mode 100644 agents/monitor/src/between.rs create mode 100644 agents/monitor/src/init.rs diff --git a/Cargo.lock b/Cargo.lock index a0f55a7c..118ca00a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,7 +20,7 @@ dependencies = [ "ethers", "js-sys", "once_cell", - "serde 1.0.137", + "serde 1.0.140", "sha3 0.9.1", "thiserror", "wasm-bindgen", @@ -94,9 +94,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "arrayvec" @@ -110,22 +110,13 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" -[[package]] -name = "ascii-canvas" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" -dependencies = [ - "term", -] - [[package]] name = "assert-json-diff" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", "serde_json", ] @@ -172,15 +163,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "atomic-shim" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cd4b51d303cf3501c301e8125df442128d3c6d7c69f71b27833d253de47e77" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "atty" version = "0.2.14" @@ -212,9 +194,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" dependencies = [ "addr2line", "cc", @@ -229,7 +211,7 @@ dependencies = [ name = "balance-exporter" version = "0.1.0" dependencies = [ - "clap 3.1.18", + "clap 3.2.14", "color-eyre", "ethers-core", "futures", @@ -238,7 +220,7 @@ dependencies = [ "metrics-exporter-prometheus", "nomad-base", "nomad-ethereum", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "tokio", ] @@ -295,7 +277,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -317,21 +299,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "bit-set" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -350,9 +317,9 @@ dependencies = [ [[package]] name = "bitvec" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium 0.7.0", @@ -362,13 +329,11 @@ dependencies = [ [[package]] name = "blake2" -version = "0.9.2" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388" dependencies = [ - "crypto-mac 0.8.0", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest 0.10.3", ] [[package]] @@ -459,11 +424,11 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -483,7 +448,7 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "869119e97797867fd90f5e22af7d0bd274bd4635ebb9eb68c04f3f513ae6c412" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -492,19 +457,19 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "cargo_metadata" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +checksum = "3abb7553d5b9b8421c6de7cb02606ff15e0c6eea7d8eadd75ef013fd636bec36" dependencies = [ "camino", "cargo-platform", "semver", - "serde 1.0.137", + "serde 1.0.140", "serde_json", ] @@ -547,7 +512,7 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.15", - "serde 1.0.137", + "serde 1.0.140", "winapi", ] @@ -588,16 +553,16 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.18" +version = "3.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" +checksum = "54635806b078b7925d6e36810b1755f2a4b5b4d57560432c1ecf60bcbe10602b" dependencies = [ "atty", "bitflags", "clap_derive", "clap_lex", "indexmap", - "lazy_static", + "once_cell", "strsim 0.10.0", "termcolor", "textwrap 0.15.0", @@ -605,9 +570,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.18" +version = "3.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" +checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -618,75 +583,75 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ "os_str_bytes", ] [[package]] name = "coins-bip32" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "471b39eadc9323de375dce5eff149a5a1ebd21c67f1da34a56f87ee62191d4ea" +checksum = "634c509653de24b439672164bbf56f5f582a2ab0e313d3b0f6af0b7345cf2560" dependencies = [ "bincode", "bs58", "coins-core", - "digest 0.9.0", + "digest 0.10.3", "getrandom", - "hmac 0.11.0", + "hmac 0.12.1", "k256", "lazy_static", - "serde 1.0.137", - "sha2 0.9.9", + "serde 1.0.140", + "sha2 0.10.2", "thiserror", ] [[package]] name = "coins-bip39" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f473ea37dfc9d2cb94fdde50c3d41f28c3f384b367573d66386fea38d76d466" +checksum = "2a11892bcac83b4c6e95ab84b5b06c76d9d70ad73548dd07418269c5c7977171" dependencies = [ "bitvec 0.17.4", "coins-bip32", "getrandom", "hex", - "hmac 0.11.0", - "pbkdf2 0.8.0", + "hmac 0.12.1", + "pbkdf2 0.11.0", "rand", - "sha2 0.9.9", + "sha2 0.10.2", "thiserror", ] [[package]] name = "coins-core" -version = "0.2.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d257d975731955ee86fa7f348000c3fea09c262e84c70c11e994a85aa4f467a7" +checksum = "c94090a6663f224feae66ab01e41a2555a8296ee07b5f20dab8888bdefc9f617" dependencies = [ "base58check", "base64 0.12.3", "bech32", "blake2", - "digest 0.9.0", + "digest 0.10.3", "generic-array 0.14.5", "hex", - "ripemd160", - "serde 1.0.137", + "ripemd", + "serde 1.0.140", "serde_derive", - "sha2 0.9.9", - "sha3 0.9.1", + "sha2 0.10.2", + "sha3 0.10.1", "thiserror", ] [[package]] name = "color-eyre" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" dependencies = [ "backtrace", "color-spantrace", @@ -729,7 +694,7 @@ dependencies = [ "lazy_static", "nom 5.1.2", "rust-ini", - "serde 1.0.137", + "serde 1.0.140", "serde-hjson", "serde_json", "toml", @@ -745,19 +710,13 @@ dependencies = [ "lazy_static", "nom 5.1.2", "rust-ini", - "serde 1.0.137", + "serde 1.0.140", "serde-hjson", "serde_json", "toml", "yaml-rust", ] -[[package]] -name = "const-oid" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" - [[package]] name = "const-oid" version = "0.9.0" @@ -806,47 +765,36 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" dependencies = [ "autocfg", "cfg-if 1.0.0", "crossbeam-utils", - "lazy_static", "memoffset", + "once_cell", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" dependencies = [ "cfg-if 1.0.0", - "lazy_static", + "once_cell", ] [[package]] @@ -857,9 +805,9 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-bigint" -version = "0.3.2" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" +checksum = "9f2b443d17d49dad5ef0ede301c3179cc923b8822f3393b4d2c28c269dd4a122" dependencies = [ "generic-array 0.14.5", "rand_core", @@ -869,24 +817,14 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.3" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array 0.14.5", "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array 0.14.5", - "subtle", -] - [[package]] name = "crypto-mac" version = "0.11.1" @@ -906,30 +844,16 @@ dependencies = [ "cipher", ] -[[package]] -name = "der" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" -dependencies = [ - "const-oid 0.7.1", -] - [[package]] name = "der" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" dependencies = [ - "const-oid 0.9.0", + "const-oid", + "zeroize", ] -[[package]] -name = "diff" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" - [[package]] name = "difference" version = "2.0.0" @@ -1006,11 +930,11 @@ checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541" [[package]] name = "ecdsa" -version = "0.13.4" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d69ae62e0ce582d56380743515fefaf1a8c70cec685d9677636d7e30ae9dc9" +checksum = "3bd46e0c364655e5baf2f5e99b603e7a09905da9966d7928d7470af393b28670" dependencies = [ - "der 0.5.1", + "der", "elliptic-curve", "rfc6979", "signature", @@ -1018,37 +942,30 @@ dependencies = [ [[package]] name = "either" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "elliptic-curve" -version = "0.11.12" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b477563c2bfed38a3b7a60964c49e058b2510ad3f12ba3483fd8f62c2306d6" +checksum = "c47abd0a791d2ac0c7aa1118715f85b83689e4522c4e3a244e159d4fc9848a8d" dependencies = [ "base16ct", "crypto-bigint", - "der 0.5.1", + "der", + "digest 0.10.3", "ff", "generic-array 0.14.5", "group", + "pkcs8", "rand_core", "sec1", "subtle", "zeroize", ] -[[package]] -name = "ena" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3" -dependencies = [ - "log", -] - [[package]] name = "encoding_rs" version = "0.8.31" @@ -1072,7 +989,7 @@ dependencies = [ "pbkdf2 0.10.1", "rand", "scrypt", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "sha2 0.10.2", "sha3 0.10.1", @@ -1082,15 +999,15 @@ dependencies = [ [[package]] name = "ethabi" -version = "17.0.0" +version = "17.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69517146dfab88e9238c00c724fd8e277951c3cc6f22b016d72f422a832213e" +checksum = "f186de076b3e77b8e6d73c99d1b52edc2a229e604f4b5eb6992c06c11d79d537" dependencies = [ "ethereum-types", "hex", "once_cell", "regex", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "sha3 0.10.1", "thiserror", @@ -1126,8 +1043,8 @@ dependencies = [ [[package]] name = "ethers" -version = "0.6.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "ethers-addressbook", "ethers-contract", @@ -1136,24 +1053,23 @@ dependencies = [ "ethers-middleware", "ethers-providers", "ethers-signers", - "ethers-solc", ] [[package]] name = "ethers-addressbook" -version = "0.1.0" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "ethers-core", "once_cell", - "serde 1.0.137", + "serde 1.0.140", "serde_json", ] [[package]] name = "ethers-contract" -version = "0.6.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "ethers-contract-abigen", "ethers-contract-derive", @@ -1163,15 +1079,15 @@ dependencies = [ "hex", "once_cell", "pin-project", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", ] [[package]] name = "ethers-contract-abigen" -version = "0.6.3" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "Inflector", "cfg-if 1.0.0", @@ -1183,7 +1099,7 @@ dependencies = [ "proc-macro2", "quote", "reqwest", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "syn", "url", @@ -1192,8 +1108,8 @@ dependencies = [ [[package]] name = "ethers-contract-derive" -version = "0.6.3" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "ethers-contract-abigen", "ethers-core", @@ -1206,8 +1122,8 @@ dependencies = [ [[package]] name = "ethers-core" -version = "0.6.3" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "arrayvec 0.7.2", "bytes", @@ -1216,6 +1132,7 @@ dependencies = [ "convert_case", "elliptic-curve", "ethabi", + "fastrlp", "generic-array 0.14.5", "hex", "k256", @@ -1225,7 +1142,7 @@ dependencies = [ "rlp", "rlp-derive", "rust_decimal", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "strum", "syn", @@ -1236,14 +1153,14 @@ dependencies = [ [[package]] name = "ethers-etherscan" -version = "0.2.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "ethers-core", - "ethers-solc", + "getrandom", "reqwest", "semver", - "serde 1.0.137", + "serde 1.0.140", "serde-aux", "serde_json", "thiserror", @@ -1252,8 +1169,8 @@ dependencies = [ [[package]] name = "ethers-middleware" -version = "0.6.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "async-trait", "ethers-contract", @@ -1265,7 +1182,7 @@ dependencies = [ "futures-util", "instant", "reqwest", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", "tokio", @@ -1276,8 +1193,8 @@ dependencies = [ [[package]] name = "ethers-providers" -version = "0.6.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "async-trait", "auto_impl", @@ -1287,6 +1204,7 @@ dependencies = [ "futures-core", "futures-timer", "futures-util", + "getrandom", "hashers", "hex", "http", @@ -1294,11 +1212,11 @@ dependencies = [ "parking_lot 0.11.2", "pin-project", "reqwest", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", "tokio", - "tokio-tungstenite 0.17.1", + "tokio-tungstenite 0.17.2", "tracing", "tracing-futures", "url", @@ -1311,8 +1229,8 @@ dependencies = [ [[package]] name = "ethers-signers" -version = "0.6.2" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" +version = "0.13.0" +source = "git+https://github.com/gakonst/ethers-rs?branch=master#a958b4fd28709a995c57ef5535cbee1e4e4be543" dependencies = [ "async-trait", "coins-bip32", @@ -1324,40 +1242,10 @@ dependencies = [ "rand", "rusoto_core", "rusoto_kms", - "sha2 0.9.9", - "spki 0.6.0", - "thiserror", - "tracing", -] - -[[package]] -name = "ethers-solc" -version = "0.3.0" -source = "git+https://github.com/gakonst/ethers-rs?branch=master#00b38c437a7cae6fbe9f846ffbb12a03fbed0df5" -dependencies = [ - "cfg-if 1.0.0", - "colored", - "dunce", - "ethers-core", - "getrandom", - "glob", - "hex", - "home", - "md-5 0.10.1", - "num_cpus", - "once_cell", - "path-slash", - "rayon", - "regex", - "semver", - "serde 1.0.137", - "serde_json", - "solang-parser", + "sha2 0.10.2", + "spki", "thiserror", - "tiny-keccak", - "tokio", "tracing", - "walkdir", ] [[package]] @@ -1385,11 +1273,36 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrlp" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c60b758dc5bf92743e1b863ac88b84a4750bd096b19c2f524359659dc1f1ac" +dependencies = [ + "arrayvec 0.7.2", + "auto_impl", + "bytes", + "ethereum-types", + "fastrlp-derive", +] + +[[package]] +name = "fastrlp-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9499f20a2fa1a744422de24d1b4d1ec58f240147de1d0a3ceacadf2e240b4fc2" +dependencies = [ + "bytes", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ff" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "131655483be284720a17d74ff97592b8e76576dc25563148601df2d7c9080924" +checksum = "df689201f395c6b90dfe87127685f8dbfc083a5e779e613575d8bd7314300c3e" dependencies = [ "rand_core", "subtle", @@ -1407,12 +1320,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "fixedbitset" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" - [[package]] name = "float-cmp" version = "0.8.0" @@ -1455,9 +1362,9 @@ dependencies = [ [[package]] name = "fragile" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d758e60b45e8d749c89c1b389ad8aee550f86aa12e2b9298b546dda7a82ab1" +checksum = "85dcb89d2b10c5f6133de2efd8c11959ce9dbb46a2f7a4cab208c4eeda6ce1ab" [[package]] name = "funty" @@ -1594,7 +1501,7 @@ dependencies = [ "once_cell", "pin-project", "reqwest", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serde_repr", "thiserror", @@ -1623,22 +1530,22 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" [[package]] name = "glob" @@ -1648,9 +1555,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "group" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5ac374b108929de78460075f3dc439fa66df9d8fc77e8f12caa5165fcf0c89" +checksum = "7391856def869c1c81063a03457c676fbcd419709c3dfb33d8d319de484b154d" dependencies = [ "ff", "rand_core", @@ -1672,15 +1579,15 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.2", + "tokio-util 0.7.3", "tracing", ] [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "ahash", ] @@ -1755,7 +1662,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ - "crypto-mac 0.11.1", + "crypto-mac", "digest 0.9.0", ] @@ -1768,20 +1675,11 @@ dependencies = [ "digest 0.10.3", ] -[[package]] -name = "home" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2456aef2e6b6a9784192ae780c0f15bc57df0e918585282325e8c8ac27737654" -dependencies = [ - "winapi", -] - [[package]] name = "http" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", @@ -1819,7 +1717,7 @@ checksum = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36" dependencies = [ "backtrace", "os_type", - "serde 1.0.137", + "serde 1.0.140", "serde_derive", "termcolor", "toml", @@ -1828,9 +1726,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.19" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ "bytes", "futures-channel", @@ -1911,7 +1809,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -1933,9 +1831,9 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", "hashbrown", @@ -1959,15 +1857,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" -[[package]] -name = "itertools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.2" @@ -1985,25 +1874,24 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.57" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" dependencies = [ "wasm-bindgen", ] [[package]] name = "k256" -version = "0.10.4" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c3a5e0a0b8450278feda242592512e09f61c72e018b8cd5c859482802daf2d" +checksum = "2c8a5a96d92d849c4499d99461da81c9cdc1467418a8ed2aaeb407e8d85940ed" dependencies = [ "cfg-if 1.0.0", "ecdsa", "elliptic-curve", - "sec1", - "sha2 0.9.9", - "sha3 0.9.1", + "sha2 0.10.2", + "sha3 0.10.1", ] [[package]] @@ -2025,14 +1913,14 @@ dependencies = [ "nomad-xyz-configuration", "prometheus", "rand", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serial_test", "thiserror", "tokio", "tracing", "tracing-futures", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", ] [[package]] @@ -2045,7 +1933,7 @@ checksum = "f9b7d56ba4a8344d6be9729995e6b06f928af29998cdf79fe390cbf6b1fee838" name = "kms-cli" version = "0.1.0" dependencies = [ - "clap 3.1.18", + "clap 3.2.14", "color-eyre", "ethers", "ethers-signers", @@ -2057,38 +1945,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "lalrpop" -version = "0.19.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823" -dependencies = [ - "ascii-canvas", - "atty", - "bit-set", - "diff", - "ena", - "itertools", - "lalrpop-util", - "petgraph", - "pico-args", - "regex", - "regex-syntax", - "string_cache", - "term", - "tiny-keccak", - "unicode-xid", -] - -[[package]] -name = "lalrpop-util" -version = "0.19.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4" -dependencies = [ - "regex", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2132,8 +1988,8 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.7.0+7.1.2" -source = "git+https://github.com/rust-rocksdb/rust-rocksdb#59434f93ed91e4252d8ff64bfd28b4f3203572a4" +version = "0.8.0+7.4.3" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb#78dba2100bd8d9279cd7c5cc3cd8f3e60560a527" dependencies = [ "bindgen", "bzip2-sys", @@ -2167,9 +2023,9 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "lock_api" @@ -2216,15 +2072,6 @@ dependencies = [ "opaque-debug 0.3.0", ] -[[package]] -name = "md-5" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" -dependencies = [ - "digest 0.10.3", -] - [[package]] name = "memchr" version = "2.5.0" @@ -2248,26 +2095,28 @@ checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" [[package]] name = "metrics" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" +checksum = "94a70dc6c8179c4e889928dffb92336b821462c7b7613de91304fe80040fec12" dependencies = [ "ahash", "metrics-macros", + "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953cbbb6f9ba4b9304f4df79b98cdc9d14071ed93065a9fca11c00c5d9181b66" +checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" dependencies = [ "hyper", "indexmap", "ipnet", "metrics", "metrics-util", - "parking_lot 0.11.2", + "parking_lot 0.12.1", + "portable-atomic", "quanta", "thiserror", "tokio", @@ -2287,17 +2136,17 @@ dependencies = [ [[package]] name = "metrics-util" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1f4b69bef1e2b392b2d4a12902f2af90bb438ba4a66aa222d1023fa6561b50" +checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" dependencies = [ - "atomic-shim", "crossbeam-epoch", "crossbeam-utils", "hashbrown", "metrics", "num_cpus", - "parking_lot 0.11.2", + "parking_lot 0.12.1", + "portable-atomic", "quanta", "sketches-ddsketch", ] @@ -2335,9 +2184,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", @@ -2453,12 +2302,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" - [[package]] name = "nom" version = "5.1.2" @@ -2502,7 +2345,7 @@ dependencies = [ "rocksdb", "rusoto_core", "rusoto_kms", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", "tokio", @@ -2510,7 +2353,7 @@ dependencies = [ "tracing-error", "tracing-futures", "tracing-opentelemetry", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", "warp", ] @@ -2555,7 +2398,7 @@ dependencies = [ "rusoto_credential", "rusoto_kms", "rusoto_sts", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "sha3 0.9.1", "thiserror", @@ -2587,7 +2430,7 @@ dependencies = [ "prometheus", "reqwest", "rocksdb", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", "tokio", @@ -2614,7 +2457,7 @@ dependencies = [ "prometheus", "rand", "rocksdb", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", "tokio", @@ -2630,7 +2473,7 @@ dependencies = [ "ethers", "hex", "prometheus", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "thiserror", ] @@ -2648,7 +2491,7 @@ dependencies = [ "nomad-types", "once_cell", "reqwest", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serde_yaml", "serial_test", @@ -2687,17 +2530,17 @@ dependencies = [ "autocfg", "num-integer", "num-traits 0.2.15", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "num-complex" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fbc387afefefd5e9e39493299f3069e14a140dd34dc19b4c1c1a8fddb6a790" +checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" dependencies = [ "num-traits 0.2.15", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -2723,15 +2566,15 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", "num-bigint", "num-integer", "num-traits 0.2.15", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -2764,18 +2607,18 @@ dependencies = [ [[package]] name = "object" -version = "0.28.4" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "opaque-debug" @@ -2791,9 +2634,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.40" +version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -2823,9 +2666,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.74" +version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835363342df5fba8354c5b453325b110ffd54044e588c539cf2f20a8014e4cb1" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ "autocfg", "cc", @@ -2855,9 +2698,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.1.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "os_type" @@ -2876,23 +2719,23 @@ checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" [[package]] name = "parity-scale-codec" -version = "3.1.2" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8b44461635bbb1a0300f100a841e571e7d919c81c73075ef5d152ffdb521066" +checksum = "9182e4a71cae089267ab03e67c99368db7cd877baf50f931e5d6d4b71e195ac0" dependencies = [ "arrayvec 0.7.2", - "bitvec 1.0.0", + "bitvec 1.0.1", "byte-slice-cast", "impl-trait-for-tuples", "parity-scale-codec-derive", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "parity-scale-codec-derive" -version = "3.1.2" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c45ed1f39709f5a89338fab50e59816b2e8815f5bb58276e7ddf9afd495f73f8" +checksum = "9299338969a3d2f491d65f140b00ddec470858402f888af98e8642fb5e8965cd" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2950,9 +2793,9 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.2.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +checksum = "1d791538a6dcc1e7cb7fe6f6b58aca40e7f79403c45b2bc274008b5e647af1d8" dependencies = [ "base64ct", "rand_core", @@ -2961,43 +2804,36 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.3.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d791538a6dcc1e7cb7fe6f6b58aca40e7f79403c45b2bc274008b5e647af1d8" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" dependencies = [ "base64ct", "rand_core", "subtle", ] -[[package]] -name = "path-slash" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cacbb3c4ff353b534a67fb8d7524d00229da4cb1dc8c79f4db96e375ab5b619" - [[package]] name = "pbkdf2" -version = "0.8.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" +checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" dependencies = [ - "base64ct", - "crypto-mac 0.11.1", - "hmac 0.11.0", - "password-hash 0.2.3", - "sha2 0.9.9", + "digest 0.10.3", + "hmac 0.12.1", + "password-hash 0.3.2", + "sha2 0.10.2", ] [[package]] name = "pbkdf2" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.3", "hmac 0.12.1", - "password-hash 0.3.2", + "password-hash 0.4.2", "sha2 0.10.2", ] @@ -3013,16 +2849,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "petgraph" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "pharos" version = "0.5.3" @@ -3033,70 +2859,20 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "phf" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" -dependencies = [ - "phf_macros", - "phf_shared", - "proc-macro-hack", -] - -[[package]] -name = "phf_generator" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_macros" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pico-args" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" - [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2", "quote", @@ -3117,13 +2893,12 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkcs8" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der 0.5.1", - "spki 0.5.4", - "zeroize", + "der", + "spki", ] [[package]] @@ -3133,16 +2908,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] -name = "ppv-lite86" -version = "0.2.16" +name = "portable-atomic" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "763095e04dcbeb889b2ab35296ecb18a20fe16b4e9877ce64aab73d8fd05a8c3" [[package]] -name = "precomputed-hash" -version = "0.1.1" +name = "ppv-lite86" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "predicates" @@ -3220,17 +2995,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" dependencies = [ "unicode-ident", ] @@ -3256,14 +3025,14 @@ dependencies = [ "rocksdb", "rusoto_core", "rusoto_s3", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serial_test", "thiserror", "tokio", "tracing", "tracing-futures", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", ] [[package]] @@ -3289,9 +3058,9 @@ checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" [[package]] name = "quanta" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" dependencies = [ "crossbeam-utils", "libc", @@ -3311,9 +3080,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] @@ -3369,30 +3138,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rayon" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" -dependencies = [ - "autocfg", - "crossbeam-deque", - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - [[package]] name = "redox_syscall" version = "0.2.13" @@ -3415,9 +3160,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.6" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -3426,9 +3171,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.26" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "relayer" @@ -3447,7 +3192,7 @@ dependencies = [ "nomad-test", "nomad-xyz-configuration", "prometheus", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serial_test", "thiserror", @@ -3455,7 +3200,7 @@ dependencies = [ "tokio-test", "tracing", "tracing-futures", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", ] [[package]] @@ -3469,9 +3214,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64 0.13.0", "bytes", @@ -3494,12 +3239,13 @@ dependencies = [ "pin-project-lite", "rustls", "rustls-pemfile", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", "tokio-rustls", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -3510,12 +3256,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ef608575f6392792f9ecf7890c00086591d29a83910939d430753f7c050525" +checksum = "88c86280f057430a52f4861551b092a01b419b8eacefc7c995eacb9dc132fe32" dependencies = [ "crypto-bigint", - "hmac 0.11.0", + "hmac 0.12.1", "zeroize", ] @@ -3535,14 +3281,12 @@ dependencies = [ ] [[package]] -name = "ripemd160" -version = "0.9.1" +name = "ripemd" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eca4ecc81b7f313189bf73ce724400a07da2a6dac19588b03c8bd76a2dcc251" +checksum = "1facec54cb5e0dc08553501fa740091086d0259ad0067e0d4103448e4cb22ed3" dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug 0.3.0", + "digest 0.10.3", ] [[package]] @@ -3569,7 +3313,7 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.18.0" -source = "git+https://github.com/rust-rocksdb/rust-rocksdb#59434f93ed91e4252d8ff64bfd28b4f3203572a4" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb#78dba2100bd8d9279cd7c5cc3cd8f3e60560a527" dependencies = [ "libc", "librocksdb-sys", @@ -3594,7 +3338,7 @@ dependencies = [ "rusoto_credential", "rusoto_signature", "rustc_version", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "tokio", "xml-rs", @@ -3611,7 +3355,7 @@ dependencies = [ "dirs-next", "futures", "hyper", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "shlex", "tokio", @@ -3628,7 +3372,7 @@ dependencies = [ "bytes", "futures", "rusoto_core", - "serde 1.0.137", + "serde 1.0.140", "serde_json", ] @@ -3661,12 +3405,12 @@ dependencies = [ "http", "hyper", "log", - "md-5 0.9.1", + "md-5", "percent-encoding", "pin-project-lite", "rusoto_credential", "rustc_version", - "serde 1.0.137", + "serde 1.0.140", "sha2 0.9.9", "tokio", ] @@ -3694,13 +3438,13 @@ checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" [[package]] name = "rust_decimal" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2ee7337df68898256ad0d4af4aad178210d9e44d2ff900ce44064a97cd86530" +checksum = "34a3bb58e85333f1ab191bf979104b586ebd77475bc6681882825f4532dfe87c" dependencies = [ "arrayvec 0.7.2", "num-traits 0.2.15", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -3744,18 +3488,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "0.3.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" dependencies = [ "base64 0.13.0", ] [[package]] name = "rustversion" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" [[package]] name = "ryu" @@ -3834,11 +3578,12 @@ dependencies = [ [[package]] name = "sec1" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08da66b8b0965a5555b6bd6639e68ccba85e1e2506f5fbb089e93f8a04e1a2d1" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ - "der 0.5.1", + "base16ct", + "der", "generic-array 0.14.5", "pkcs8", "subtle", @@ -3870,11 +3615,11 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -3891,20 +3636,20 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" dependencies = [ "serde_derive", ] [[package]] name = "serde-aux" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93abf9799c576f004252b2a05168d58527fb7c54de12e94b4d12fe3475ffad24" +checksum = "d0a77223b653fa95f3f9864f3eb25b93e4ed170687eb42d85b6b98af21d5e1de" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", "serde_json", ] @@ -3923,9 +3668,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" dependencies = [ "proc-macro2", "quote", @@ -3934,13 +3679,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ "itoa", "ryu", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -3972,18 +3717,18 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "serde_yaml" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ "indexmap", "ryu", - "serde 1.0.137", + "serde 1.0.140", "yaml-rust", ] @@ -4119,11 +3864,11 @@ dependencies = [ [[package]] name = "signature" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02658e48d89f2bec991f9a78e69cfa4c316f8d6a6c4ec12fae1aeb263d486788" +checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" dependencies = [ - "digest 0.9.0", + "digest 0.10.3", "rand_core", ] @@ -4133,29 +3878,26 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e24979f63a11545f5f2c60141afe249d4f19f84581ea2138065e400941d83d3" -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - [[package]] name = "sketches-ddsketch" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2ecae5fcf33b122e2e6bd520a57ccf152d2dde3b38c71039df1a6867264ee" +checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" [[package]] name = "slab" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" @@ -4167,42 +3909,20 @@ dependencies = [ "winapi", ] -[[package]] -name = "solang-parser" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395b6e1ec5af117bd08f963c7cd80f8efd4eed51c5a332aed42b13e3f9bc860b" -dependencies = [ - "itertools", - "lalrpop", - "lalrpop-util", - "phf", - "unicode-xid", -] - [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "spki" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" -dependencies = [ - "base64ct", - "der 0.5.1", -] - [[package]] name = "spki" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ - "der 0.6.0", + "base64ct", + "der", ] [[package]] @@ -4211,19 +3931,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "string_cache" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" -dependencies = [ - "new_debug_unreachable", - "once_cell", - "parking_lot 0.12.1", - "phf_shared", - "precomputed-hash", -] - [[package]] name = "strsim" version = "0.8.0" @@ -4262,18 +3969,18 @@ dependencies = [ [[package]] name = "strum" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" +checksum = "4faebde00e8ff94316c01800f9054fd2ba77d30d9e922541913051d1d978918b" dependencies = [ "heck 0.4.0", "proc-macro2", @@ -4290,9 +3997,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.96" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", @@ -4319,17 +4026,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "termcolor" version = "1.1.3" @@ -4415,10 +4111,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f392c8f16bda3456c0b00c6de39cb100449b98de55ac41c6cdd2bfcf53a1245" +checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e" dependencies = [ + "autocfg", "bytes", "libc", "memchr", @@ -4434,9 +4131,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ "proc-macro2", "quote", @@ -4466,9 +4163,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ "futures-core", "pin-project-lite", @@ -4503,16 +4200,16 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.17.1" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06cda1232a49558c46f8a504d5b93101d42c0bf7f911f12a105ba48168f821ae" +checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", "rustls", "tokio", "tokio-rustls", - "tungstenite 0.17.2", + "tungstenite 0.17.3", "webpki", "webpki-roots", ] @@ -4533,9 +4230,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", @@ -4551,14 +4248,14 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", ] [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" @@ -4575,9 +4272,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -4601,7 +4298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" dependencies = [ "tracing", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", ] [[package]] @@ -4644,7 +4341,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" dependencies = [ - "serde 1.0.137", + "serde 1.0.140", "tracing-core", ] @@ -4661,12 +4358,12 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a713421342a5a666b7577783721d3117f1b69a393df803ee17bb73b1e122a59" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" dependencies = [ "ansi_term", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "sharded-slab", "smallvec", @@ -4703,9 +4400,9 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.17.2" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" +checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ "base64 0.13.0", "byteorder", @@ -4766,15 +4463,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" +checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] @@ -4824,14 +4521,14 @@ dependencies = [ "nomad-xyz-configuration", "prometheus", "rocksdb", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serial_test", "thiserror", "tokio", "tracing", "tracing-futures", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", "warp", ] @@ -4860,7 +4557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom", - "serde 1.0.137", + "serde 1.0.140", ] [[package]] @@ -4927,7 +4624,7 @@ dependencies = [ "percent-encoding", "pin-project", "scoped-tls", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serde_urlencoded", "tokio", @@ -4952,21 +4649,21 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" dependencies = [ "cfg-if 1.0.0", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" dependencies = [ "bumpalo", "lazy_static", @@ -4979,9 +4676,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4991,9 +4688,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5001,9 +4698,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" dependencies = [ "proc-macro2", "quote", @@ -5014,9 +4711,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" [[package]] name = "wasm-timer" @@ -5052,7 +4749,7 @@ dependencies = [ "nomad-xyz-configuration", "prometheus", "rocksdb", - "serde 1.0.137", + "serde 1.0.140", "serde_json", "serial_test", "thiserror", @@ -5060,14 +4757,14 @@ dependencies = [ "tokio-test", "tracing", "tracing-futures", - "tracing-subscriber 0.3.14", + "tracing-subscriber 0.3.15", ] [[package]] name = "web-sys" -version = "0.3.57" +version = "0.3.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" dependencies = [ "js-sys", "wasm-bindgen", @@ -5085,9 +4782,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" dependencies = [ "webpki", ] @@ -5226,20 +4923,20 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ - "linked-hash-map 0.5.4", + "linked-hash-map 0.5.6", ] [[package]] name = "zeroize" -version = "1.5.5" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" [[package]] name = "zstd-sys" -version = "1.6.3+zstd.1.5.2" +version = "2.0.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" dependencies = [ "cc", "libc", diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index 8de310e9..37bbada9 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -12,4 +12,7 @@ tracing = "0.1.35" ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false, features = ['legacy'] } -nomad-core = { path = "../../nomad-core", default-features = false } \ No newline at end of file +nomad-core = { path = "../../nomad-core", default-features = false } +nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../../configuration" } +tracing-subscriber = "0.2.15" +eyre = "0.6.8" diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs new file mode 100644 index 00000000..6451ca28 --- /dev/null +++ b/agents/monitor/src/between.rs @@ -0,0 +1,76 @@ +use tracing::{info_span, instrument::Instrumented, Instrument}; + +use tokio::task::JoinHandle; + +use crate::NomadEvent; + +// Track time between events of the same kind +pub struct BetweenEvents { + pub(crate) incoming: tokio::sync::mpsc::Receiver, + pub(crate) outgoing: tokio::sync::mpsc::Sender, + pub(crate) count: prometheus::IntCounter, + pub(crate) wallclock_latency: prometheus::Histogram, + pub(crate) timestamp_latency: prometheus::Histogram, + pub(crate) network: String, +} + +/// Track latency between blockchain events +impl BetweenEvents +where + T: NomadEvent + 'static, +{ + pub fn new( + incoming: tokio::sync::mpsc::Receiver, + outgoing: tokio::sync::mpsc::Sender, + count: prometheus::IntCounter, + wallclock_latency: prometheus::Histogram, + timestamp_latency: prometheus::Histogram, + network: String, + ) -> Self { + Self { + incoming, + outgoing, + count, + wallclock_latency, + timestamp_latency, + network, + } + } + + pub fn spawn(mut self) -> Instrumented> { + let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); + + tokio::spawn(async move { + let mut last_timestamp = 0; + let mut wallclock_latency = self.wallclock_latency.start_timer(); + + loop { + // get the next event from the channel + let incoming = self.incoming.recv().await; + if incoming.is_none() { + break; + } + let incoming = incoming.unwrap(); + + // calculate the blockchain-reported latency in seconds + let timestamp = incoming.timestamp(); + let event_latency = timestamp.saturating_sub(last_timestamp); + last_timestamp = timestamp; + + // update our metrics + self.count.inc(); + self.timestamp_latency.observe(event_latency as f64); + wallclock_latency.observe_duration(); + + // send the next event out + if self.outgoing.send(incoming).await.is_err() { + break; + } + + // restart the timer + wallclock_latency = self.wallclock_latency.start_timer(); + } + }) + .instrument(span) + } +} diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs new file mode 100644 index 00000000..ff5c96eb --- /dev/null +++ b/agents/monitor/src/init.rs @@ -0,0 +1,85 @@ +use std::{collections::HashMap, sync::Arc}; + +use ethers::{ + middleware::TimeLag, + prelude::{Http, Provider}, +}; + +use nomad_xyz_configuration::{get_builtin, NomadConfig}; + +pub(crate) fn config_from_file() -> Option { + std::env::var("CONFIG_PATH") + .ok() + .and_then(|path| NomadConfig::from_file(path).ok()) +} + +pub(crate) fn config_from_env() -> Option { + std::env::var("RUN_ENV") + .ok() + .and_then(|env| get_builtin(&env)) + .map(ToOwned::to_owned) +} + +pub(crate) fn config() -> NomadConfig { + config_from_file() + .or_else(config_from_env) + .expect("could not load config from file or env") +} + +pub(crate) fn init_tracing() { + tracing_subscriber::FmtSubscriber::builder() + .json() + .with_level(true) + .init(); +} + +pub(crate) fn rpc_from_env(network: &str) -> Option { + std::env::var(format!("{}_CONNECTION_URL", network.to_uppercase())).ok() +} + +pub(crate) fn provider_for( + config: &NomadConfig, + network: &str, +) -> eyre::Result>>> { + tracing::info!(network, "Instantiating provider"); + + let url = rpc_from_env(network).or_else(|| { + config + .rpcs + .get(network) + .and_then(|set| set.iter().next().cloned()) + }); + eyre::ensure!( + url.is_some(), + "Missing Url. Please specify by config or env var." + ); + + let url = url.unwrap(); + let provider = Provider::::try_from(&url)?; + + let timelag = config + .protocol() + .networks + .get(network) + .expect("missing protocol block in config") + .specs + .finalization_blocks; + + tracing::debug!(url = url.as_str(), timelag, network, "Connect network"); + Ok(TimeLag::new(provider, timelag).into()) +} + +pub(crate) fn init_providers( + config: &NomadConfig, +) -> HashMap>>> { + config + .networks + .iter() + .map(|network| { + ( + network.clone(), + provider_for(config, network).expect("unable to make provider"), + ) + }) + .collect() +} diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 4862ef6e..2882f18d 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,12 +1,24 @@ -use tokio::task::JoinHandle; -use tracing::{info_span, instrument::Instrumented, Instrument}; +use tracing::info_span; + +mod between; + +mod init; #[tokio::main] async fn main() { - println!("Hello, world!"); + init::init_tracing(); + + { + let span = info_span!("MonitorBootup"); + let _span = span.enter(); + let config = init::config(); + let providers = init::init_providers(&config); + + tracing::info!("setup complete!") + } } -/// Has timestamp +/// Simple event trait pub trait NomadEvent: Send + Sync { /// Get the timestamp fn timestamp(&self) -> u32; @@ -17,74 +29,3 @@ pub trait NomadEvent: Send + Sync { /// tx hash fn tx_hash(&self) -> ethers::types::H256; } - -// Simple struct -pub struct BetweenEvents { - incoming: tokio::sync::mpsc::Receiver, - outgoing: tokio::sync::mpsc::Sender, - count: prometheus::IntCounter, - wallclock_latency: prometheus::Histogram, - timestamp_latency: prometheus::Histogram, - network: String, -} - -/// Track latency between blockchain events -impl BetweenEvents -where - T: NomadEvent + 'static, -{ - pub fn new( - incoming: tokio::sync::mpsc::Receiver, - outgoing: tokio::sync::mpsc::Sender, - count: prometheus::IntCounter, - wallclock_latency: prometheus::Histogram, - timestamp_latency: prometheus::Histogram, - network: String, - ) -> Self { - Self { - incoming, - outgoing, - count, - wallclock_latency, - timestamp_latency, - network, - } - } - - pub fn spawn(mut self) -> Instrumented> { - let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); - - tokio::spawn(async move { - let mut last_timestamp = 0; - let mut wallclock_latency = self.wallclock_latency.start_timer(); - - loop { - // get the next event from the channel - let incoming = self.incoming.recv().await; - if incoming.is_none() { - break; - } - let incoming = incoming.unwrap(); - - // calculate the blockchain-reported latency in seconds - let timestamp = incoming.timestamp(); - let event_latency = timestamp.saturating_sub(last_timestamp); - last_timestamp = timestamp; - - // update our metrics - self.count.inc(); - self.timestamp_latency.observe(event_latency as f64); - wallclock_latency.observe_duration(); - - // send the next event out - if self.outgoing.send(incoming).await.is_err() { - break; - } - - // restart the timer - wallclock_latency = self.wallclock_latency.start_timer(); - } - }) - .instrument(span) - } -} diff --git a/chains/nomad-ethereum/src/bindings/home.rs b/chains/nomad-ethereum/src/bindings/home.rs index b4487b29..3a5a9a0b 100644 --- a/chains/nomad-ethereum/src/bindings/home.rs +++ b/chains/nomad-ethereum/src/bindings/home.rs @@ -18,7 +18,7 @@ pub mod home_mod { use std::sync::Arc; pub static HOME_ABI: ethers::contract::Lazy = ethers::contract::Lazy::new(|| { - serde_json :: from_str ("[\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_localDomain\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"messageHash\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"uint256\",\n \"name\": \"leafIndex\",\n \"type\": \"uint256\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"uint64\",\n \"name\": \"destinationAndNonce\",\n \"type\": \"uint64\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"message\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Dispatch\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32[2]\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32[2]\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature2\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"DoubleUpdate\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"ImproperUpdate\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"oldUpdater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"newUpdater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdater\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdaterManager\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"homeDomain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Update\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"updater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"reporter\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"UpdaterSlashed\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [],\n \"name\": \"MAX_MESSAGE_BODY_BYTES\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"VERSION\",\n \"outputs\": [\n {\n \"internalType\": \"uint8\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"committedRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"count\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_destinationDomain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_recipientAddress\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_messageBody\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"dispatch\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32[2]\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32[2]\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature2\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"doubleUpdate\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"homeDomainHash\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"improperUpdate\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"contract IUpdaterManager\",\n \"name\": \"_updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"initialize\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"nonces\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_item\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"queueContains\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"queueEnd\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"queueLength\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"root\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdater\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdaterManager\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"state\",\n \"outputs\": [\n {\n \"internalType\": \"enum NomadBase.States\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"suggestUpdate\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_new\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"tree\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"count\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"update\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updater\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updaterManager\",\n \"outputs\": [\n {\n \"internalType\": \"contract IUpdaterManager\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") + ethers :: core :: utils :: __serde_json :: from_str ("[\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_localDomain\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"messageHash\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"uint256\",\n \"name\": \"leafIndex\",\n \"type\": \"uint256\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"uint64\",\n \"name\": \"destinationAndNonce\",\n \"type\": \"uint64\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"message\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Dispatch\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32[2]\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32[2]\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature2\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"DoubleUpdate\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"ImproperUpdate\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"oldUpdater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"newUpdater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdater\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdaterManager\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"homeDomain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Update\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"updater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"reporter\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"UpdaterSlashed\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [],\n \"name\": \"MAX_MESSAGE_BODY_BYTES\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"VERSION\",\n \"outputs\": [\n {\n \"internalType\": \"uint8\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"committedRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"count\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_destinationDomain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_recipientAddress\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_messageBody\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"dispatch\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32[2]\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32[2]\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature2\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"doubleUpdate\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"homeDomainHash\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"improperUpdate\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"contract IUpdaterManager\",\n \"name\": \"_updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"initialize\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"nonces\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_item\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"queueContains\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"queueEnd\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"queueLength\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"root\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdater\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updaterManager\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdaterManager\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"state\",\n \"outputs\": [\n {\n \"internalType\": \"enum NomadBase.States\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"suggestUpdate\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_new\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"tree\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"count\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"update\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updater\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updaterManager\",\n \"outputs\": [\n {\n \"internalType\": \"contract IUpdaterManager\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") }); pub struct Home(ethers::contract::Contract); impl Clone for Home { @@ -510,7 +510,7 @@ pub mod home_mod { } } } - #[doc = "Container type for all input parameters for the `MAX_MESSAGE_BODY_BYTES`function with signature `MAX_MESSAGE_BODY_BYTES()` and selector `[82, 42, 224, 2]`"] + #[doc = "Container type for all input parameters for the `MAX_MESSAGE_BODY_BYTES` function with signature `MAX_MESSAGE_BODY_BYTES()` and selector `[82, 42, 224, 2]`"] #[derive( Clone, Debug, @@ -522,7 +522,7 @@ pub mod home_mod { )] #[ethcall(name = "MAX_MESSAGE_BODY_BYTES", abi = "MAX_MESSAGE_BODY_BYTES()")] pub struct MaxMessageBodyBytesCall; - #[doc = "Container type for all input parameters for the `VERSION`function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] + #[doc = "Container type for all input parameters for the `VERSION` function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] #[derive( Clone, Debug, @@ -534,7 +534,7 @@ pub mod home_mod { )] #[ethcall(name = "VERSION", abi = "VERSION()")] pub struct VersionCall; - #[doc = "Container type for all input parameters for the `committedRoot`function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] + #[doc = "Container type for all input parameters for the `committedRoot` function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] #[derive( Clone, Debug, @@ -546,7 +546,7 @@ pub mod home_mod { )] #[ethcall(name = "committedRoot", abi = "committedRoot()")] pub struct CommittedRootCall; - #[doc = "Container type for all input parameters for the `count`function with signature `count()` and selector `[6, 102, 26, 189]`"] + #[doc = "Container type for all input parameters for the `count` function with signature `count()` and selector `[6, 102, 26, 189]`"] #[derive( Clone, Debug, @@ -558,7 +558,7 @@ pub mod home_mod { )] #[ethcall(name = "count", abi = "count()")] pub struct CountCall; - #[doc = "Container type for all input parameters for the `dispatch`function with signature `dispatch(uint32,bytes32,bytes)` and selector `[250, 49, 222, 1]`"] + #[doc = "Container type for all input parameters for the `dispatch` function with signature `dispatch(uint32,bytes32,bytes)` and selector `[250, 49, 222, 1]`"] #[derive( Clone, Debug, @@ -574,7 +574,7 @@ pub mod home_mod { pub recipient_address: [u8; 32], pub message_body: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `doubleUpdate`function with signature `doubleUpdate(bytes32,bytes32[2],bytes,bytes)` and selector `[25, 217, 210, 26]`"] + #[doc = "Container type for all input parameters for the `doubleUpdate` function with signature `doubleUpdate(bytes32,bytes32[2],bytes,bytes)` and selector `[25, 217, 210, 26]`"] #[derive( Clone, Debug, @@ -594,7 +594,7 @@ pub mod home_mod { pub signature: ethers::core::types::Bytes, pub signature_2: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `homeDomainHash`function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] + #[doc = "Container type for all input parameters for the `homeDomainHash` function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] #[derive( Clone, Debug, @@ -606,7 +606,7 @@ pub mod home_mod { )] #[ethcall(name = "homeDomainHash", abi = "homeDomainHash()")] pub struct HomeDomainHashCall; - #[doc = "Container type for all input parameters for the `improperUpdate`function with signature `improperUpdate(bytes32,bytes32,bytes)` and selector `[142, 78, 48, 224]`"] + #[doc = "Container type for all input parameters for the `improperUpdate` function with signature `improperUpdate(bytes32,bytes32,bytes)` and selector `[142, 78, 48, 224]`"] #[derive( Clone, Debug, @@ -622,7 +622,7 @@ pub mod home_mod { pub new_root: [u8; 32], pub signature: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `initialize`function with signature `initialize(address)` and selector `[196, 214, 109, 232]`"] + #[doc = "Container type for all input parameters for the `initialize` function with signature `initialize(address)` and selector `[196, 214, 109, 232]`"] #[derive( Clone, Debug, @@ -636,7 +636,7 @@ pub mod home_mod { pub struct InitializeCall { pub updater_manager: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `localDomain`function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[doc = "Container type for all input parameters for the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] #[derive( Clone, Debug, @@ -648,7 +648,7 @@ pub mod home_mod { )] #[ethcall(name = "localDomain", abi = "localDomain()")] pub struct LocalDomainCall; - #[doc = "Container type for all input parameters for the `nonces`function with signature `nonces(uint32)` and selector `[185, 90, 32, 1]`"] + #[doc = "Container type for all input parameters for the `nonces` function with signature `nonces(uint32)` and selector `[185, 90, 32, 1]`"] #[derive( Clone, Debug, @@ -660,7 +660,7 @@ pub mod home_mod { )] #[ethcall(name = "nonces", abi = "nonces(uint32)")] pub struct NoncesCall(pub u32); - #[doc = "Container type for all input parameters for the `owner`function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[doc = "Container type for all input parameters for the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] #[derive( Clone, Debug, @@ -672,7 +672,7 @@ pub mod home_mod { )] #[ethcall(name = "owner", abi = "owner()")] pub struct OwnerCall; - #[doc = "Container type for all input parameters for the `queueContains`function with signature `queueContains(bytes32)` and selector `[43, 239, 40, 146]`"] + #[doc = "Container type for all input parameters for the `queueContains` function with signature `queueContains(bytes32)` and selector `[43, 239, 40, 146]`"] #[derive( Clone, Debug, @@ -686,7 +686,7 @@ pub mod home_mod { pub struct QueueContainsCall { pub item: [u8; 32], } - #[doc = "Container type for all input parameters for the `queueEnd`function with signature `queueEnd()` and selector `[246, 209, 97, 2]`"] + #[doc = "Container type for all input parameters for the `queueEnd` function with signature `queueEnd()` and selector `[246, 209, 97, 2]`"] #[derive( Clone, Debug, @@ -698,7 +698,7 @@ pub mod home_mod { )] #[ethcall(name = "queueEnd", abi = "queueEnd()")] pub struct QueueEndCall; - #[doc = "Container type for all input parameters for the `queueLength`function with signature `queueLength()` and selector `[171, 145, 199, 176]`"] + #[doc = "Container type for all input parameters for the `queueLength` function with signature `queueLength()` and selector `[171, 145, 199, 176]`"] #[derive( Clone, Debug, @@ -710,7 +710,7 @@ pub mod home_mod { )] #[ethcall(name = "queueLength", abi = "queueLength()")] pub struct QueueLengthCall; - #[doc = "Container type for all input parameters for the `renounceOwnership`function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] + #[doc = "Container type for all input parameters for the `renounceOwnership` function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] #[derive( Clone, Debug, @@ -722,7 +722,7 @@ pub mod home_mod { )] #[ethcall(name = "renounceOwnership", abi = "renounceOwnership()")] pub struct RenounceOwnershipCall; - #[doc = "Container type for all input parameters for the `root`function with signature `root()` and selector `[235, 240, 199, 23]`"] + #[doc = "Container type for all input parameters for the `root` function with signature `root()` and selector `[235, 240, 199, 23]`"] #[derive( Clone, Debug, @@ -734,7 +734,7 @@ pub mod home_mod { )] #[ethcall(name = "root", abi = "root()")] pub struct RootCall; - #[doc = "Container type for all input parameters for the `setUpdater`function with signature `setUpdater(address)` and selector `[157, 84, 244, 25]`"] + #[doc = "Container type for all input parameters for the `setUpdater` function with signature `setUpdater(address)` and selector `[157, 84, 244, 25]`"] #[derive( Clone, Debug, @@ -748,7 +748,7 @@ pub mod home_mod { pub struct SetUpdaterCall { pub updater: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `setUpdaterManager`function with signature `setUpdaterManager(address)` and selector `[151, 118, 18, 14]`"] + #[doc = "Container type for all input parameters for the `setUpdaterManager` function with signature `setUpdaterManager(address)` and selector `[151, 118, 18, 14]`"] #[derive( Clone, Debug, @@ -762,7 +762,7 @@ pub mod home_mod { pub struct SetUpdaterManagerCall { pub updater_manager: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `state`function with signature `state()` and selector `[193, 157, 147, 251]`"] + #[doc = "Container type for all input parameters for the `state` function with signature `state()` and selector `[193, 157, 147, 251]`"] #[derive( Clone, Debug, @@ -774,7 +774,7 @@ pub mod home_mod { )] #[ethcall(name = "state", abi = "state()")] pub struct StateCall; - #[doc = "Container type for all input parameters for the `suggestUpdate`function with signature `suggestUpdate()` and selector `[54, 225, 4, 222]`"] + #[doc = "Container type for all input parameters for the `suggestUpdate` function with signature `suggestUpdate()` and selector `[54, 225, 4, 222]`"] #[derive( Clone, Debug, @@ -786,7 +786,7 @@ pub mod home_mod { )] #[ethcall(name = "suggestUpdate", abi = "suggestUpdate()")] pub struct SuggestUpdateCall; - #[doc = "Container type for all input parameters for the `transferOwnership`function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] + #[doc = "Container type for all input parameters for the `transferOwnership` function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] #[derive( Clone, Debug, @@ -800,7 +800,7 @@ pub mod home_mod { pub struct TransferOwnershipCall { pub new_owner: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `tree`function with signature `tree()` and selector `[253, 84, 178, 40]`"] + #[doc = "Container type for all input parameters for the `tree` function with signature `tree()` and selector `[253, 84, 178, 40]`"] #[derive( Clone, Debug, @@ -812,7 +812,7 @@ pub mod home_mod { )] #[ethcall(name = "tree", abi = "tree()")] pub struct TreeCall; - #[doc = "Container type for all input parameters for the `update`function with signature `update(bytes32,bytes32,bytes)` and selector `[179, 28, 1, 251]`"] + #[doc = "Container type for all input parameters for the `update` function with signature `update(bytes32,bytes32,bytes)` and selector `[179, 28, 1, 251]`"] #[derive( Clone, Debug, @@ -828,7 +828,7 @@ pub mod home_mod { pub new_root: [u8; 32], pub signature: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `updater`function with signature `updater()` and selector `[223, 3, 76, 208]`"] + #[doc = "Container type for all input parameters for the `updater` function with signature `updater()` and selector `[223, 3, 76, 208]`"] #[derive( Clone, Debug, @@ -840,7 +840,7 @@ pub mod home_mod { )] #[ethcall(name = "updater", abi = "updater()")] pub struct UpdaterCall; - #[doc = "Container type for all input parameters for the `updaterManager`function with signature `updaterManager()` and selector `[157, 246, 200, 225]`"] + #[doc = "Container type for all input parameters for the `updaterManager` function with signature `updaterManager()` and selector `[157, 246, 200, 225]`"] #[derive( Clone, Debug, @@ -1201,4 +1201,207 @@ pub mod home_mod { HomeCalls::UpdaterManager(var) } } + #[doc = "Container type for all return fields from the `MAX_MESSAGE_BODY_BYTES` function with signature `MAX_MESSAGE_BODY_BYTES()` and selector `[82, 42, 224, 2]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct MaxMessageBodyBytesReturn(pub ethers::core::types::U256); + #[doc = "Container type for all return fields from the `VERSION` function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct VersionReturn(pub u8); + #[doc = "Container type for all return fields from the `committedRoot` function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct CommittedRootReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `count` function with signature `count()` and selector `[6, 102, 26, 189]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct CountReturn(pub ethers::core::types::U256); + #[doc = "Container type for all return fields from the `homeDomainHash` function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct HomeDomainHashReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `improperUpdate` function with signature `improperUpdate(bytes32,bytes32,bytes)` and selector `[142, 78, 48, 224]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct ImproperUpdateReturn(pub bool); + #[doc = "Container type for all return fields from the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LocalDomainReturn(pub u32); + #[doc = "Container type for all return fields from the `nonces` function with signature `nonces(uint32)` and selector `[185, 90, 32, 1]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct NoncesReturn(pub u32); + #[doc = "Container type for all return fields from the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct OwnerReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `queueContains` function with signature `queueContains(bytes32)` and selector `[43, 239, 40, 146]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct QueueContainsReturn(pub bool); + #[doc = "Container type for all return fields from the `queueEnd` function with signature `queueEnd()` and selector `[246, 209, 97, 2]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct QueueEndReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `queueLength` function with signature `queueLength()` and selector `[171, 145, 199, 176]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct QueueLengthReturn(pub ethers::core::types::U256); + #[doc = "Container type for all return fields from the `root` function with signature `root()` and selector `[235, 240, 199, 23]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct RootReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `state` function with signature `state()` and selector `[193, 157, 147, 251]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct StateReturn(pub u8); + #[doc = "Container type for all return fields from the `suggestUpdate` function with signature `suggestUpdate()` and selector `[54, 225, 4, 222]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct SuggestUpdateReturn { + pub committed_root: [u8; 32], + pub new: [u8; 32], + } + #[doc = "Container type for all return fields from the `tree` function with signature `tree()` and selector `[253, 84, 178, 40]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct TreeReturn { + pub count: ethers::core::types::U256, + } + #[doc = "Container type for all return fields from the `updater` function with signature `updater()` and selector `[223, 3, 76, 208]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct UpdaterReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `updaterManager` function with signature `updaterManager()` and selector `[157, 246, 200, 225]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct UpdaterManagerReturn(pub ethers::core::types::Address); } diff --git a/chains/nomad-ethereum/src/bindings/replica.rs b/chains/nomad-ethereum/src/bindings/replica.rs index ea0b9767..fe5d6103 100644 --- a/chains/nomad-ethereum/src/bindings/replica.rs +++ b/chains/nomad-ethereum/src/bindings/replica.rs @@ -18,7 +18,7 @@ pub mod replica_mod { use std::sync::Arc; pub static REPLICA_ABI: ethers::contract::Lazy = ethers::contract::Lazy::new(|| { - serde_json :: from_str ("[\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_localDomain\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"oldUpdater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"newUpdater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdater\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"messageHash\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bool\",\n \"name\": \"success\",\n \"type\": \"bool\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes\",\n \"name\": \"returnData\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Process\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"root\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"previousConfirmAt\",\n \"type\": \"uint256\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"newConfirmAt\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"SetConfirmation\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"timeout\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"SetOptimisticTimeout\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"homeDomain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Update\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_NONE\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_PROCESSED\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_PROVEN\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"VERSION\",\n \"outputs\": [\n {\n \"internalType\": \"uint8\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_root\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"acceptableRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"committedRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"confirmAt\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"homeDomainHash\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_remoteDomain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_optimisticSeconds\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"initialize\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"messages\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"optimisticSeconds\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes\",\n \"name\": \"_message\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"process\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"_success\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_leaf\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32[32]\",\n \"name\": \"_proof\",\n \"type\": \"bytes32[32]\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_index\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"prove\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes\",\n \"name\": \"_message\",\n \"type\": \"bytes\"\n },\n {\n \"internalType\": \"bytes32[32]\",\n \"name\": \"_proof\",\n \"type\": \"bytes32[32]\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_index\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"proveAndProcess\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"remoteDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_root\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_confirmAt\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"setConfirmation\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"_optimisticSeconds\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"setOptimisticTimeout\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdater\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"state\",\n \"outputs\": [\n {\n \"internalType\": \"enum NomadBase.States\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"update\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updater\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") + ethers :: core :: utils :: __serde_json :: from_str ("[\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_localDomain\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"oldUpdater\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"newUpdater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"NewUpdater\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"messageHash\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bool\",\n \"name\": \"success\",\n \"type\": \"bool\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes\",\n \"name\": \"returnData\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Process\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"root\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"previousConfirmAt\",\n \"type\": \"uint256\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"newConfirmAt\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"SetConfirmation\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": false,\n \"internalType\": \"uint256\",\n \"name\": \"timeout\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"SetOptimisticTimeout\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"homeDomain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"bytes32\",\n \"name\": \"newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bytes\",\n \"name\": \"signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"Update\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_NONE\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_PROCESSED\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"LEGACY_STATUS_PROVEN\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"VERSION\",\n \"outputs\": [\n {\n \"internalType\": \"uint8\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_root\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"acceptableRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"committedRoot\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"confirmAt\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"homeDomainHash\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_remoteDomain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_committedRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_optimisticSeconds\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"initialize\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"name\": \"messages\",\n \"outputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"\",\n \"type\": \"bytes32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"optimisticSeconds\",\n \"outputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"\",\n \"type\": \"uint256\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes\",\n \"name\": \"_message\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"process\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"_success\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_leaf\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32[32]\",\n \"name\": \"_proof\",\n \"type\": \"bytes32[32]\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_index\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"prove\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes\",\n \"name\": \"_message\",\n \"type\": \"bytes\"\n },\n {\n \"internalType\": \"bytes32[32]\",\n \"name\": \"_proof\",\n \"type\": \"bytes32[32]\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_index\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"proveAndProcess\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"remoteDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_root\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"uint256\",\n \"name\": \"_confirmAt\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"setConfirmation\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint256\",\n \"name\": \"_optimisticSeconds\",\n \"type\": \"uint256\"\n }\n ],\n \"name\": \"setOptimisticTimeout\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_updater\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setUpdater\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"state\",\n \"outputs\": [\n {\n \"internalType\": \"enum NomadBase.States\",\n \"name\": \"\",\n \"type\": \"uint8\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_oldRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_newRoot\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"update\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"updater\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") }); pub struct Replica(ethers::contract::Contract); impl Clone for Replica { @@ -447,7 +447,7 @@ pub mod replica_mod { } } } - #[doc = "Container type for all input parameters for the `LEGACY_STATUS_NONE`function with signature `LEGACY_STATUS_NONE()` and selector `[24, 129, 11, 137]`"] + #[doc = "Container type for all input parameters for the `LEGACY_STATUS_NONE` function with signature `LEGACY_STATUS_NONE()` and selector `[24, 129, 11, 137]`"] #[derive( Clone, Debug, @@ -459,7 +459,7 @@ pub mod replica_mod { )] #[ethcall(name = "LEGACY_STATUS_NONE", abi = "LEGACY_STATUS_NONE()")] pub struct LegacyStatusNoneCall; - #[doc = "Container type for all input parameters for the `LEGACY_STATUS_PROCESSED`function with signature `LEGACY_STATUS_PROCESSED()` and selector `[177, 227, 251, 140]`"] + #[doc = "Container type for all input parameters for the `LEGACY_STATUS_PROCESSED` function with signature `LEGACY_STATUS_PROCESSED()` and selector `[177, 227, 251, 140]`"] #[derive( Clone, Debug, @@ -471,7 +471,7 @@ pub mod replica_mod { )] #[ethcall(name = "LEGACY_STATUS_PROCESSED", abi = "LEGACY_STATUS_PROCESSED()")] pub struct LegacyStatusProcessedCall; - #[doc = "Container type for all input parameters for the `LEGACY_STATUS_PROVEN`function with signature `LEGACY_STATUS_PROVEN()` and selector `[46, 123, 179, 44]`"] + #[doc = "Container type for all input parameters for the `LEGACY_STATUS_PROVEN` function with signature `LEGACY_STATUS_PROVEN()` and selector `[46, 123, 179, 44]`"] #[derive( Clone, Debug, @@ -483,7 +483,7 @@ pub mod replica_mod { )] #[ethcall(name = "LEGACY_STATUS_PROVEN", abi = "LEGACY_STATUS_PROVEN()")] pub struct LegacyStatusProvenCall; - #[doc = "Container type for all input parameters for the `VERSION`function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] + #[doc = "Container type for all input parameters for the `VERSION` function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] #[derive( Clone, Debug, @@ -495,7 +495,7 @@ pub mod replica_mod { )] #[ethcall(name = "VERSION", abi = "VERSION()")] pub struct VersionCall; - #[doc = "Container type for all input parameters for the `acceptableRoot`function with signature `acceptableRoot(bytes32)` and selector `[163, 248, 29, 104]`"] + #[doc = "Container type for all input parameters for the `acceptableRoot` function with signature `acceptableRoot(bytes32)` and selector `[163, 248, 29, 104]`"] #[derive( Clone, Debug, @@ -509,7 +509,7 @@ pub mod replica_mod { pub struct AcceptableRootCall { pub root: [u8; 32], } - #[doc = "Container type for all input parameters for the `committedRoot`function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] + #[doc = "Container type for all input parameters for the `committedRoot` function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] #[derive( Clone, Debug, @@ -521,7 +521,7 @@ pub mod replica_mod { )] #[ethcall(name = "committedRoot", abi = "committedRoot()")] pub struct CommittedRootCall; - #[doc = "Container type for all input parameters for the `confirmAt`function with signature `confirmAt(bytes32)` and selector `[113, 191, 183, 184]`"] + #[doc = "Container type for all input parameters for the `confirmAt` function with signature `confirmAt(bytes32)` and selector `[113, 191, 183, 184]`"] #[derive( Clone, Debug, @@ -533,7 +533,7 @@ pub mod replica_mod { )] #[ethcall(name = "confirmAt", abi = "confirmAt(bytes32)")] pub struct ConfirmAtCall(pub [u8; 32]); - #[doc = "Container type for all input parameters for the `homeDomainHash`function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] + #[doc = "Container type for all input parameters for the `homeDomainHash` function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] #[derive( Clone, Debug, @@ -545,7 +545,7 @@ pub mod replica_mod { )] #[ethcall(name = "homeDomainHash", abi = "homeDomainHash()")] pub struct HomeDomainHashCall; - #[doc = "Container type for all input parameters for the `initialize`function with signature `initialize(uint32,address,bytes32,uint256)` and selector `[231, 231, 167, 183]`"] + #[doc = "Container type for all input parameters for the `initialize` function with signature `initialize(uint32,address,bytes32,uint256)` and selector `[231, 231, 167, 183]`"] #[derive( Clone, Debug, @@ -565,7 +565,7 @@ pub mod replica_mod { pub committed_root: [u8; 32], pub optimistic_seconds: ethers::core::types::U256, } - #[doc = "Container type for all input parameters for the `localDomain`function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[doc = "Container type for all input parameters for the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] #[derive( Clone, Debug, @@ -577,7 +577,7 @@ pub mod replica_mod { )] #[ethcall(name = "localDomain", abi = "localDomain()")] pub struct LocalDomainCall; - #[doc = "Container type for all input parameters for the `messages`function with signature `messages(bytes32)` and selector `[43, 189, 89, 202]`"] + #[doc = "Container type for all input parameters for the `messages` function with signature `messages(bytes32)` and selector `[43, 189, 89, 202]`"] #[derive( Clone, Debug, @@ -589,7 +589,7 @@ pub mod replica_mod { )] #[ethcall(name = "messages", abi = "messages(bytes32)")] pub struct MessagesCall(pub [u8; 32]); - #[doc = "Container type for all input parameters for the `optimisticSeconds`function with signature `optimisticSeconds()` and selector `[57, 153, 38, 104]`"] + #[doc = "Container type for all input parameters for the `optimisticSeconds` function with signature `optimisticSeconds()` and selector `[57, 153, 38, 104]`"] #[derive( Clone, Debug, @@ -601,7 +601,7 @@ pub mod replica_mod { )] #[ethcall(name = "optimisticSeconds", abi = "optimisticSeconds()")] pub struct OptimisticSecondsCall; - #[doc = "Container type for all input parameters for the `owner`function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[doc = "Container type for all input parameters for the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] #[derive( Clone, Debug, @@ -613,7 +613,7 @@ pub mod replica_mod { )] #[ethcall(name = "owner", abi = "owner()")] pub struct OwnerCall; - #[doc = "Container type for all input parameters for the `process`function with signature `process(bytes)` and selector `[146, 139, 196, 178]`"] + #[doc = "Container type for all input parameters for the `process` function with signature `process(bytes)` and selector `[146, 139, 196, 178]`"] #[derive( Clone, Debug, @@ -627,7 +627,7 @@ pub mod replica_mod { pub struct ProcessCall { pub message: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `prove`function with signature `prove(bytes32,bytes32[32],uint256)` and selector `[55, 29, 48, 113]`"] + #[doc = "Container type for all input parameters for the `prove` function with signature `prove(bytes32,bytes32[32],uint256)` and selector `[55, 29, 48, 113]`"] #[derive( Clone, Debug, @@ -643,7 +643,7 @@ pub mod replica_mod { pub proof: [[u8; 32]; 32usize], pub index: ethers::core::types::U256, } - #[doc = "Container type for all input parameters for the `proveAndProcess`function with signature `proveAndProcess(bytes,bytes32[32],uint256)` and selector `[97, 136, 175, 14]`"] + #[doc = "Container type for all input parameters for the `proveAndProcess` function with signature `proveAndProcess(bytes,bytes32[32],uint256)` and selector `[97, 136, 175, 14]`"] #[derive( Clone, Debug, @@ -662,7 +662,7 @@ pub mod replica_mod { pub proof: [[u8; 32]; 32usize], pub index: ethers::core::types::U256, } - #[doc = "Container type for all input parameters for the `remoteDomain`function with signature `remoteDomain()` and selector `[150, 22, 129, 220]`"] + #[doc = "Container type for all input parameters for the `remoteDomain` function with signature `remoteDomain()` and selector `[150, 22, 129, 220]`"] #[derive( Clone, Debug, @@ -674,7 +674,7 @@ pub mod replica_mod { )] #[ethcall(name = "remoteDomain", abi = "remoteDomain()")] pub struct RemoteDomainCall; - #[doc = "Container type for all input parameters for the `renounceOwnership`function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] + #[doc = "Container type for all input parameters for the `renounceOwnership` function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] #[derive( Clone, Debug, @@ -686,7 +686,7 @@ pub mod replica_mod { )] #[ethcall(name = "renounceOwnership", abi = "renounceOwnership()")] pub struct RenounceOwnershipCall; - #[doc = "Container type for all input parameters for the `setConfirmation`function with signature `setConfirmation(bytes32,uint256)` and selector `[8, 139, 94, 211]`"] + #[doc = "Container type for all input parameters for the `setConfirmation` function with signature `setConfirmation(bytes32,uint256)` and selector `[8, 139, 94, 211]`"] #[derive( Clone, Debug, @@ -701,7 +701,7 @@ pub mod replica_mod { pub root: [u8; 32], pub confirm_at: ethers::core::types::U256, } - #[doc = "Container type for all input parameters for the `setOptimisticTimeout`function with signature `setOptimisticTimeout(uint256)` and selector `[136, 91, 94, 45]`"] + #[doc = "Container type for all input parameters for the `setOptimisticTimeout` function with signature `setOptimisticTimeout(uint256)` and selector `[136, 91, 94, 45]`"] #[derive( Clone, Debug, @@ -715,7 +715,7 @@ pub mod replica_mod { pub struct SetOptimisticTimeoutCall { pub optimistic_seconds: ethers::core::types::U256, } - #[doc = "Container type for all input parameters for the `setUpdater`function with signature `setUpdater(address)` and selector `[157, 84, 244, 25]`"] + #[doc = "Container type for all input parameters for the `setUpdater` function with signature `setUpdater(address)` and selector `[157, 84, 244, 25]`"] #[derive( Clone, Debug, @@ -729,7 +729,7 @@ pub mod replica_mod { pub struct SetUpdaterCall { pub updater: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `state`function with signature `state()` and selector `[193, 157, 147, 251]`"] + #[doc = "Container type for all input parameters for the `state` function with signature `state()` and selector `[193, 157, 147, 251]`"] #[derive( Clone, Debug, @@ -741,7 +741,7 @@ pub mod replica_mod { )] #[ethcall(name = "state", abi = "state()")] pub struct StateCall; - #[doc = "Container type for all input parameters for the `transferOwnership`function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] + #[doc = "Container type for all input parameters for the `transferOwnership` function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] #[derive( Clone, Debug, @@ -755,7 +755,7 @@ pub mod replica_mod { pub struct TransferOwnershipCall { pub new_owner: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `update`function with signature `update(bytes32,bytes32,bytes)` and selector `[179, 28, 1, 251]`"] + #[doc = "Container type for all input parameters for the `update` function with signature `update(bytes32,bytes32,bytes)` and selector `[179, 28, 1, 251]`"] #[derive( Clone, Debug, @@ -771,7 +771,7 @@ pub mod replica_mod { pub new_root: [u8; 32], pub signature: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `updater`function with signature `updater()` and selector `[223, 3, 76, 208]`"] + #[doc = "Container type for all input parameters for the `updater` function with signature `updater()` and selector `[223, 3, 76, 208]`"] #[derive( Clone, Debug, @@ -1124,4 +1124,193 @@ pub mod replica_mod { ReplicaCalls::Updater(var) } } + #[doc = "Container type for all return fields from the `LEGACY_STATUS_NONE` function with signature `LEGACY_STATUS_NONE()` and selector `[24, 129, 11, 137]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LegacyStatusNoneReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `LEGACY_STATUS_PROCESSED` function with signature `LEGACY_STATUS_PROCESSED()` and selector `[177, 227, 251, 140]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LegacyStatusProcessedReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `LEGACY_STATUS_PROVEN` function with signature `LEGACY_STATUS_PROVEN()` and selector `[46, 123, 179, 44]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LegacyStatusProvenReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `VERSION` function with signature `VERSION()` and selector `[255, 161, 173, 116]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct VersionReturn(pub u8); + #[doc = "Container type for all return fields from the `acceptableRoot` function with signature `acceptableRoot(bytes32)` and selector `[163, 248, 29, 104]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct AcceptableRootReturn(pub bool); + #[doc = "Container type for all return fields from the `committedRoot` function with signature `committedRoot()` and selector `[103, 166, 119, 29]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct CommittedRootReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `confirmAt` function with signature `confirmAt(bytes32)` and selector `[113, 191, 183, 184]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct ConfirmAtReturn(pub ethers::core::types::U256); + #[doc = "Container type for all return fields from the `homeDomainHash` function with signature `homeDomainHash()` and selector `[69, 99, 11, 26]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct HomeDomainHashReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LocalDomainReturn(pub u32); + #[doc = "Container type for all return fields from the `messages` function with signature `messages(bytes32)` and selector `[43, 189, 89, 202]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct MessagesReturn(pub [u8; 32]); + #[doc = "Container type for all return fields from the `optimisticSeconds` function with signature `optimisticSeconds()` and selector `[57, 153, 38, 104]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct OptimisticSecondsReturn(pub ethers::core::types::U256); + #[doc = "Container type for all return fields from the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct OwnerReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `process` function with signature `process(bytes)` and selector `[146, 139, 196, 178]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct ProcessReturn { + pub success: bool, + } + #[doc = "Container type for all return fields from the `prove` function with signature `prove(bytes32,bytes32[32],uint256)` and selector `[55, 29, 48, 113]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct ProveReturn(pub bool); + #[doc = "Container type for all return fields from the `remoteDomain` function with signature `remoteDomain()` and selector `[150, 22, 129, 220]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct RemoteDomainReturn(pub u32); + #[doc = "Container type for all return fields from the `state` function with signature `state()` and selector `[193, 157, 147, 251]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct StateReturn(pub u8); + #[doc = "Container type for all return fields from the `updater` function with signature `updater()` and selector `[223, 3, 76, 208]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct UpdaterReturn(pub ethers::core::types::Address); } diff --git a/chains/nomad-ethereum/src/bindings/xappconnectionmanager.rs b/chains/nomad-ethereum/src/bindings/xappconnectionmanager.rs index c664ea8a..c107368a 100644 --- a/chains/nomad-ethereum/src/bindings/xappconnectionmanager.rs +++ b/chains/nomad-ethereum/src/bindings/xappconnectionmanager.rs @@ -18,7 +18,7 @@ pub mod xappconnectionmanager_mod { use std::sync::Arc; pub static XAPPCONNECTIONMANAGER_ABI: ethers::contract::Lazy = ethers::contract::Lazy::new(|| { - serde_json :: from_str ("[\n {\n \"inputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ReplicaEnrolled\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ReplicaUnenrolled\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"watcher\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bool\",\n \"name\": \"access\",\n \"type\": \"bool\"\n }\n ],\n \"name\": \"WatcherPermissionSet\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"domainToReplica\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"home\",\n \"outputs\": [\n {\n \"internalType\": \"contract Home\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"isReplica\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"ownerEnrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ownerUnenrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"replicaToDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_home\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setHome\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_watcher\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bool\",\n \"name\": \"_access\",\n \"type\": \"bool\"\n }\n ],\n \"name\": \"setWatcherPermission\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_updater\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"unenrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_watcher\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"watcherPermission\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") + ethers :: core :: utils :: __serde_json :: from_str ("[\n {\n \"inputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"constructor\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"previousOwner\",\n \"type\": \"address\"\n },\n {\n \"indexed\": true,\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"OwnershipTransferred\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ReplicaEnrolled\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ReplicaUnenrolled\",\n \"type\": \"event\"\n },\n {\n \"anonymous\": false,\n \"inputs\": [\n {\n \"indexed\": true,\n \"internalType\": \"uint32\",\n \"name\": \"domain\",\n \"type\": \"uint32\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"address\",\n \"name\": \"watcher\",\n \"type\": \"address\"\n },\n {\n \"indexed\": false,\n \"internalType\": \"bool\",\n \"name\": \"access\",\n \"type\": \"bool\"\n }\n ],\n \"name\": \"WatcherPermissionSet\",\n \"type\": \"event\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"domainToReplica\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"home\",\n \"outputs\": [\n {\n \"internalType\": \"contract Home\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"isReplica\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"localDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"owner\",\n \"outputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"ownerEnrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_replica\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"ownerUnenrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [],\n \"name\": \"renounceOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"replicaToDomain\",\n \"outputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"\",\n \"type\": \"uint32\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_home\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"setHome\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_watcher\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bool\",\n \"name\": \"_access\",\n \"type\": \"bool\"\n }\n ],\n \"name\": \"setWatcherPermission\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"newOwner\",\n \"type\": \"address\"\n }\n ],\n \"name\": \"transferOwnership\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n },\n {\n \"internalType\": \"bytes32\",\n \"name\": \"_updater\",\n \"type\": \"bytes32\"\n },\n {\n \"internalType\": \"bytes\",\n \"name\": \"_signature\",\n \"type\": \"bytes\"\n }\n ],\n \"name\": \"unenrollReplica\",\n \"outputs\": [],\n \"stateMutability\": \"nonpayable\",\n \"type\": \"function\"\n },\n {\n \"inputs\": [\n {\n \"internalType\": \"address\",\n \"name\": \"_watcher\",\n \"type\": \"address\"\n },\n {\n \"internalType\": \"uint32\",\n \"name\": \"_domain\",\n \"type\": \"uint32\"\n }\n ],\n \"name\": \"watcherPermission\",\n \"outputs\": [\n {\n \"internalType\": \"bool\",\n \"name\": \"\",\n \"type\": \"bool\"\n }\n ],\n \"stateMutability\": \"view\",\n \"type\": \"function\"\n }\n]\n") . expect ("invalid abi") }); pub struct XAppConnectionManager(ethers::contract::Contract); impl Clone for XAppConnectionManager { @@ -325,7 +325,7 @@ pub mod xappconnectionmanager_mod { } } } - #[doc = "Container type for all input parameters for the `domainToReplica`function with signature `domainToReplica(uint32)` and selector `[185, 207, 241, 98]`"] + #[doc = "Container type for all input parameters for the `domainToReplica` function with signature `domainToReplica(uint32)` and selector `[185, 207, 241, 98]`"] #[derive( Clone, Debug, @@ -337,7 +337,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "domainToReplica", abi = "domainToReplica(uint32)")] pub struct DomainToReplicaCall(pub u32); - #[doc = "Container type for all input parameters for the `home`function with signature `home()` and selector `[159, 169, 47, 157]`"] + #[doc = "Container type for all input parameters for the `home` function with signature `home()` and selector `[159, 169, 47, 157]`"] #[derive( Clone, Debug, @@ -349,7 +349,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "home", abi = "home()")] pub struct HomeCall; - #[doc = "Container type for all input parameters for the `isReplica`function with signature `isReplica(address)` and selector `[81, 144, 188, 83]`"] + #[doc = "Container type for all input parameters for the `isReplica` function with signature `isReplica(address)` and selector `[81, 144, 188, 83]`"] #[derive( Clone, Debug, @@ -363,7 +363,7 @@ pub mod xappconnectionmanager_mod { pub struct IsReplicaCall { pub replica: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `localDomain`function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[doc = "Container type for all input parameters for the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] #[derive( Clone, Debug, @@ -375,7 +375,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "localDomain", abi = "localDomain()")] pub struct LocalDomainCall; - #[doc = "Container type for all input parameters for the `owner`function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[doc = "Container type for all input parameters for the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] #[derive( Clone, Debug, @@ -387,7 +387,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "owner", abi = "owner()")] pub struct OwnerCall; - #[doc = "Container type for all input parameters for the `ownerEnrollReplica`function with signature `ownerEnrollReplica(address,uint32)` and selector `[243, 31, 174, 251]`"] + #[doc = "Container type for all input parameters for the `ownerEnrollReplica` function with signature `ownerEnrollReplica(address,uint32)` and selector `[243, 31, 174, 251]`"] #[derive( Clone, Debug, @@ -405,7 +405,7 @@ pub mod xappconnectionmanager_mod { pub replica: ethers::core::types::Address, pub domain: u32, } - #[doc = "Container type for all input parameters for the `ownerUnenrollReplica`function with signature `ownerUnenrollReplica(address)` and selector `[143, 93, 144, 224]`"] + #[doc = "Container type for all input parameters for the `ownerUnenrollReplica` function with signature `ownerUnenrollReplica(address)` and selector `[143, 93, 144, 224]`"] #[derive( Clone, Debug, @@ -419,7 +419,7 @@ pub mod xappconnectionmanager_mod { pub struct OwnerUnenrollReplicaCall { pub replica: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `renounceOwnership`function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] + #[doc = "Container type for all input parameters for the `renounceOwnership` function with signature `renounceOwnership()` and selector `[113, 80, 24, 166]`"] #[derive( Clone, Debug, @@ -431,7 +431,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "renounceOwnership", abi = "renounceOwnership()")] pub struct RenounceOwnershipCall; - #[doc = "Container type for all input parameters for the `replicaToDomain`function with signature `replicaToDomain(address)` and selector `[95, 139, 29, 186]`"] + #[doc = "Container type for all input parameters for the `replicaToDomain` function with signature `replicaToDomain(address)` and selector `[95, 139, 29, 186]`"] #[derive( Clone, Debug, @@ -443,7 +443,7 @@ pub mod xappconnectionmanager_mod { )] #[ethcall(name = "replicaToDomain", abi = "replicaToDomain(address)")] pub struct ReplicaToDomainCall(pub ethers::core::types::Address); - #[doc = "Container type for all input parameters for the `setHome`function with signature `setHome(address)` and selector `[110, 240, 243, 127]`"] + #[doc = "Container type for all input parameters for the `setHome` function with signature `setHome(address)` and selector `[110, 240, 243, 127]`"] #[derive( Clone, Debug, @@ -457,7 +457,7 @@ pub mod xappconnectionmanager_mod { pub struct SetHomeCall { pub home: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `setWatcherPermission`function with signature `setWatcherPermission(address,uint32,bool)` and selector `[145, 108, 52, 112]`"] + #[doc = "Container type for all input parameters for the `setWatcherPermission` function with signature `setWatcherPermission(address,uint32,bool)` and selector `[145, 108, 52, 112]`"] #[derive( Clone, Debug, @@ -476,7 +476,7 @@ pub mod xappconnectionmanager_mod { pub domain: u32, pub access: bool, } - #[doc = "Container type for all input parameters for the `transferOwnership`function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] + #[doc = "Container type for all input parameters for the `transferOwnership` function with signature `transferOwnership(address)` and selector `[242, 253, 227, 139]`"] #[derive( Clone, Debug, @@ -490,7 +490,7 @@ pub mod xappconnectionmanager_mod { pub struct TransferOwnershipCall { pub new_owner: ethers::core::types::Address, } - #[doc = "Container type for all input parameters for the `unenrollReplica`function with signature `unenrollReplica(uint32,bytes32,bytes)` and selector `[224, 231, 169, 19]`"] + #[doc = "Container type for all input parameters for the `unenrollReplica` function with signature `unenrollReplica(uint32,bytes32,bytes)` and selector `[224, 231, 169, 19]`"] #[derive( Clone, Debug, @@ -509,7 +509,7 @@ pub mod xappconnectionmanager_mod { pub updater: [u8; 32], pub signature: ethers::core::types::Bytes, } - #[doc = "Container type for all input parameters for the `watcherPermission`function with signature `watcherPermission(address,uint32)` and selector `[66, 126, 190, 245]`"] + #[doc = "Container type for all input parameters for the `watcherPermission` function with signature `watcherPermission(address,uint32)` and selector `[66, 126, 190, 245]`"] #[derive( Clone, Debug, @@ -723,4 +723,81 @@ pub mod xappconnectionmanager_mod { XAppConnectionManagerCalls::WatcherPermission(var) } } + #[doc = "Container type for all return fields from the `domainToReplica` function with signature `domainToReplica(uint32)` and selector `[185, 207, 241, 98]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct DomainToReplicaReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `home` function with signature `home()` and selector `[159, 169, 47, 157]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct HomeReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `isReplica` function with signature `isReplica(address)` and selector `[81, 144, 188, 83]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct IsReplicaReturn(pub bool); + #[doc = "Container type for all return fields from the `localDomain` function with signature `localDomain()` and selector `[141, 54, 56, 244]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct LocalDomainReturn(pub u32); + #[doc = "Container type for all return fields from the `owner` function with signature `owner()` and selector `[141, 165, 203, 91]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct OwnerReturn(pub ethers::core::types::Address); + #[doc = "Container type for all return fields from the `replicaToDomain` function with signature `replicaToDomain(address)` and selector `[95, 139, 29, 186]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct ReplicaToDomainReturn(pub u32); + #[doc = "Container type for all return fields from the `watcherPermission` function with signature `watcherPermission(address,uint32)` and selector `[66, 126, 190, 245]`"] + #[derive( + Clone, + Debug, + Default, + Eq, + PartialEq, + ethers :: contract :: EthAbiType, + ethers :: contract :: EthAbiCodec, + )] + pub struct WatcherPermissionReturn(pub bool); } diff --git a/configuration/src/builtin.rs b/configuration/src/builtin.rs index 46d84224..f0e716bf 100644 --- a/configuration/src/builtin.rs +++ b/configuration/src/builtin.rs @@ -28,7 +28,7 @@ fn deser(name: &str, json: &str) -> NomadConfig { } /// Get a built-in config object -pub fn get_builtin(name: &str) -> Option<&NomadConfig> { +pub fn get_builtin(name: &str) -> Option<&'static NomadConfig> { let builtins = BUILTINS.get_or_init(|| { let mut map: HashMap<_, _> = Default::default(); From 3e5d0ed48d2e670e0f31c7da7ac03fdba4031acc Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Jul 2022 18:59:42 -0700 Subject: [PATCH 03/80] feat: full init logic --- agents/monitor/Cargo.toml | 1 + agents/monitor/src/init.rs | 101 +++++++++++++++++----- agents/monitor/src/main.rs | 8 +- chains/nomad-ethereum/src/bindings/mod.rs | 8 +- chains/nomad-ethereum/src/lib.rs | 2 +- 5 files changed, 91 insertions(+), 29 deletions(-) diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index 37bbada9..f468d6b5 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -16,3 +16,4 @@ nomad-core = { path = "../../nomad-core", default-features = false } nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../../configuration" } tracing-subscriber = "0.2.15" eyre = "0.6.8" +nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index ff5c96eb..10113ab7 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -2,10 +2,14 @@ use std::{collections::HashMap, sync::Arc}; use ethers::{ middleware::TimeLag, - prelude::{Http, Provider}, + prelude::{Http, Provider as EthersProvider}, }; -use nomad_xyz_configuration::{get_builtin, NomadConfig}; +use nomad_ethereum::bindings::{home::Home, replica::Replica}; +use nomad_xyz_configuration::{contracts::CoreContracts, get_builtin, NomadConfig}; + +pub(crate) type Provider = TimeLag>; +pub(crate) type ArcProvider = Arc; pub(crate) fn config_from_file() -> Option { std::env::var("CONFIG_PATH") @@ -20,10 +24,10 @@ pub(crate) fn config_from_env() -> Option { .map(ToOwned::to_owned) } -pub(crate) fn config() -> NomadConfig { +pub(crate) fn config() -> eyre::Result { config_from_file() .or_else(config_from_env) - .expect("could not load config from file or env") + .ok_or_else(|| eyre::eyre!("Unable to load config from file or env")) } pub(crate) fn init_tracing() { @@ -37,10 +41,7 @@ pub(crate) fn rpc_from_env(network: &str) -> Option { std::env::var(format!("{}_CONNECTION_URL", network.to_uppercase())).ok() } -pub(crate) fn provider_for( - config: &NomadConfig, - network: &str, -) -> eyre::Result>>> { +pub(crate) fn provider_for(config: &NomadConfig, network: &str) -> eyre::Result { tracing::info!(network, "Instantiating provider"); let url = rpc_from_env(network).or_else(|| { @@ -55,7 +56,7 @@ pub(crate) fn provider_for( ); let url = url.unwrap(); - let provider = Provider::::try_from(&url)?; + let provider = EthersProvider::::try_from(&url)?; let timelag = config .protocol() @@ -69,17 +70,75 @@ pub(crate) fn provider_for( Ok(TimeLag::new(provider, timelag).into()) } -pub(crate) fn init_providers( - config: &NomadConfig, -) -> HashMap>>> { - config - .networks - .iter() - .map(|network| { - ( - network.clone(), - provider_for(config, network).expect("unable to make provider"), - ) +pub(crate) fn monitor() -> eyre::Result { + Monitor::from_config(config()?) +} + +pub(crate) struct Network { + name: String, + provider: ArcProvider, + home: Home, + replicas: HashMap>, +} + +impl Network { + fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { + let name = network.to_owned(); + let provider = provider_for(config, network)?; + + let CoreContracts::Evm(core) = config.core().get(network).expect("invalid config"); + + let home = Home::new(core.home.proxy.as_ethereum_address()?, provider.clone()); + + let replicas = core + .replicas + .iter() + .map(|(k, v)| { + let replica = Replica::new( + v.proxy.as_ethereum_address().expect("invalid addr"), + provider.clone(), + ); + (k.clone(), replica) + }) + .collect(); + + Ok(Network { + name, + provider, + home, + replicas, }) - .collect() + } + + pub(crate) fn name(&self) -> &str { + self.name.as_ref() + } + + pub(crate) fn provider(&self) -> &TimeLag> { + self.provider.as_ref() + } + + pub(crate) fn home(&self) -> &Home { + &self.home + } + + pub(crate) fn replicas(&self) -> &HashMap> { + &self.replicas + } +} + +pub(crate) struct Monitor { + config: NomadConfig, + networks: HashMap, +} + +impl Monitor { + pub(crate) fn from_config(config: NomadConfig) -> eyre::Result { + let mut networks = HashMap::new(); + for network in config.networks.iter() { + networks.insert(network.to_owned(), Network::from_config(&config, network)?); + } + + Ok(Monitor { config, networks }) + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 2882f18d..ccac7cf2 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,3 +1,4 @@ +use init::Monitor; use tracing::info_span; mod between; @@ -5,17 +6,18 @@ mod between; mod init; #[tokio::main] -async fn main() { +async fn main() -> eyre::Result<()> { init::init_tracing(); { let span = info_span!("MonitorBootup"); let _span = span.enter(); - let config = init::config(); - let providers = init::init_providers(&config); + + let _monitor = init::monitor()?; tracing::info!("setup complete!") } + Ok(()) } /// Simple event trait diff --git a/chains/nomad-ethereum/src/bindings/mod.rs b/chains/nomad-ethereum/src/bindings/mod.rs index cd21f412..8eb48dc6 100644 --- a/chains/nomad-ethereum/src/bindings/mod.rs +++ b/chains/nomad-ethereum/src/bindings/mod.rs @@ -1,4 +1,4 @@ -#![allow(clippy::all)] -pub(crate) mod home; -pub(crate) mod replica; -pub(crate) mod xappconnectionmanager; +#![allow(clippy::all, missing_docs)] +pub mod home; +pub mod replica; +pub mod xappconnectionmanager; diff --git a/chains/nomad-ethereum/src/lib.rs b/chains/nomad-ethereum/src/lib.rs index 30b2ca82..3094e366 100644 --- a/chains/nomad-ethereum/src/lib.rs +++ b/chains/nomad-ethereum/src/lib.rs @@ -30,7 +30,7 @@ pub use submitter::*; /// Contract binding #[cfg(not(doctest))] -pub(crate) mod bindings; +pub mod bindings; /// Home abi #[cfg(not(doctest))] From 575cc4b7b21206b91f26506297d519573fa0e21c Mon Sep 17 00:00:00 2001 From: James Date: Sat, 16 Jul 2022 20:44:52 -0700 Subject: [PATCH 04/80] features: a lot fix: re-export bindings wat --- agents/monitor/src/between.rs | 10 +- agents/monitor/src/domain.rs | 170 ++++++++++++++++++++++++++++++++++ agents/monitor/src/init.rs | 82 +++++----------- agents/monitor/src/main.rs | 24 ++++- agents/monitor/src/metrics.rs | 55 +++++++++++ 5 files changed, 271 insertions(+), 70 deletions(-) create mode 100644 agents/monitor/src/domain.rs create mode 100644 agents/monitor/src/metrics.rs diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 6451ca28..a17bb9fd 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -6,8 +6,8 @@ use crate::NomadEvent; // Track time between events of the same kind pub struct BetweenEvents { - pub(crate) incoming: tokio::sync::mpsc::Receiver, - pub(crate) outgoing: tokio::sync::mpsc::Sender, + pub(crate) incoming: tokio::sync::mpsc::UnboundedReceiver, + pub(crate) outgoing: tokio::sync::mpsc::UnboundedSender, pub(crate) count: prometheus::IntCounter, pub(crate) wallclock_latency: prometheus::Histogram, pub(crate) timestamp_latency: prometheus::Histogram, @@ -20,8 +20,8 @@ where T: NomadEvent + 'static, { pub fn new( - incoming: tokio::sync::mpsc::Receiver, - outgoing: tokio::sync::mpsc::Sender, + incoming: tokio::sync::mpsc::UnboundedReceiver, + outgoing: tokio::sync::mpsc::UnboundedSender, count: prometheus::IntCounter, wallclock_latency: prometheus::Histogram, timestamp_latency: prometheus::Histogram, @@ -63,7 +63,7 @@ where wallclock_latency.observe_duration(); // send the next event out - if self.outgoing.send(incoming).await.is_err() { + if self.outgoing.send(incoming).is_err() { break; } diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs new file mode 100644 index 00000000..46705966 --- /dev/null +++ b/agents/monitor/src/domain.rs @@ -0,0 +1,170 @@ +use std::{collections::HashMap, sync::Arc}; + +use ethers::{ + contract::builders::Event, + middleware::TimeLag, + prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}, +}; + +use nomad_ethereum::bindings::{ + home::{DispatchFilter, Home, UpdateFilter as HomeUpdateFilter}, + replica::{ProcessFilter, Replica, UpdateFilter as ReplicaUpdateFilter}, +}; +use nomad_xyz_configuration::{contracts::CoreContracts, get_builtin, NomadConfig}; +use prometheus::{Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec}; +use tokio::{ + sync::mpsc::{self, UnboundedReceiver}, + task::JoinHandle, +}; + +use crate::{between::BetweenEvents, init::provider_for, ArcProvider, NomadEvent, Provider}; + +macro_rules! unwrap_event_stream_item { + ($event:ident, $net:ident, $name:literal) => {{ + match $event { + None => break, + Some(Err(error)) => { + tracing::error!(%error, net = AsRef::::as_ref(&$net), event = $name, "Stream ended"); + break; + } + Some(Ok(event)) => event, + } + }} +} + +#[derive(Debug)] +pub(crate) struct Domain { + pub(crate) name: String, + pub(crate) provider: ArcProvider, + pub(crate) home: Home, + pub(crate) replicas: HashMap>, +} + +impl Domain { + pub(crate) fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { + let name = network.to_owned(); + let provider = provider_for(config, network)?; + + let CoreContracts::Evm(core) = config.core().get(network).expect("invalid config"); + + let home = Home::new(core.home.proxy.as_ethereum_address()?, provider.clone()); + + let replicas = core + .replicas + .iter() + .map(|(k, v)| { + let replica = Replica::new( + v.proxy.as_ethereum_address().expect("invalid addr"), + provider.clone(), + ); + (k.clone(), replica) + }) + .collect(); + + Ok(Domain { + name, + provider, + home, + replicas, + }) + } + + pub(crate) fn name(&self) -> &str { + self.name.as_ref() + } + + pub(crate) fn provider(&self) -> &TimeLag> { + self.provider.as_ref() + } + + pub(crate) fn home(&self) -> &Home { + &self.home + } + + pub(crate) fn replicas(&self) -> &HashMap> { + &self.replicas + } + + pub(crate) fn update_filter(&self) -> Event { + self.home.update_filter() + } + + pub(crate) fn relay_filters(&self) -> HashMap<&str, Event> { + self.replicas + .iter() + .map(|(k, v)| (k.as_str(), v.update_filter())) + .collect() + } + + pub(crate) fn process_filters(&self) -> HashMap<&str, Event> { + self.replicas + .iter() + .map(|(k, v)| (k.as_str(), v.process_filter())) + .collect() + } + + pub(crate) fn dispatch_stream(&self) -> mpsc::UnboundedReceiver { + let home = self.home.clone(); + let (tx, rx) = mpsc::unbounded_channel(); + let name = self.name.clone(); + + tokio::spawn(async move { + let filter = home.dispatch_filter(); + let mut stream = filter + .stream() + .await + .expect("unable to get dispatch stream"); + loop { + let event = stream.next().await; + let event = unwrap_event_stream_item!(event, name, "dispatch"); + tx.send(event).unwrap(); + } + }); + + rx + } + + pub(crate) fn count( + &self, + incoming: mpsc::UnboundedReceiver, + count: IntCounter, + wallclock_latency: Histogram, + timestamp_latency: Histogram, + ) -> UnboundedReceiver + where + T: NomadEvent + 'static, + { + let (tx, rx) = mpsc::unbounded_channel(); + + let between = BetweenEvents::new( + incoming, + tx, + count, + wallclock_latency, + timestamp_latency, + self.name.clone(), + ); + + between.spawn(); + + rx + } + + // fn update_stream(&self) -> mpsc::UnboundedReceiver { + // let home = self.home.clone(); + // let (tx, rx) = mpsc::unbounded_channel(); + // let name = self.name.clone(); + + // tokio::spawn(async move { + // let filter = home.update_filter(); + // let mut stream = filter.stream().await.expect("unable to get update stream"); + // loop { + // let event = stream.next().await; + // let event = unwrap_event_stream_item!(event, name, "update"); + // tx.send(event).unwrap(); + // } + // }); + + // rx + // } +} diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 10113ab7..7d8dde27 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -1,15 +1,22 @@ use std::{collections::HashMap, sync::Arc}; use ethers::{ + contract::builders::Event, middleware::TimeLag, - prelude::{Http, Provider as EthersProvider}, + prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}, }; -use nomad_ethereum::bindings::{home::Home, replica::Replica}; +use nomad_ethereum::bindings::{ + home::UpdateFilter as HomeUpdateFilter, replica::UpdateFilter as ReplicaUpdateFilter, +}; use nomad_xyz_configuration::{contracts::CoreContracts, get_builtin, NomadConfig}; +use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; +use tokio::{ + sync::mpsc::{self}, + task::JoinHandle, +}; -pub(crate) type Provider = TimeLag>; -pub(crate) type ArcProvider = Arc; +use crate::{between::BetweenEvents, domain::Domain, metrics::Metrics, ArcProvider}; pub(crate) fn config_from_file() -> Option { std::env::var("CONFIG_PATH") @@ -74,71 +81,24 @@ pub(crate) fn monitor() -> eyre::Result { Monitor::from_config(config()?) } -pub(crate) struct Network { - name: String, - provider: ArcProvider, - home: Home, - replicas: HashMap>, -} - -impl Network { - fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { - let name = network.to_owned(); - let provider = provider_for(config, network)?; - - let CoreContracts::Evm(core) = config.core().get(network).expect("invalid config"); - - let home = Home::new(core.home.proxy.as_ethereum_address()?, provider.clone()); - - let replicas = core - .replicas - .iter() - .map(|(k, v)| { - let replica = Replica::new( - v.proxy.as_ethereum_address().expect("invalid addr"), - provider.clone(), - ); - (k.clone(), replica) - }) - .collect(); - - Ok(Network { - name, - provider, - home, - replicas, - }) - } - - pub(crate) fn name(&self) -> &str { - self.name.as_ref() - } - - pub(crate) fn provider(&self) -> &TimeLag> { - self.provider.as_ref() - } - - pub(crate) fn home(&self) -> &Home { - &self.home - } - - pub(crate) fn replicas(&self) -> &HashMap> { - &self.replicas - } -} - +#[derive(Debug)] pub(crate) struct Monitor { config: NomadConfig, - networks: HashMap, + networks: HashMap, + metrics: Arc, } impl Monitor { pub(crate) fn from_config(config: NomadConfig) -> eyre::Result { let mut networks = HashMap::new(); for network in config.networks.iter() { - networks.insert(network.to_owned(), Network::from_config(&config, network)?); + networks.insert(network.to_owned(), Domain::from_config(&config, network)?); } - - Ok(Monitor { config, networks }) + let metrics = Metrics::new()?.into(); + Ok(Monitor { + config, + networks, + metrics, + }) } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index ccac7cf2..6ec56d41 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,14 +1,30 @@ -use init::Monitor; use tracing::info_span; -mod between; +pub(crate) mod between; +pub(crate) mod domain; +pub(crate) mod init; +pub(crate) mod metrics; -mod init; +use std::sync::Arc; + +use ethers::prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}; + +use nomad_ethereum::bindings::{ + home::UpdateFilter as HomeUpdateFilter, replica::UpdateFilter as ReplicaUpdateFilter, +}; +use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; +use tokio::{ + sync::mpsc::{self}, + task::JoinHandle, +}; + +pub(crate) type Provider = ethers::prelude::TimeLag>; +pub(crate) type ArcProvider = Arc; +pub(crate) type ProviderError = ContractError; #[tokio::main] async fn main() -> eyre::Result<()> { init::init_tracing(); - { let span = info_span!("MonitorBootup"); let _span = span.enter(); diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs new file mode 100644 index 00000000..c8a58e2e --- /dev/null +++ b/agents/monitor/src/metrics.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; +use tokio::task::JoinHandle; + +#[derive(Debug)] +pub(crate) struct Metrics { + wallclock_times: prometheus::HistogramVec, + event_times: prometheus::HistogramVec, + counts: prometheus::IntCounterVec, +} + +impl Metrics { + pub(crate) fn new() -> eyre::Result { + let wallclock_times = HistogramVec::new( + HistogramOpts::new( + "inter_event_period_wallclock_ms", + "Ms between events periods, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "event", "emitter", "replica_of"], + )?; + + let event_times = HistogramVec::new( + HistogramOpts::new( + "inter_event_chain_time_seconds", + "Seconds between events, as marked by the chain timestamp", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "event", "emitter", "replica_of"], + )?; + + let counts = IntCounterVec::new( + prometheus::core::Opts::new( + "event_counts", + "Counts of each event, labeled by name and chain", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "event", "emitter", "replica_of"], + )?; + + Ok(Self { + wallclock_times, + event_times, + counts, + }) + } + + pub(crate) fn run_http_server(self: Arc) -> JoinHandle<()> { + todo!() + } +} From 960c327730811f4e4cd13b2762fbc537856275b9 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 16 Jul 2022 22:54:59 -0700 Subject: [PATCH 05/80] feature: event annotation --- agents/monitor/Cargo.toml | 2 + agents/monitor/src/annotate.rs | 12 ++++++ agents/monitor/src/between.rs | 55 +++++++++++++++------------ agents/monitor/src/domain.rs | 49 ++++++++++++------------ agents/monitor/src/init.rs | 20 ++++------ agents/monitor/src/main.rs | 42 ++++++++------------- agents/monitor/src/metrics.rs | 68 ++++++++++++++++++++++++++++++---- 7 files changed, 155 insertions(+), 93 deletions(-) create mode 100644 agents/monitor/src/annotate.rs diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index f468d6b5..9d799dfa 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -17,3 +17,5 @@ nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../../configuration tracing-subscriber = "0.2.15" eyre = "0.6.8" nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } +warp = "0.3.2" +async-trait = "0.1.56" diff --git a/agents/monitor/src/annotate.rs b/agents/monitor/src/annotate.rs new file mode 100644 index 00000000..6f328474 --- /dev/null +++ b/agents/monitor/src/annotate.rs @@ -0,0 +1,12 @@ +use ethers::prelude::LogMeta; + +pub(crate) struct Annotated { + pub(crate) log: T, + pub(crate) meta: LogMeta, +} + +impl From<(T, LogMeta)> for Annotated { + fn from((log, meta): (T, LogMeta)) -> Self { + Self { log, meta } + } +} diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index a17bb9fd..b8198cd5 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -1,47 +1,51 @@ -use tracing::{info_span, instrument::Instrumented, Instrument}; +use tracing::{info_span, Instrument}; -use tokio::task::JoinHandle; +use tokio::sync::mpsc; -use crate::NomadEvent; +use crate::{annotate::Annotated, ProcessStep, StepHandle}; // Track time between events of the same kind -pub struct BetweenEvents { - pub(crate) incoming: tokio::sync::mpsc::UnboundedReceiver, - pub(crate) outgoing: tokio::sync::mpsc::UnboundedSender, +pub(crate) struct BetweenEvents { + pub(crate) incoming: mpsc::UnboundedReceiver>, pub(crate) count: prometheus::IntCounter, pub(crate) wallclock_latency: prometheus::Histogram, - pub(crate) timestamp_latency: prometheus::Histogram, + pub(crate) block_latency: prometheus::Histogram, pub(crate) network: String, } /// Track latency between blockchain events impl BetweenEvents where - T: NomadEvent + 'static, + T: 'static, { - pub fn new( - incoming: tokio::sync::mpsc::UnboundedReceiver, - outgoing: tokio::sync::mpsc::UnboundedSender, + pub(crate) fn new( + incoming: mpsc::UnboundedReceiver>, count: prometheus::IntCounter, wallclock_latency: prometheus::Histogram, - timestamp_latency: prometheus::Histogram, + block_latency: prometheus::Histogram, network: String, ) -> Self { Self { incoming, - outgoing, count, wallclock_latency, - timestamp_latency, + block_latency, network, } } +} - pub fn spawn(mut self) -> Instrumented> { +impl ProcessStep for BetweenEvents +where + T: 'static + Send + Sync, +{ + fn spawn(mut self) -> StepHandle { let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); - tokio::spawn(async move { - let mut last_timestamp = 0; + let (outgoing, rx) = mpsc::unbounded_channel(); + + let handle = tokio::spawn(async move { + let mut last_block_number = 0; let mut wallclock_latency = self.wallclock_latency.start_timer(); loop { @@ -53,17 +57,20 @@ where let incoming = incoming.unwrap(); // calculate the blockchain-reported latency in seconds - let timestamp = incoming.timestamp(); - let event_latency = timestamp.saturating_sub(last_timestamp); - last_timestamp = timestamp; + let block_number = incoming.meta.block_number.as_u64(); + let event_latency = block_number.saturating_sub(last_block_number); + last_block_number = block_number; + + if event_latency != last_block_number { + self.block_latency.observe(event_latency as f64); + } // update our metrics self.count.inc(); - self.timestamp_latency.observe(event_latency as f64); wallclock_latency.observe_duration(); // send the next event out - if self.outgoing.send(incoming).is_err() { + if outgoing.send(incoming).is_err() { break; } @@ -71,6 +78,8 @@ where wallclock_latency = self.wallclock_latency.start_timer(); } }) - .instrument(span) + .instrument(span); + + StepHandle { handle, rx } } } diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 46705966..ffe9016a 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -1,24 +1,25 @@ -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; use ethers::{ contract::builders::Event, middleware::TimeLag, - prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}, + prelude::{Http, Provider as EthersProvider, StreamExt}, }; use nomad_ethereum::bindings::{ home::{DispatchFilter, Home, UpdateFilter as HomeUpdateFilter}, replica::{ProcessFilter, Replica, UpdateFilter as ReplicaUpdateFilter}, }; -use nomad_xyz_configuration::{contracts::CoreContracts, get_builtin, NomadConfig}; -use prometheus::{Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec}; -use tokio::{ - sync::mpsc::{self, UnboundedReceiver}, - task::JoinHandle, +use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; +use prometheus::{Histogram, IntCounter}; +use tokio::sync::mpsc; +use tracing::{info_span, Instrument}; + +use crate::{ + annotate::Annotated, between::BetweenEvents, init::provider_for, ArcProvider, ProcessStep, + Provider, StepHandle, }; -use crate::{between::BetweenEvents, init::provider_for, ArcProvider, NomadEvent, Provider}; - macro_rules! unwrap_event_stream_item { ($event:ident, $net:ident, $name:literal) => {{ match $event { @@ -103,12 +104,14 @@ impl Domain { .collect() } - pub(crate) fn dispatch_stream(&self) -> mpsc::UnboundedReceiver { + pub(crate) fn dispatch_stream(&self) -> StepHandle { let home = self.home.clone(); let (tx, rx) = mpsc::unbounded_channel(); let name = self.name.clone(); - tokio::spawn(async move { + let span = info_span!("dispatch stream convert loop", name = name.as_str()); + + let handle = tokio::spawn(async move { let filter = home.dispatch_filter(); let mut stream = filter .stream() @@ -119,35 +122,31 @@ impl Domain { let event = unwrap_event_stream_item!(event, name, "dispatch"); tx.send(event).unwrap(); } - }); + }) + .instrument(span); - rx + todo!("use new stream_with_meta") + // StepHandle { handle, rx } } pub(crate) fn count( &self, - incoming: mpsc::UnboundedReceiver, + incoming: mpsc::UnboundedReceiver>, count: IntCounter, wallclock_latency: Histogram, timestamp_latency: Histogram, - ) -> UnboundedReceiver + ) -> StepHandle where - T: NomadEvent + 'static, + T: 'static + Send + Sync, { - let (tx, rx) = mpsc::unbounded_channel(); - - let between = BetweenEvents::new( + BetweenEvents::new( incoming, - tx, count, wallclock_latency, timestamp_latency, self.name.clone(), - ); - - between.spawn(); - - rx + ) + .spawn() } // fn update_stream(&self) -> mpsc::UnboundedReceiver { diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 7d8dde27..154449a8 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -1,22 +1,14 @@ use std::{collections::HashMap, sync::Arc}; use ethers::{ - contract::builders::Event, middleware::TimeLag, - prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}, + prelude::{Http, Provider as EthersProvider}, }; -use nomad_ethereum::bindings::{ - home::UpdateFilter as HomeUpdateFilter, replica::UpdateFilter as ReplicaUpdateFilter, -}; -use nomad_xyz_configuration::{contracts::CoreContracts, get_builtin, NomadConfig}; -use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; -use tokio::{ - sync::mpsc::{self}, - task::JoinHandle, -}; +use nomad_xyz_configuration::{get_builtin, NomadConfig}; +use tokio::task::JoinHandle; -use crate::{between::BetweenEvents, domain::Domain, metrics::Metrics, ArcProvider}; +use crate::{domain::Domain, metrics::Metrics, ArcProvider}; pub(crate) fn config_from_file() -> Option { std::env::var("CONFIG_PATH") @@ -101,4 +93,8 @@ impl Monitor { metrics, }) } + + pub(crate) fn run_http_server(&self) -> JoinHandle<()> { + self.metrics.clone().run_http_server() + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 6ec56d41..e4d6edbb 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,23 +1,16 @@ -use tracing::info_span; +use annotate::Annotated; +use std::sync::Arc; +use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; +use tracing::{info_span, instrument::Instrumented}; + +use ethers::prelude::{ContractError, Http, Provider as EthersProvider}; +pub(crate) mod annotate; pub(crate) mod between; pub(crate) mod domain; pub(crate) mod init; pub(crate) mod metrics; -use std::sync::Arc; - -use ethers::prelude::{ContractError, Http, Provider as EthersProvider, StreamExt}; - -use nomad_ethereum::bindings::{ - home::UpdateFilter as HomeUpdateFilter, replica::UpdateFilter as ReplicaUpdateFilter, -}; -use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; -use tokio::{ - sync::mpsc::{self}, - task::JoinHandle, -}; - pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; pub(crate) type ProviderError = ContractError; @@ -29,21 +22,18 @@ async fn main() -> eyre::Result<()> { let span = info_span!("MonitorBootup"); let _span = span.enter(); - let _monitor = init::monitor()?; - - tracing::info!("setup complete!") + let monitor = init::monitor()?; + tracing::info!("setup complete!"); + let _http = monitor.run_http_server(); } Ok(()) } -/// Simple event trait -pub trait NomadEvent: Send + Sync { - /// Get the timestamp - fn timestamp(&self) -> u32; - - /// block number - fn block_number(&self) -> u64; +pub(crate) struct StepHandle { + handle: Instrumented>, + rx: UnboundedReceiver>, +} - /// tx hash - fn tx_hash(&self) -> ethers::types::H256; +pub(crate) trait ProcessStep { + fn spawn(self) -> StepHandle; } diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index c8a58e2e..dead1c54 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -1,15 +1,28 @@ use std::sync::Arc; -use prometheus::{HistogramOpts, HistogramVec, IntCounterVec}; +use prometheus::{Encoder, HistogramOpts, HistogramVec, IntCounterVec}; use tokio::task::JoinHandle; +use warp::Filter; #[derive(Debug)] pub(crate) struct Metrics { wallclock_times: prometheus::HistogramVec, - event_times: prometheus::HistogramVec, + event_blocks: prometheus::HistogramVec, counts: prometheus::IntCounterVec, } +fn u16_from_env(s: impl AsRef) -> Option { + std::env::var(s.as_ref()).ok().and_then(|i| i.parse().ok()) +} + +fn gather() -> prometheus::Result> { + let collected_metrics = prometheus::default_registry().gather(); + let mut out_buf = Vec::with_capacity(1024 * 64); + let encoder = prometheus::TextEncoder::new(); + encoder.encode(&collected_metrics, &mut out_buf)?; + Ok(out_buf) +} + impl Metrics { pub(crate) fn new() -> eyre::Result { let wallclock_times = HistogramVec::new( @@ -22,10 +35,10 @@ impl Metrics { &["chain", "event", "emitter", "replica_of"], )?; - let event_times = HistogramVec::new( + let event_blocks = HistogramVec::new( HistogramOpts::new( - "inter_event_chain_time_seconds", - "Seconds between events, as marked by the chain timestamp", + "inter_event_blocks", + "Blocks between events, as marked by the chain timestamp", ) .namespace("nomad") .const_label("VERSION", env!("CARGO_PKG_VERSION")), @@ -42,14 +55,55 @@ impl Metrics { &["chain", "event", "emitter", "replica_of"], )?; + let registry = prometheus::default_registry(); + registry + .register(Box::new(wallclock_times.clone())) + .expect("unable to register metric"); + registry + .register(Box::new(event_blocks.clone())) + .expect("unable to register metric"); + registry + .register(Box::new(counts.clone())) + .expect("unable to register metric"); + Ok(Self { wallclock_times, - event_times, + event_blocks, counts, }) } pub(crate) fn run_http_server(self: Arc) -> JoinHandle<()> { - todo!() + let port = u16_from_env("METRICS_PORT").unwrap_or(9090); + + tracing::info!( + port, + "starting prometheus server on 0.0.0.0:{port}", + port = port + ); + + tokio::spawn(async move { + warp::serve( + warp::path!("metrics") + .map(move || { + warp::reply::with_header( + gather().expect("failed to encode metrics"), + "Content-Type", + // OpenMetrics specs demands "application/openmetrics-text; version=1.0.0; charset=utf-8" + // but the prometheus scraper itself doesn't seem to care? + // try text/plain to make web browsers happy. + "text/plain; charset=utf-8", + ) + }) + .or(warp::any().map(|| { + warp::reply::with_status( + "go look at /metrics", + warp::http::StatusCode::NOT_FOUND, + ) + })), + ) + .run(([0, 0, 0, 0], port)) + .await; + }) } } From 7d7f2f09d3c1408df5087a2e33ddd747a879380e Mon Sep 17 00:00:00 2001 From: James Date: Sat, 16 Jul 2022 23:02:47 -0700 Subject: [PATCH 06/80] feature: terminal step --- agents/monitor/src/main.rs | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index e4d6edbb..4a5bbe54 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,7 +1,10 @@ use annotate::Annotated; use std::sync::Arc; -use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; -use tracing::{info_span, instrument::Instrumented}; +use tokio::{ + sync::mpsc::{self, UnboundedReceiver}, + task::JoinHandle, +}; +use tracing::{debug_span, info_span, instrument::Instrumented, Instrument}; use ethers::prelude::{ContractError, Http, Provider as EthersProvider}; @@ -35,5 +38,33 @@ pub(crate) struct StepHandle { } pub(crate) trait ProcessStep { - fn spawn(self) -> StepHandle; + fn spawn(self) -> StepHandle + where + T: 'static + Send + Sync; +} + +/// A process step that just drains its input and drops everything +pub(crate) struct Terminal { + rx: UnboundedReceiver>, +} + +impl ProcessStep for Terminal +where + T: 'static + Send + Sync, +{ + fn spawn(mut self) -> StepHandle { + let span = debug_span!("Terminal Handler"); + let handle = tokio::spawn(async move { + loop { + if self.rx.recv().await.is_none() { + tracing::info!("Upstream broke, shutting down"); + break; + } + } + }) + .instrument(span); + + let (_, rx) = mpsc::unbounded_channel(); + StepHandle { handle, rx } + } } From e727661715b70cd4d18b9842719d7c1a8daee3f1 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 10:06:33 -0700 Subject: [PATCH 07/80] wip --- agents/monitor/Cargo.toml | 2 +- agents/monitor/src/annotate.rs | 5 +- agents/monitor/src/between.rs | 48 ++++++++-------- agents/monitor/src/domain.rs | 101 +++++++++++++++------------------ agents/monitor/src/init.rs | 35 +++++++++++- agents/monitor/src/main.rs | 64 +++++++++++++++------ agents/monitor/src/metrics.rs | 66 ++++++++++++++++++++- agents/monitor/src/producer.rs | 42 ++++++++++++++ chains/nomad-ethereum/build.rs | 4 +- 9 files changed, 263 insertions(+), 104 deletions(-) create mode 100644 agents/monitor/src/producer.rs diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index 9d799dfa..fe34f060 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -10,7 +10,7 @@ tokio = { version = "1.0.1", features = ["rt", "macros"] } prometheus = "0.13.1" tracing = "0.1.35" -ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false, features = ['legacy'] } +ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false } nomad-core = { path = "../../nomad-core", default-features = false } nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../../configuration" } diff --git a/agents/monitor/src/annotate.rs b/agents/monitor/src/annotate.rs index 6f328474..f978b2ea 100644 --- a/agents/monitor/src/annotate.rs +++ b/agents/monitor/src/annotate.rs @@ -1,11 +1,12 @@ use ethers::prelude::LogMeta; -pub(crate) struct Annotated { +#[derive(Debug)] +pub(crate) struct WithMeta { pub(crate) log: T, pub(crate) meta: LogMeta, } -impl From<(T, LogMeta)> for Annotated { +impl From<(T, LogMeta)> for WithMeta { fn from((log, meta): (T, LogMeta)) -> Self { Self { log, meta } } diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index b8198cd5..001a6ed6 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -1,15 +1,20 @@ -use tracing::{info_span, Instrument}; +use nomad_ethereum::bindings::home::DispatchFilter; +use tracing::{info_span, instrument::Instrumented, Instrument}; -use tokio::sync::mpsc; +use tokio::{sync::mpsc, task::JoinHandle}; -use crate::{annotate::Annotated, ProcessStep, StepHandle}; +use crate::{annotate::WithMeta, ProcessStep, StepHandle}; -// Track time between events of the same kind -pub(crate) struct BetweenEvents { - pub(crate) incoming: mpsc::UnboundedReceiver>, +pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, pub(crate) wallclock_latency: prometheus::Histogram, pub(crate) block_latency: prometheus::Histogram, +} + +// Track time between events of the same kind +pub(crate) struct BetweenEvents { + pub(crate) incoming: mpsc::UnboundedReceiver, + pub(crate) metrics: BetweenMetrics, pub(crate) network: String, } @@ -19,34 +24,32 @@ where T: 'static, { pub(crate) fn new( - incoming: mpsc::UnboundedReceiver>, - count: prometheus::IntCounter, - wallclock_latency: prometheus::Histogram, - block_latency: prometheus::Histogram, + incoming: mpsc::UnboundedReceiver, + metrics: BetweenMetrics, network: String, ) -> Self { Self { incoming, - count, - wallclock_latency, - block_latency, + metrics, network, } } } -impl ProcessStep for BetweenEvents +pub(crate) type BetweenHandle = Instrumented>>; + +impl ProcessStep> for BetweenEvents> where T: 'static + Send + Sync, { - fn spawn(mut self) -> StepHandle { + fn spawn(mut self) -> BetweenHandle> { let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); let (outgoing, rx) = mpsc::unbounded_channel(); - let handle = tokio::spawn(async move { + tokio::spawn(async move { let mut last_block_number = 0; - let mut wallclock_latency = self.wallclock_latency.start_timer(); + let mut wallclock_latency = self.metrics.wallclock_latency.start_timer(); loop { // get the next event from the channel @@ -62,11 +65,11 @@ where last_block_number = block_number; if event_latency != last_block_number { - self.block_latency.observe(event_latency as f64); + self.metrics.block_latency.observe(event_latency as f64); } // update our metrics - self.count.inc(); + self.metrics.count.inc(); wallclock_latency.observe_duration(); // send the next event out @@ -75,11 +78,10 @@ where } // restart the timer - wallclock_latency = self.wallclock_latency.start_timer(); + wallclock_latency = self.metrics.wallclock_latency.start_timer(); } + self }) - .instrument(span); - - StepHandle { handle, rx } + .instrument(span) } } diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index ffe9016a..2800b466 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -16,8 +16,10 @@ use tokio::sync::mpsc; use tracing::{info_span, Instrument}; use crate::{ - annotate::Annotated, between::BetweenEvents, init::provider_for, ArcProvider, ProcessStep, - Provider, StepHandle, + annotate::WithMeta, + between::{BetweenEvents, BetweenHandle, BetweenMetrics}, + init::provider_for, + ArcProvider, ProcessStep, Provider, StepHandle, }; macro_rules! unwrap_event_stream_item { @@ -104,66 +106,53 @@ impl Domain { .collect() } - pub(crate) fn dispatch_stream(&self) -> StepHandle { - let home = self.home.clone(); - let (tx, rx) = mpsc::unbounded_channel(); - let name = self.name.clone(); - - let span = info_span!("dispatch stream convert loop", name = name.as_str()); - - let handle = tokio::spawn(async move { - let filter = home.dispatch_filter(); - let mut stream = filter - .stream() - .await - .expect("unable to get dispatch stream"); - loop { - let event = stream.next().await; - let event = unwrap_event_stream_item!(event, name, "dispatch"); - tx.send(event).unwrap(); - } - }) - .instrument(span); - - todo!("use new stream_with_meta") - // StepHandle { handle, rx } + pub(crate) fn dispatch_stream(&self) -> BetweenHandle> { + todo!() + // let home = self.home.clone(); + // let (tx, rx) = mpsc::unbounded_channel(); + // let name = self.name.clone(); + + // let span = info_span!("dispatch stream convert loop", name = name.as_str()); + + // let handle = tokio::spawn(async move { + // let filter = home.dispatch_filter(); + // let stream = filter.stream_with_meta().await; + + // let mut stream = match stream { + // Ok(stream) => stream, + // Err(e) => { + // tracing::error!( + // network = name.as_str(), + // home = format!("{:?}", home.address()), + // event = "dispatch", + // error = %e, + // "unable to get dispatch stream", + // ); + // panic!("unable to get dispatch stream"); + // } + // }; + + // loop { + // let event = stream.next().await; + // let (log, meta) = unwrap_event_stream_item!(event, name, "dispatch"); + // tx.send(WithMeta { log, meta }).unwrap(); + // } + // self + // }) + // .instrument(span); + + // StepHandle { handle, rx } } pub(crate) fn count( &self, - incoming: mpsc::UnboundedReceiver>, - count: IntCounter, - wallclock_latency: Histogram, - timestamp_latency: Histogram, - ) -> StepHandle + incoming: mpsc::UnboundedReceiver>, + metrics: BetweenMetrics, + ) -> BetweenHandle> where T: 'static + Send + Sync, { - BetweenEvents::new( - incoming, - count, - wallclock_latency, - timestamp_latency, - self.name.clone(), - ) - .spawn() + let network = self.name.clone(); + BetweenEvents::>::new(incoming, metrics, network).spawn() } - - // fn update_stream(&self) -> mpsc::UnboundedReceiver { - // let home = self.home.clone(); - // let (tx, rx) = mpsc::unbounded_channel(); - // let name = self.name.clone(); - - // tokio::spawn(async move { - // let filter = home.update_filter(); - // let mut stream = filter.stream().await.expect("unable to get update stream"); - // loop { - // let event = stream.next().await; - // let event = unwrap_event_stream_item!(event, name, "update"); - // tx.send(event).unwrap(); - // } - // }); - - // rx - // } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 154449a8..3286540b 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -5,10 +5,17 @@ use ethers::{ prelude::{Http, Provider as EthersProvider}, }; +use nomad_ethereum::bindings::home::DispatchFilter; use nomad_xyz_configuration::{get_builtin, NomadConfig}; use tokio::task::JoinHandle; -use crate::{domain::Domain, metrics::Metrics, ArcProvider}; +use crate::{ + annotate::WithMeta, + between::{BetweenEvents, BetweenHandle}, + domain::Domain, + metrics::Metrics, + ArcProvider, StepHandle, +}; pub(crate) fn config_from_file() -> Option { std::env::var("CONFIG_PATH") @@ -97,4 +104,30 @@ impl Monitor { pub(crate) fn run_http_server(&self) -> JoinHandle<()> { self.metrics.clone().run_http_server() } + + pub(crate) fn run_between_dispatch(&self) -> HashMap<&str, BetweenHandle> { + self.networks + .iter() + .map(|(chain, domain)| { + let emitter = format!("{:?}", domain.home().address()); + let event = "dispatch"; + + tracing::info!( + network = chain.as_str(), + home = emitter, + event, + "starting dispatch counter", + ); + todo!() + + // let stream = domain.dispatch_stream(); + + // let metrics = self.metrics.between_metrics(chain, event, &emitter, None); + + // let task = domain.count(stream.rx, metrics); + + // (chain.as_str(), task) + }) + .collect() + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 4a5bbe54..36290583 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,9 +1,6 @@ -use annotate::Annotated; -use std::sync::Arc; -use tokio::{ - sync::mpsc::{self, UnboundedReceiver}, - task::JoinHandle, -}; +use annotate::WithMeta; +use std::{panic, sync::Arc}; +use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; use tracing::{debug_span, info_span, instrument::Instrumented, Instrument}; use ethers::prelude::{ContractError, Http, Provider as EthersProvider}; @@ -13,6 +10,7 @@ pub(crate) mod between; pub(crate) mod domain; pub(crate) mod init; pub(crate) mod metrics; +pub(crate) mod producer; pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; @@ -28,43 +26,73 @@ async fn main() -> eyre::Result<()> { let monitor = init::monitor()?; tracing::info!("setup complete!"); let _http = monitor.run_http_server(); + + let dispatch_trackers = monitor.run_between_dispatch(); + + // should cause it to run until crashes occur + dispatch_trackers.into_iter().next().unwrap().1.await; } Ok(()) } -pub(crate) struct StepHandle { - handle: Instrumented>, - rx: UnboundedReceiver>, +pub(crate) struct StepHandle { + handle: Instrumented>, + rx: UnboundedReceiver, } pub(crate) trait ProcessStep { - fn spawn(self) -> StepHandle + fn spawn(self) -> Instrumented> + where + T: 'static + Send + Sync, + Self: 'static + Send + Sync + Sized; + + fn forever(self) -> JoinHandle<()> where - T: 'static + Send + Sync; + T: 'static + Send + Sync, + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { + let mut handle = self.spawn(); + loop { + let result = handle.await; + + let again = match result { + Ok(handle) => handle, + Err(e) => { + tracing::error!(err = %e, "JoinError in forever"); + panic!("JoinError in forever"); + } + }; + tracing::warn!("restarting task"); + handle = again.spawn() + } + }) + } } /// A process step that just drains its input and drops everything +/// Its [`StepHandle`] will never produce values. pub(crate) struct Terminal { - rx: UnboundedReceiver>, + rx: UnboundedReceiver>, } +pub(crate) type TerminalHandle = Instrumented>>; + impl ProcessStep for Terminal where T: 'static + Send + Sync, { - fn spawn(mut self) -> StepHandle { + fn spawn(mut self) -> TerminalHandle { let span = debug_span!("Terminal Handler"); - let handle = tokio::spawn(async move { + tokio::spawn(async move { loop { if self.rx.recv().await.is_none() { tracing::info!("Upstream broke, shutting down"); break; } } + self }) - .instrument(span); - - let (_, rx) = mpsc::unbounded_channel(); - StepHandle { handle, rx } + .instrument(span) } } diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index dead1c54..faf50e6d 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -1,9 +1,11 @@ use std::sync::Arc; -use prometheus::{Encoder, HistogramOpts, HistogramVec, IntCounterVec}; +use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec}; use tokio::task::JoinHandle; use warp::Filter; +use crate::between::BetweenMetrics; + #[derive(Debug)] pub(crate) struct Metrics { wallclock_times: prometheus::HistogramVec, @@ -106,4 +108,66 @@ impl Metrics { .await; }) } + + pub(crate) fn event_counter( + &self, + chain: &str, + event: &str, + emitter: &str, + replica_of: Option<&str>, + ) -> IntCounter { + self.counts.with_label_values(&[ + chain.as_ref(), + event.as_ref(), + emitter.as_ref(), + replica_of.unwrap_or("n/a"), + ]) + } + + pub(crate) fn wallclock_latency( + &self, + chain: &str, + event: &str, + emitter: &str, + replica_of: Option<&str>, + ) -> Histogram { + // &["chain", "event", "emitter", "replica_of"], + + self.wallclock_times.with_label_values(&[ + chain.as_ref(), + event.as_ref(), + emitter.as_ref(), + replica_of.unwrap_or("n/a"), + ]) + } + + pub(crate) fn block_latency( + &self, + chain: &str, + event: &str, + emitter: &str, + replica_of: Option<&str>, + ) -> Histogram { + // &["chain", "event", "emitter", "replica_of"], + self.event_blocks.with_label_values(&[ + chain.as_ref(), + event.as_ref(), + emitter.as_ref(), + replica_of.unwrap_or("n/a"), + ]) + } + + pub(crate) fn between_metrics( + &self, + chain: &str, + event: &str, + emitter: &str, + replica_of: Option<&str>, + ) -> BetweenMetrics { + BetweenMetrics { + count: self.event_counter(chain, event, emitter, replica_of), + wallclock_latency: self.wallclock_latency(chain, event, emitter, replica_of), + block_latency: self.block_latency(chain, event, emitter, replica_of), + } + } } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs new file mode 100644 index 00000000..2940dde3 --- /dev/null +++ b/agents/monitor/src/producer.rs @@ -0,0 +1,42 @@ +use crate::{annotate::WithMeta, ProcessStep, StepHandle}; + +use ethers::prelude::Middleware; +use nomad_ethereum::bindings::{ + home::{DispatchFilter, Home, UpdateFilter as HomeUpdateFilter}, + replica::{ProcessFilter, Replica, UpdateFilter as ReplicaUpdateFilter}, +}; +use tokio::{ + sync::mpsc::{UnboundedReceiver, UnboundedSender}, + task::JoinHandle, +}; +use tracing::{info_span, instrument::Instrumented, Instrument}; + +#[derive(Debug)] +pub struct DispatchProducer { + home: Home, + network: String, + tx: UnboundedSender>, +} + +pub(crate) type DispatchProducerHandle = Instrumented>; + +impl ProcessStep> for DispatchProducer { + fn spawn(self) -> DispatchProducerHandle { + let span = info_span!( + "DispatchProducer", + home = format!("{:?}", self.home.address()), + network = self.network.as_str(), + event = "dispatch", + ); + + tokio::spawn(async move { + let provider = self.home.client(); + let height = provider.get_block_number().await.unwrap(); + loop { + todo!() + } + self + }) + .instrument(span) + } +} diff --git a/chains/nomad-ethereum/build.rs b/chains/nomad-ethereum/build.rs index 2fcabfa3..afd78980 100644 --- a/chains/nomad-ethereum/build.rs +++ b/chains/nomad-ethereum/build.rs @@ -32,14 +32,14 @@ fn main() { names.sort(); for name in names.iter() { - writeln!(mod_file, "pub(crate) mod {};", name).expect("failed to write to modfile"); + writeln!(mod_file, "pub mod {};", name).expect("failed to write to modfile"); } } fn create_mod_rs() -> File { let mod_file_path = PathBuf::from(&format!("{}/mod.rs", BINDINGS_DIR)); let mut mod_file = std::fs::File::create(&mod_file_path).expect("could not create modfile"); - writeln!(mod_file, "#![allow(clippy::all)]").unwrap(); + writeln!(mod_file, "#![allow(clippy::all, missing_docs)]").unwrap(); mod_file } From 02ac15b091b594141979ff7ebd6b0642522efe33 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 13:43:32 -0700 Subject: [PATCH 08/80] chore: rebuild --- Cargo.lock | 81 +++++++++++++++++++++++++++++++++++++++++++++++------- Cargo.toml | 1 + 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 118ca00a..7fa1bdea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1911,7 +1911,7 @@ dependencies = [ "nomad-test", "nomad-types", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "rand", "serde 1.0.140", "serde_json", @@ -2055,6 +2055,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.9" @@ -2266,6 +2275,23 @@ dependencies = [ "similar", ] +[[package]] +name = "monitor" +version = "0.1.0" +dependencies = [ + "async-trait", + "ethers", + "eyre", + "nomad-core", + "nomad-ethereum", + "nomad-xyz-configuration", + "prometheus 0.13.1", + "tokio", + "tracing", + "tracing-subscriber 0.2.25", + "warp", +] + [[package]] name = "multipart" version = "0.18.0" @@ -2340,7 +2366,7 @@ dependencies = [ "nomad-xyz-configuration", "once_cell", "opentelemetry", - "prometheus", + "prometheus 0.12.0", "rand", "rocksdb", "rusoto_core", @@ -2392,7 +2418,7 @@ dependencies = [ "nomad-xyz-configuration", "num", "once_cell", - "prometheus", + "prometheus 0.12.0", "rocksdb", "rusoto_core", "rusoto_credential", @@ -2427,7 +2453,7 @@ dependencies = [ "nomad-xyz-configuration", "num", "once_cell", - "prometheus", + "prometheus 0.12.0", "reqwest", "rocksdb", "serde 1.0.140", @@ -2454,7 +2480,7 @@ dependencies = [ "nomad-core", "nomad-ethereum", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "rand", "rocksdb", "serde 1.0.140", @@ -2472,7 +2498,7 @@ dependencies = [ "color-eyre", "ethers", "hex", - "prometheus", + "prometheus 0.12.0", "serde 1.0.140", "serde_json", "thiserror", @@ -3021,7 +3047,7 @@ dependencies = [ "nomad-test", "nomad-types", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "rocksdb", "rusoto_core", "rusoto_s3", @@ -3050,6 +3076,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "prometheus" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot 0.12.1", + "protobuf", + "thiserror", +] + [[package]] name = "protobuf" version = "2.27.1" @@ -3169,6 +3210,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.27" @@ -3191,7 +3241,7 @@ dependencies = [ "nomad-core", "nomad-test", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "serde 1.0.140", "serde_json", "serial_test", @@ -4351,9 +4401,20 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde 1.0.140", + "serde_json", "sharded-slab", + "smallvec", "thread_local", + "tracing", "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -4519,7 +4580,7 @@ dependencies = [ "nomad-ethereum", "nomad-test", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "rocksdb", "serde 1.0.140", "serde_json", @@ -4747,7 +4808,7 @@ dependencies = [ "nomad-ethereum", "nomad-test", "nomad-xyz-configuration", - "prometheus", + "prometheus 0.12.0", "rocksdb", "serde 1.0.140", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 54350ccb..60ed217c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "agents/relayer", "agents/watcher", "agents/processor", + "agents/monitor", "tools/kms-cli", "tools/nomad-cli", "tools/balance-exporter", From e12e84e539514693fef791fbad6e239aad303bbb Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 14:19:28 -0700 Subject: [PATCH 09/80] feature: dispatch producer --- agents/monitor/src/annotate.rs | 10 +++++-- agents/monitor/src/between.rs | 29 +++++++++++-------- agents/monitor/src/domain.rs | 31 ++++++++------------ agents/monitor/src/macros.rs | 11 ++++++++ agents/monitor/src/main.rs | 32 +++++++++++++-------- agents/monitor/src/producer.rs | 44 +++++++++++++++++++++++++---- chains/nomad-ethereum/src/gelato.rs | 6 ++-- 7 files changed, 109 insertions(+), 54 deletions(-) create mode 100644 agents/monitor/src/macros.rs diff --git a/agents/monitor/src/annotate.rs b/agents/monitor/src/annotate.rs index f978b2ea..929d8356 100644 --- a/agents/monitor/src/annotate.rs +++ b/agents/monitor/src/annotate.rs @@ -1,12 +1,18 @@ use ethers::prelude::LogMeta; #[derive(Debug)] -pub(crate) struct WithMeta { +pub(crate) struct WithMeta +where + T: std::fmt::Debug, +{ pub(crate) log: T, pub(crate) meta: LogMeta, } -impl From<(T, LogMeta)> for WithMeta { +impl From<(T, LogMeta)> for WithMeta +where + T: std::fmt::Debug, +{ fn from((log, meta): (T, LogMeta)) -> Self { Self { log, meta } } diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 001a6ed6..044ca776 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -3,8 +3,9 @@ use tracing::{info_span, instrument::Instrumented, Instrument}; use tokio::{sync::mpsc, task::JoinHandle}; -use crate::{annotate::WithMeta, ProcessStep, StepHandle}; +use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHandle}; +#[derive(Debug)] pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, pub(crate) wallclock_latency: prometheus::Histogram, @@ -12,7 +13,11 @@ pub(crate) struct BetweenMetrics { } // Track time between events of the same kind -pub(crate) struct BetweenEvents { +#[derive(Debug)] +pub(crate) struct BetweenEvents +where + T: std::fmt::Debug, +{ pub(crate) incoming: mpsc::UnboundedReceiver, pub(crate) metrics: BetweenMetrics, pub(crate) network: String, @@ -21,7 +26,7 @@ pub(crate) struct BetweenEvents { /// Track latency between blockchain events impl BetweenEvents where - T: 'static, + T: 'static + std::fmt::Debug, { pub(crate) fn new( incoming: mpsc::UnboundedReceiver, @@ -36,11 +41,11 @@ where } } -pub(crate) type BetweenHandle = Instrumented>>; +pub(crate) type BetweenHandle = Restartable>; impl ProcessStep> for BetweenEvents> where - T: 'static + Send + Sync, + T: 'static + Send + Sync + std::fmt::Debug, { fn spawn(mut self) -> BetweenHandle> { let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); @@ -54,9 +59,8 @@ where loop { // get the next event from the channel let incoming = self.incoming.recv().await; - if incoming.is_none() { - break; - } + task_bail_if!(incoming.is_none(), self, "inbound channel broke"); + let incoming = incoming.unwrap(); // calculate the blockchain-reported latency in seconds @@ -73,14 +77,15 @@ where wallclock_latency.observe_duration(); // send the next event out - if outgoing.send(incoming).is_err() { - break; - } + task_bail_if!( + outgoing.send(incoming).is_err(), + self, + "outbound channel broke" + ); // restart the timer wallclock_latency = self.metrics.wallclock_latency.start_timer(); } - self }) .instrument(span) } diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 2800b466..011d5663 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -19,6 +19,7 @@ use crate::{ annotate::WithMeta, between::{BetweenEvents, BetweenHandle, BetweenMetrics}, init::provider_for, + producer::{DispatchProducer, DispatchProducerHandle}, ArcProvider, ProcessStep, Provider, StepHandle, }; @@ -37,7 +38,7 @@ macro_rules! unwrap_event_stream_item { #[derive(Debug)] pub(crate) struct Domain { - pub(crate) name: String, + pub(crate) network: String, pub(crate) provider: ArcProvider, pub(crate) home: Home, pub(crate) replicas: HashMap>, @@ -65,7 +66,7 @@ impl Domain { .collect(); Ok(Domain { - name, + network: name, provider, home, replicas, @@ -73,7 +74,7 @@ impl Domain { } pub(crate) fn name(&self) -> &str { - self.name.as_ref() + self.network.as_ref() } pub(crate) fn provider(&self) -> &TimeLag> { @@ -88,22 +89,14 @@ impl Domain { &self.replicas } - pub(crate) fn update_filter(&self) -> Event { - self.home.update_filter() - } + pub(crate) fn dispatch_producer( + &self, + ) -> StepHandle> { + let (tx, rx) = mpsc::unbounded_channel(); - pub(crate) fn relay_filters(&self) -> HashMap<&str, Event> { - self.replicas - .iter() - .map(|(k, v)| (k.as_str(), v.update_filter())) - .collect() - } + let handle = DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); - pub(crate) fn process_filters(&self) -> HashMap<&str, Event> { - self.replicas - .iter() - .map(|(k, v)| (k.as_str(), v.process_filter())) - .collect() + StepHandle { handle, rx } } pub(crate) fn dispatch_stream(&self) -> BetweenHandle> { @@ -150,9 +143,9 @@ impl Domain { metrics: BetweenMetrics, ) -> BetweenHandle> where - T: 'static + Send + Sync, + T: 'static + Send + Sync + std::fmt::Debug, { - let network = self.name.clone(); + let network = self.network.clone(); BetweenEvents::>::new(incoming, metrics, network).spawn() } } diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs new file mode 100644 index 00000000..cd1edb6f --- /dev/null +++ b/agents/monitor/src/macros.rs @@ -0,0 +1,11 @@ +// bails a restartable task +#[macro_export] +macro_rules! task_bail_if { + ($cond:expr, $self:ident, $err:expr) => { + if $cond { + let err = eyre::eyre!($err); + tracing::error!(task = ?$self, err = %err, "Task failed"); + return ($self, err); + } + }; +} diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 36290583..4b55c31d 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -9,6 +9,7 @@ pub(crate) mod annotate; pub(crate) mod between; pub(crate) mod domain; pub(crate) mod init; +pub(crate) mod macros; pub(crate) mod metrics; pub(crate) mod producer; @@ -35,20 +36,23 @@ async fn main() -> eyre::Result<()> { Ok(()) } +pub type Restartable = Instrumented>; + pub(crate) struct StepHandle { - handle: Instrumented>, + handle: Restartable, rx: UnboundedReceiver, } -pub(crate) trait ProcessStep { - fn spawn(self) -> Instrumented> +pub(crate) trait ProcessStep +where + T: 'static + Send + Sync + std::fmt::Debug, +{ + fn spawn(self) -> Restartable where - T: 'static + Send + Sync, Self: 'static + Send + Sync + Sized; fn forever(self) -> JoinHandle<()> where - T: 'static + Send + Sync, Self: 'static + Send + Sync + Sized, { tokio::spawn(async move { @@ -57,13 +61,15 @@ pub(crate) trait ProcessStep { let result = handle.await; let again = match result { - Ok(handle) => handle, + Ok((handle, report)) => { + tracing::warn!(error = %report, "Restarting task"); + handle + } Err(e) => { tracing::error!(err = %e, "JoinError in forever"); panic!("JoinError in forever"); } }; - tracing::warn!("restarting task"); handle = again.spawn() } }) @@ -72,15 +78,18 @@ pub(crate) trait ProcessStep { /// A process step that just drains its input and drops everything /// Its [`StepHandle`] will never produce values. -pub(crate) struct Terminal { +pub(crate) struct Terminal +where + T: std::fmt::Debug, +{ rx: UnboundedReceiver>, } -pub(crate) type TerminalHandle = Instrumented>>; +pub(crate) type TerminalHandle = Restartable>; impl ProcessStep for Terminal where - T: 'static + Send + Sync, + T: 'static + Send + Sync + std::fmt::Debug, { fn spawn(mut self) -> TerminalHandle { let span = debug_span!("Terminal Handler"); @@ -88,10 +97,9 @@ where loop { if self.rx.recv().await.is_none() { tracing::info!("Upstream broke, shutting down"); - break; + return (self, eyre::eyre!("")); } } - self }) .instrument(span) } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 2940dde3..b61d59c7 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -1,4 +1,4 @@ -use crate::{annotate::WithMeta, ProcessStep, StepHandle}; +use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHandle}; use ethers::prelude::Middleware; use nomad_ethereum::bindings::{ @@ -9,16 +9,26 @@ use tokio::{ sync::mpsc::{UnboundedReceiver, UnboundedSender}, task::JoinHandle, }; -use tracing::{info_span, instrument::Instrumented, Instrument}; +use tracing::{info_span, Instrument}; #[derive(Debug)] -pub struct DispatchProducer { +pub(crate) struct DispatchProducer { home: Home, network: String, tx: UnboundedSender>, } -pub(crate) type DispatchProducerHandle = Instrumented>; +impl DispatchProducer { + pub(crate) fn new( + home: Home, + network: String, + tx: UnboundedSender>, + ) -> Self { + Self { home, network, tx } + } +} + +pub(crate) type DispatchProducerHandle = Restartable; impl ProcessStep> for DispatchProducer { fn spawn(self) -> DispatchProducerHandle { @@ -32,10 +42,32 @@ impl ProcessStep> for DispatchProducer { tokio::spawn(async move { let provider = self.home.client(); let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; loop { - todo!() + if from < to { + let res = self + .home + .dispatch_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await + .map_err(|e| eyre::eyre!(e)); + + task_bail_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + task_bail_if!(res.is_err(), self, res.unwrap_err()); + } + } + let tip = provider.get_block_number().await.unwrap() - 5; + from = to; + to = std::cmp::min(to, tip); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; } - self }) .instrument(span) } diff --git a/chains/nomad-ethereum/src/gelato.rs b/chains/nomad-ethereum/src/gelato.rs index 4e3a0a7e..349bcc00 100644 --- a/chains/nomad-ethereum/src/gelato.rs +++ b/chains/nomad-ethereum/src/gelato.rs @@ -182,7 +182,7 @@ where .await .map_err(|e| ChainCommunicationError::CustomError(e.into()))?; - let request = ForwardRequestBuilder::default() + let params = ForwardRequestBuilder::default() .chain_id(self.chain_id) .target(target) .data(data.into()) @@ -197,12 +197,12 @@ where .expect("signer doesn't fail"); info!( - request = serde_json::to_string(&request).unwrap().as_str(), + request = serde_json::to_string(¶ms).unwrap().as_str(), "Signed gelato forward request." ); self.gelato() - .send_forward_request(&request) + .send_forward_request(¶ms) .await .map_err(|e| ChainCommunicationError::TxSubmissionError(e.into())) } From 4b872ac84f0e5f98d8f535242bde76f47518313f Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 14:41:16 -0700 Subject: [PATCH 10/80] feature: basic dispatch tracking --- agents/monitor/src/between.rs | 51 ++++++++++++++++++++++------------ agents/monitor/src/domain.rs | 47 +++++-------------------------- agents/monitor/src/init.rs | 17 ++++++------ agents/monitor/src/main.rs | 2 +- agents/monitor/src/producer.rs | 8 +++--- 5 files changed, 54 insertions(+), 71 deletions(-) diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 044ca776..cdb870bc 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -1,11 +1,9 @@ -use nomad_ethereum::bindings::home::DispatchFilter; -use tracing::{info_span, instrument::Instrumented, Instrument}; +use tracing::{info_span, Instrument}; -use tokio::{sync::mpsc, task::JoinHandle}; +use tokio::sync::mpsc; use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHandle}; -#[derive(Debug)] pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, pub(crate) wallclock_latency: prometheus::Histogram, @@ -13,14 +11,21 @@ pub(crate) struct BetweenMetrics { } // Track time between events of the same kind -#[derive(Debug)] -pub(crate) struct BetweenEvents -where - T: std::fmt::Debug, -{ +pub(crate) struct BetweenEvents { pub(crate) incoming: mpsc::UnboundedReceiver, pub(crate) metrics: BetweenMetrics, pub(crate) network: String, + pub(crate) event: String, + pub(crate) outgoing: mpsc::UnboundedSender, +} + +impl std::fmt::Debug for BetweenEvents { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BetweenEvents") + .field("network", &self.network) + .field("event", &self.event) + .finish() + } } /// Track latency between blockchain events @@ -31,26 +36,33 @@ where pub(crate) fn new( incoming: mpsc::UnboundedReceiver, metrics: BetweenMetrics, - network: String, + network: impl AsRef, + event: impl AsRef, + outgoing: mpsc::UnboundedSender, ) -> Self { Self { incoming, metrics, - network, + network: network.as_ref().to_owned(), + event: event.as_ref().to_owned(), + outgoing, } } } -pub(crate) type BetweenHandle = Restartable>; +pub(crate) type BetweenHandle = StepHandle, T>; +pub(crate) type BetweenTask = Restartable>; impl ProcessStep> for BetweenEvents> where T: 'static + Send + Sync + std::fmt::Debug, { - fn spawn(mut self) -> BetweenHandle> { - let span = info_span!("LatencyMetricsTask", network = self.network.as_str()); - - let (outgoing, rx) = mpsc::unbounded_channel(); + fn spawn(mut self) -> BetweenTask> { + let span = info_span!( + "LatencyMetricsTask", + network = self.network.as_str(), + event = self.event.as_str() + ); tokio::spawn(async move { let mut last_block_number = 0; @@ -62,6 +74,11 @@ where task_bail_if!(incoming.is_none(), self, "inbound channel broke"); let incoming = incoming.unwrap(); + tracing::debug!( + block_number = %incoming.meta.block_number, + event = self.event.as_str(), + "received incoming event" + ); // calculate the blockchain-reported latency in seconds let block_number = incoming.meta.block_number.as_u64(); @@ -78,7 +95,7 @@ where // send the next event out task_bail_if!( - outgoing.send(incoming).is_err(), + self.outgoing.send(incoming).is_err(), self, "outbound channel broke" ); diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 011d5663..876b4e3c 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -17,7 +17,7 @@ use tracing::{info_span, Instrument}; use crate::{ annotate::WithMeta, - between::{BetweenEvents, BetweenHandle, BetweenMetrics}, + between::{BetweenEvents, BetweenHandle, BetweenMetrics, BetweenTask}, init::provider_for, producer::{DispatchProducer, DispatchProducerHandle}, ArcProvider, ProcessStep, Provider, StepHandle, @@ -99,53 +99,20 @@ impl Domain { StepHandle { handle, rx } } - pub(crate) fn dispatch_stream(&self) -> BetweenHandle> { - todo!() - // let home = self.home.clone(); - // let (tx, rx) = mpsc::unbounded_channel(); - // let name = self.name.clone(); - - // let span = info_span!("dispatch stream convert loop", name = name.as_str()); - - // let handle = tokio::spawn(async move { - // let filter = home.dispatch_filter(); - // let stream = filter.stream_with_meta().await; - - // let mut stream = match stream { - // Ok(stream) => stream, - // Err(e) => { - // tracing::error!( - // network = name.as_str(), - // home = format!("{:?}", home.address()), - // event = "dispatch", - // error = %e, - // "unable to get dispatch stream", - // ); - // panic!("unable to get dispatch stream"); - // } - // }; - - // loop { - // let event = stream.next().await; - // let (log, meta) = unwrap_event_stream_item!(event, name, "dispatch"); - // tx.send(WithMeta { log, meta }).unwrap(); - // } - // self - // }) - // .instrument(span); - - // StepHandle { handle, rx } - } - pub(crate) fn count( &self, incoming: mpsc::UnboundedReceiver>, metrics: BetweenMetrics, + event: impl AsRef, ) -> BetweenHandle> where T: 'static + Send + Sync + std::fmt::Debug, { let network = self.network.clone(); - BetweenEvents::>::new(incoming, metrics, network).spawn() + let (tx, rx) = mpsc::unbounded_channel(); + let handle = + BetweenEvents::>::new(incoming, metrics, network, event, tx).spawn(); + + StepHandle { handle, rx } } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 3286540b..5fc7fdb5 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -14,7 +14,7 @@ use crate::{ between::{BetweenEvents, BetweenHandle}, domain::Domain, metrics::Metrics, - ArcProvider, StepHandle, + ArcProvider, ProcessStep, StepHandle, }; pub(crate) fn config_from_file() -> Option { @@ -105,7 +105,9 @@ impl Monitor { self.metrics.clone().run_http_server() } - pub(crate) fn run_between_dispatch(&self) -> HashMap<&str, BetweenHandle> { + pub(crate) fn run_between_dispatch( + &self, + ) -> HashMap<&str, BetweenHandle>> { self.networks .iter() .map(|(chain, domain)| { @@ -118,15 +120,12 @@ impl Monitor { event, "starting dispatch counter", ); - todo!() + let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - // let stream = domain.dispatch_stream(); + let producer = domain.dispatch_producer(); + let between = domain.count(producer.rx, metrics, event); - // let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - - // let task = domain.count(stream.rx, metrics); - - // (chain.as_str(), task) + (chain.as_str(), between) }) .collect() } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 4b55c31d..701dd15b 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -31,7 +31,7 @@ async fn main() -> eyre::Result<()> { let dispatch_trackers = monitor.run_between_dispatch(); // should cause it to run until crashes occur - dispatch_trackers.into_iter().next().unwrap().1.await; + dispatch_trackers.into_iter().next().unwrap().1.handle.await; } Ok(()) } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index b61d59c7..733cb199 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -28,10 +28,11 @@ impl DispatchProducer { } } -pub(crate) type DispatchProducerHandle = Restartable; +pub(crate) type DispatchProducerTask = Restartable; +pub(crate) type DispatchProducerHandle = StepHandle; impl ProcessStep> for DispatchProducer { - fn spawn(self) -> DispatchProducerHandle { + fn spawn(self) -> DispatchProducerTask { let span = info_span!( "DispatchProducer", home = format!("{:?}", self.home.address()), @@ -52,8 +53,7 @@ impl ProcessStep> for DispatchProducer { .from_block(from) .to_block(to) .query_with_meta() - .await - .map_err(|e| eyre::eyre!(e)); + .await; task_bail_if!(res.is_err(), self, res.unwrap_err()); From c56749192fddab5d47ed27353a02fb4f4ca35552 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 17:24:46 -0700 Subject: [PATCH 11/80] feature: basic event latency tracking --- agents/monitor/src/between.rs | 2 + agents/monitor/src/domain.rs | 163 ++++++++++++++++++------ agents/monitor/src/init.rs | 75 ++++++++--- agents/monitor/src/main.rs | 9 +- agents/monitor/src/metrics.rs | 41 +++--- agents/monitor/src/producer.rs | 226 +++++++++++++++++++++++++++++++-- configuration/build.rs | 24 ++-- 7 files changed, 429 insertions(+), 111 deletions(-) diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index cdb870bc..4b46dbaa 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -59,6 +59,7 @@ where { fn spawn(mut self) -> BetweenTask> { let span = info_span!( + target: "monitor::between", "LatencyMetricsTask", network = self.network.as_str(), event = self.event.as_str() @@ -75,6 +76,7 @@ where let incoming = incoming.unwrap(); tracing::debug!( + target: "monitor::between", block_number = %incoming.meta.block_number, event = self.event.as_str(), "received incoming event" diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 876b4e3c..30da1a01 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -1,55 +1,37 @@ -use std::collections::HashMap; - -use ethers::{ - contract::builders::Event, - middleware::TimeLag, - prelude::{Http, Provider as EthersProvider, StreamExt}, -}; +use std::{collections::HashMap, sync::Arc}; use nomad_ethereum::bindings::{ - home::{DispatchFilter, Home, UpdateFilter as HomeUpdateFilter}, - replica::{ProcessFilter, Replica, UpdateFilter as ReplicaUpdateFilter}, + home::Home, + replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; -use prometheus::{Histogram, IntCounter}; use tokio::sync::mpsc; -use tracing::{info_span, Instrument}; use crate::{ annotate::WithMeta, - between::{BetweenEvents, BetweenHandle, BetweenMetrics, BetweenTask}, + between::{BetweenEvents, BetweenHandle, BetweenMetrics}, init::provider_for, - producer::{DispatchProducer, DispatchProducerHandle}, - ArcProvider, ProcessStep, Provider, StepHandle, + metrics::Metrics, + producer::{ + DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, + RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, + }, + ProcessStep, Provider, StepHandle, }; -macro_rules! unwrap_event_stream_item { - ($event:ident, $net:ident, $name:literal) => {{ - match $event { - None => break, - Some(Err(error)) => { - tracing::error!(%error, net = AsRef::::as_ref(&$net), event = $name, "Stream ended"); - break; - } - Some(Ok(event)) => event, - } - }} -} - #[derive(Debug)] pub(crate) struct Domain { pub(crate) network: String, - pub(crate) provider: ArcProvider, pub(crate) home: Home, pub(crate) replicas: HashMap>, } impl Domain { pub(crate) fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { - let name = network.to_owned(); - let provider = provider_for(config, network)?; + let network = network.to_owned(); + let provider = provider_for(config, &network)?; - let CoreContracts::Evm(core) = config.core().get(network).expect("invalid config"); + let CoreContracts::Evm(core) = config.core().get(&network).expect("invalid config"); let home = Home::new(core.home.proxy.as_ethereum_address()?, provider.clone()); @@ -66,8 +48,7 @@ impl Domain { .collect(); Ok(Domain { - network: name, - provider, + network, home, replicas, }) @@ -77,10 +58,6 @@ impl Domain { self.network.as_ref() } - pub(crate) fn provider(&self) -> &TimeLag> { - self.provider.as_ref() - } - pub(crate) fn home(&self) -> &Home { &self.home } @@ -89,9 +66,7 @@ impl Domain { &self.replicas } - pub(crate) fn dispatch_producer( - &self, - ) -> StepHandle> { + pub(crate) fn dispatch_producer(&self) -> DispatchProducerHandle { let (tx, rx) = mpsc::unbounded_channel(); let handle = DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); @@ -99,6 +74,56 @@ impl Domain { StepHandle { handle, rx } } + pub(crate) fn update_producer(&self) -> UpdateProducerHandle { + let (tx, rx) = mpsc::unbounded_channel(); + + let handle = UpdateProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); + + StepHandle { handle, rx } + } + + pub fn relay_producer_for( + &self, + replica: &Replica, + replica_of: &str, + ) -> RelayProducerHandle { + let (tx, rx) = mpsc::unbounded_channel(); + + let handle = RelayProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + StepHandle { handle, rx } + } + + pub(crate) fn relay_producers(&self) -> HashMap<&str, RelayProducerHandle> { + self.replicas() + .iter() + .map(|(network, replica)| { + let producer = self.relay_producer_for(replica, network); + (network.as_str(), producer) + }) + .collect() + } + + pub(crate) fn process_producer_for( + &self, + replica: &Replica, + replica_of: &str, + ) -> ProcessProducerHandle { + let (tx, rx) = mpsc::unbounded_channel(); + + let handle = ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + StepHandle { handle, rx } + } + + pub(crate) fn process_producers(&self) -> HashMap<&str, ProcessProducerHandle> { + self.replicas() + .iter() + .map(|(replica_of, replica)| { + let producer = self.process_producer_for(replica, replica_of); + (replica_of.as_str(), producer) + }) + .collect() + } + pub(crate) fn count( &self, incoming: mpsc::UnboundedReceiver>, @@ -115,4 +140,62 @@ impl Domain { StepHandle { handle, rx } } + + pub(crate) fn count_relays( + &self, + metrics: Arc, + ) -> HashMap<&str, BetweenHandle>> { + self.relay_producers() + .into_iter() + .map(|(replica_of, producer)| { + let emitter = format!( + "{:?}", + self.replicas.get(&replica_of.to_owned()).unwrap().address() + ); + let network = &self.network; + let event = "relay"; + tracing::info!( + network = self.name(), + replica = emitter, + event, + "starting relay counter", + ); + + let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); + + let between = self.count(producer.rx, metrics, "relay"); + + (replica_of, between) + }) + .collect() + } + + pub(crate) fn count_processes( + &self, + metrics: Arc, + ) -> HashMap<&str, BetweenHandle>> { + self.process_producers() + .into_iter() + .map(|(replica_of, producer)| { + let emitter = format!( + "{:?}", + self.replicas.get(&replica_of.to_owned()).unwrap().address() + ); + let network = &self.network; + let event = "process"; + tracing::info!( + network = self.name(), + replica = emitter, + event, + "starting process counter", + ); + + let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); + + let between = self.count(producer.rx, metrics, "relay"); + + (replica_of, between) + }) + .collect() + } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 5fc7fdb5..7a0659ac 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -5,16 +5,16 @@ use ethers::{ prelude::{Http, Provider as EthersProvider}, }; -use nomad_ethereum::bindings::home::DispatchFilter; +use nomad_ethereum::bindings::{ + home::{DispatchFilter, UpdateFilter}, + replica::{ProcessFilter, UpdateFilter as RelayFilter}, +}; use nomad_xyz_configuration::{get_builtin, NomadConfig}; use tokio::task::JoinHandle; +use tracing_subscriber::EnvFilter; use crate::{ - annotate::WithMeta, - between::{BetweenEvents, BetweenHandle}, - domain::Domain, - metrics::Metrics, - ArcProvider, ProcessStep, StepHandle, + annotate::WithMeta, between::BetweenHandle, domain::Domain, metrics::Metrics, ArcProvider, }; pub(crate) fn config_from_file() -> Option { @@ -39,6 +39,7 @@ pub(crate) fn config() -> eyre::Result { pub(crate) fn init_tracing() { tracing_subscriber::FmtSubscriber::builder() .json() + .with_env_filter(EnvFilter::from_default_env()) .with_level(true) .init(); } @@ -56,6 +57,7 @@ pub(crate) fn provider_for(config: &NomadConfig, network: &str) -> eyre::Result< .get(network) .and_then(|set| set.iter().next().cloned()) }); + eyre::ensure!( url.is_some(), "Missing Url. Please specify by config or env var." @@ -77,28 +79,23 @@ pub(crate) fn provider_for(config: &NomadConfig, network: &str) -> eyre::Result< } pub(crate) fn monitor() -> eyre::Result { - Monitor::from_config(config()?) + Monitor::from_config(&config()?) } #[derive(Debug)] pub(crate) struct Monitor { - config: NomadConfig, networks: HashMap, metrics: Arc, } impl Monitor { - pub(crate) fn from_config(config: NomadConfig) -> eyre::Result { + pub(crate) fn from_config(config: &NomadConfig) -> eyre::Result { let mut networks = HashMap::new(); for network in config.networks.iter() { - networks.insert(network.to_owned(), Domain::from_config(&config, network)?); + networks.insert(network.to_owned(), Domain::from_config(config, network)?); } let metrics = Metrics::new()?.into(); - Ok(Monitor { - config, - networks, - metrics, - }) + Ok(Monitor { networks, metrics }) } pub(crate) fn run_http_server(&self) -> JoinHandle<()> { @@ -114,12 +111,6 @@ impl Monitor { let emitter = format!("{:?}", domain.home().address()); let event = "dispatch"; - tracing::info!( - network = chain.as_str(), - home = emitter, - event, - "starting dispatch counter", - ); let metrics = self.metrics.between_metrics(chain, event, &emitter, None); let producer = domain.dispatch_producer(); @@ -129,4 +120,46 @@ impl Monitor { }) .collect() } + + pub(crate) fn run_between_update( + &self, + ) -> HashMap<&str, BetweenHandle>> { + self.networks + .iter() + .map(|(chain, domain)| { + let emitter = format!("{:?}", domain.home().address()); + let event = "update"; + + let metrics = self.metrics.between_metrics(chain, event, &emitter, None); + + let producer = domain.update_producer(); + let between = domain.count(producer.rx, metrics, event); + + (chain.as_str(), between) + }) + .collect() + } + + pub(crate) fn run_between_relay( + &self, + ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { + self.networks + .iter() + .map(|(network, domain)| (network.as_str(), domain.count_relays(self.metrics.clone()))) + .collect() + } + + pub(crate) fn run_between_process( + &self, + ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { + self.networks + .iter() + .map(|(network, domain)| { + ( + network.as_str(), + domain.count_processes(self.metrics.clone()), + ) + }) + .collect() + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 701dd15b..a199ffb7 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -3,7 +3,7 @@ use std::{panic, sync::Arc}; use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; use tracing::{debug_span, info_span, instrument::Instrumented, Instrument}; -use ethers::prelude::{ContractError, Http, Provider as EthersProvider}; +use ethers::prelude::{Http, Provider as EthersProvider}; pub(crate) mod annotate; pub(crate) mod between; @@ -15,7 +15,7 @@ pub(crate) mod producer; pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; -pub(crate) type ProviderError = ContractError; +// pub(crate) type ProviderError = ContractError; #[tokio::main] async fn main() -> eyre::Result<()> { @@ -29,9 +29,12 @@ async fn main() -> eyre::Result<()> { let _http = monitor.run_http_server(); let dispatch_trackers = monitor.run_between_dispatch(); + let _update_counters = monitor.run_between_update(); + let _relay_counters = monitor.run_between_relay(); + let _process_counters = monitor.run_between_process(); // should cause it to run until crashes occur - dispatch_trackers.into_iter().next().unwrap().1.handle.await; + let _ = dispatch_trackers.into_iter().next().unwrap().1.handle.await; } Ok(()) } diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index faf50e6d..ac89c2f6 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -40,7 +40,7 @@ impl Metrics { let event_blocks = HistogramVec::new( HistogramOpts::new( "inter_event_blocks", - "Blocks between events, as marked by the chain timestamp", + "Blocks between events, as marked by the chain (i.e. 0 means same block, 1 means next block, etc)", ) .namespace("nomad") .const_label("VERSION", env!("CARGO_PKG_VERSION")), @@ -111,63 +111,52 @@ impl Metrics { pub(crate) fn event_counter( &self, - chain: &str, + network: &str, event: &str, emitter: &str, replica_of: Option<&str>, ) -> IntCounter { - self.counts.with_label_values(&[ - chain.as_ref(), - event.as_ref(), - emitter.as_ref(), - replica_of.unwrap_or("n/a"), - ]) + self.counts + .with_label_values(&[network, event, emitter, replica_of.unwrap_or("n/a")]) } pub(crate) fn wallclock_latency( &self, - chain: &str, + network: &str, event: &str, emitter: &str, replica_of: Option<&str>, ) -> Histogram { - // &["chain", "event", "emitter", "replica_of"], - self.wallclock_times.with_label_values(&[ - chain.as_ref(), - event.as_ref(), - emitter.as_ref(), + network, + event, + emitter, replica_of.unwrap_or("n/a"), ]) } pub(crate) fn block_latency( &self, - chain: &str, + network: &str, event: &str, emitter: &str, replica_of: Option<&str>, ) -> Histogram { - // &["chain", "event", "emitter", "replica_of"], - self.event_blocks.with_label_values(&[ - chain.as_ref(), - event.as_ref(), - emitter.as_ref(), - replica_of.unwrap_or("n/a"), - ]) + self.event_blocks + .with_label_values(&[network, event, emitter, replica_of.unwrap_or("n/a")]) } pub(crate) fn between_metrics( &self, - chain: &str, + network: &str, event: &str, emitter: &str, replica_of: Option<&str>, ) -> BetweenMetrics { BetweenMetrics { - count: self.event_counter(chain, event, emitter, replica_of), - wallclock_latency: self.wallclock_latency(chain, event, emitter, replica_of), - block_latency: self.block_latency(chain, event, emitter, replica_of), + count: self.event_counter(network, event, emitter, replica_of), + wallclock_latency: self.wallclock_latency(network, event, emitter, replica_of), + block_latency: self.block_latency(network, event, emitter, replica_of), } } } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 733cb199..4983d346 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -2,13 +2,10 @@ use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHand use ethers::prelude::Middleware; use nomad_ethereum::bindings::{ - home::{DispatchFilter, Home, UpdateFilter as HomeUpdateFilter}, - replica::{ProcessFilter, Replica, UpdateFilter as ReplicaUpdateFilter}, -}; -use tokio::{ - sync::mpsc::{UnboundedReceiver, UnboundedSender}, - task::JoinHandle, + home::{DispatchFilter, Home, UpdateFilter}, + replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; +use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, Instrument}; #[derive(Debug)] @@ -21,15 +18,19 @@ pub(crate) struct DispatchProducer { impl DispatchProducer { pub(crate) fn new( home: Home, - network: String, + network: impl AsRef, tx: UnboundedSender>, ) -> Self { - Self { home, network, tx } + Self { + home, + network: network.as_ref().to_owned(), + tx, + } } } pub(crate) type DispatchProducerTask = Restartable; -pub(crate) type DispatchProducerHandle = StepHandle; +pub(crate) type DispatchProducerHandle = StepHandle>; impl ProcessStep> for DispatchProducer { fn spawn(self) -> DispatchProducerTask { @@ -72,3 +73,210 @@ impl ProcessStep> for DispatchProducer { .instrument(span) } } + +#[derive(Debug)] +pub(crate) struct UpdateProducer { + home: Home, + network: String, + tx: UnboundedSender>, +} + +impl UpdateProducer { + pub(crate) fn new( + home: Home, + network: impl AsRef, + tx: UnboundedSender>, + ) -> Self { + Self { + home, + network: network.as_ref().to_owned(), + tx, + } + } +} + +pub(crate) type UpdateProducerTask = Restartable; +pub(crate) type UpdateProducerHandle = StepHandle>; + +impl ProcessStep> for UpdateProducer { + fn spawn(self) -> UpdateProducerTask { + let span = info_span!( + "UpdateProducer", + home = format!("{:?}", self.home.address()), + network = self.network.as_str(), + event = "update", + ); + + tokio::spawn(async move { + let provider = self.home.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + if from < to { + let res = self + .home + .update_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + task_bail_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + task_bail_if!(res.is_err(), self, res.unwrap_err()); + } + } + let tip = provider.get_block_number().await.unwrap() - 5; + from = to; + to = std::cmp::min(to, tip); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + }) + .instrument(span) + } +} + +#[derive(Debug)] +pub(crate) struct RelayProducer { + replica: Replica, + network: String, + replica_of: String, + tx: UnboundedSender>, +} + +impl RelayProducer { + pub(crate) fn new( + replica: Replica, + network: impl AsRef, + replica_of: impl AsRef, + tx: UnboundedSender>, + ) -> Self { + Self { + replica, + network: network.as_ref().to_owned(), + replica_of: replica_of.as_ref().to_owned(), + tx, + } + } +} + +pub(crate) type RelayProducerTask = Restartable; +pub(crate) type RelayProducerHandle = StepHandle>; + +impl ProcessStep> for RelayProducer { + fn spawn(self) -> RelayProducerTask { + let span = info_span!( + "RelayProducer", + replica = format!("{:?}", self.replica.address()), + network = self.network.as_str(), + event = "relay", + replica_of = self.replica_of.as_str() + ); + + tokio::spawn(async move { + let provider = self.replica.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); + if from < to { + let res = self + .replica + .update_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + task_bail_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + task_bail_if!(res.is_err(), self, res.unwrap_err()); + } + } + let tip = provider.get_block_number().await.unwrap() - 5; + from = to; + to = std::cmp::max(to, tip); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + }) + .instrument(span) + } +} + +#[derive(Debug)] +pub(crate) struct ProcessProducer { + replica: Replica, + network: String, + replica_of: String, + tx: UnboundedSender>, +} + +impl ProcessProducer { + pub(crate) fn new( + replica: Replica, + network: impl AsRef, + replica_of: impl AsRef, + tx: UnboundedSender>, + ) -> Self { + Self { + replica, + network: network.as_ref().to_owned(), + replica_of: replica_of.as_ref().to_owned(), + tx, + } + } +} + +pub(crate) type ProcessProducerTask = Restartable; +pub(crate) type ProcessProducerHandle = StepHandle>; + +impl ProcessStep> for ProcessProducer { + fn spawn(self) -> ProcessProducerTask { + let span = info_span!( + "ProcessProducer", + replica = format!("{:?}", self.replica.address()), + network = self.network.as_str(), + event = "process", + replica_of = self.replica_of.as_str(), + ); + + tokio::spawn(async move { + let provider = self.replica.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + if from < to { + let res = self + .replica + .process_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + task_bail_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + task_bail_if!(res.is_err(), self, res.unwrap_err()); + } + } + let tip = provider.get_block_number().await.unwrap() - 5; + from = to; + to = std::cmp::max(to, tip); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + }) + .instrument(span) + } +} diff --git a/configuration/build.rs b/configuration/build.rs index 9cca3c56..65a757a0 100644 --- a/configuration/build.rs +++ b/configuration/build.rs @@ -100,18 +100,18 @@ async fn main() -> eyre::Result<()> { "cargo:rerun-if-changed={}", concat!(env!("CARGO_MANIFEST_DIR"), "/data/types.rs") ); - println!( - "cargo:rerun-if-changed={}", - concat!(env!("CARGO_MANIFEST_DIR"), "/configs/production.json") - ); - println!( - "cargo:rerun-if-changed={}", - concat!(env!("CARGO_MANIFEST_DIR"), "/configs/development.json") - ); - println!( - "cargo:rerun-if-changed={}", - concat!(env!("CARGO_MANIFEST_DIR"), "/configs/staging.json") - ); + // println!( + // "cargo:rerun-if-changed={}", + // concat!(env!("CARGO_MANIFEST_DIR"), "/configs/production.json") + // ); + // println!( + // "cargo:rerun-if-changed={}", + // concat!(env!("CARGO_MANIFEST_DIR"), "/configs/development.json") + // ); + // println!( + // "cargo:rerun-if-changed={}", + // concat!(env!("CARGO_MANIFEST_DIR"), "/configs/staging.json") + // ); gen_wasm_bindgen()?; // don't re-fetch configs if programmer disables build From 4371025bbfc1c6849018156c68d2a66275062a47 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 17:36:33 -0700 Subject: [PATCH 12/80] bug: fix hardcoded event name --- agents/monitor/src/domain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 30da1a01..99d21053 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -163,7 +163,7 @@ impl Domain { let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(producer.rx, metrics, "relay"); + let between = self.count(producer.rx, metrics, event); (replica_of, between) }) @@ -192,7 +192,7 @@ impl Domain { let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(producer.rx, metrics, "relay"); + let between = self.count(producer.rx, metrics, event); (replica_of, between) }) From 4a5a7ef586fd28142b612ea76c2523c0ffdf3559 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Jul 2022 19:11:53 -0700 Subject: [PATCH 13/80] wip: dispatch_wait --- agents/monitor/src/between.rs | 85 +++++----- agents/monitor/src/dispatch_wait.rs | 67 ++++++++ agents/monitor/src/domain.rs | 4 +- agents/monitor/src/init.rs | 6 +- agents/monitor/src/macros.rs | 5 +- agents/monitor/src/main.rs | 23 +-- agents/monitor/src/producer.rs | 239 +++++++++++++++------------- 7 files changed, 265 insertions(+), 164 deletions(-) create mode 100644 agents/monitor/src/dispatch_wait.rs diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 4b46dbaa..2ccb755d 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -2,7 +2,7 @@ use tracing::{info_span, Instrument}; use tokio::sync::mpsc; -use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHandle}; +use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, @@ -65,47 +65,50 @@ where event = self.event.as_str() ); - tokio::spawn(async move { - let mut last_block_number = 0; - let mut wallclock_latency = self.metrics.wallclock_latency.start_timer(); - - loop { - // get the next event from the channel - let incoming = self.incoming.recv().await; - task_bail_if!(incoming.is_none(), self, "inbound channel broke"); - - let incoming = incoming.unwrap(); - tracing::debug!( - target: "monitor::between", - block_number = %incoming.meta.block_number, - event = self.event.as_str(), - "received incoming event" - ); - - // calculate the blockchain-reported latency in seconds - let block_number = incoming.meta.block_number.as_u64(); - let event_latency = block_number.saturating_sub(last_block_number); - last_block_number = block_number; - - if event_latency != last_block_number { - self.metrics.block_latency.observe(event_latency as f64); + tokio::spawn( + async move { + let mut last_block_number = 0; + let mut wallclock_latency = self.metrics.wallclock_latency.start_timer(); + + loop { + // get the next event from the channel + let incoming = self.incoming.recv().await; + + bail_task_if!(incoming.is_none(), self, "inbound channel broke"); + let incoming = incoming.expect("checked on previous line"); + + tracing::debug!( + target: "monitor::between", + block_number = %incoming.meta.block_number, + event = self.event.as_str(), + "received incoming event" + ); + + // calculate the blockchain-reported latency in seconds + let block_number = incoming.meta.block_number.as_u64(); + let event_latency = block_number.saturating_sub(last_block_number); + last_block_number = block_number; + + if event_latency != last_block_number { + self.metrics.block_latency.observe(event_latency as f64); + } + + // update our metrics + self.metrics.count.inc(); + wallclock_latency.observe_duration(); + + // send the next event out + bail_task_if!( + self.outgoing.send(incoming).is_err(), + self, + "outbound channel broke" + ); + + // restart the timer + wallclock_latency = self.metrics.wallclock_latency.start_timer(); } - - // update our metrics - self.metrics.count.inc(); - wallclock_latency.observe_duration(); - - // send the next event out - task_bail_if!( - self.outgoing.send(incoming).is_err(), - self, - "outbound channel broke" - ); - - // restart the timer - wallclock_latency = self.metrics.wallclock_latency.start_timer(); } - }) - .instrument(span) + .instrument(span), + ) } } diff --git a/agents/monitor/src/dispatch_wait.rs b/agents/monitor/src/dispatch_wait.rs new file mode 100644 index 00000000..7d80a887 --- /dev/null +++ b/agents/monitor/src/dispatch_wait.rs @@ -0,0 +1,67 @@ +use nomad_ethereum::bindings::home::{DispatchFilter, UpdateFilter}; +use tokio::{ + select, + sync::mpsc::{UnboundedReceiver, UnboundedSender}, +}; +use tracing::{info_span, Instrument}; + +use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; + +#[derive(Debug)] +pub(crate) struct DispatchWait { + incoming_dispatch: UnboundedReceiver>, + incoming_update: UnboundedReceiver>, + network: String, + emitter: String, + outgoing_update: UnboundedSender>, +} + +pub(crate) type DispatchWaitTask = Restartable; +pub(crate) type DispatchWaitHandle = StepHandle>; + +impl ProcessStep> for DispatchWait { + fn spawn(mut self) -> DispatchWaitTask + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!( + "DispatchWait", + network = self.network.as_str(), + emitter = self.emitter.as_str(), + ); + let mut dispatches = vec![]; + // TODO: timers vec here + tokio::spawn( + async move { + loop { + select! { + dispatch_next = self.incoming_dispatch.recv() => { + bail_task_if!( + dispatch_next.is_none(), + self, + "inbound dispatch broke" + ); + // TODO: push timer + dispatches.push(dispatch_next.expect("checked in block")); + } + update_opt = self.incoming_update.recv() => { + bail_task_if!( + update_opt.is_none(), + self, + "inbound update broke" + ); + // TODO: close out all timers here + bail_task_if!( + self.outgoing_update.send(update_opt.expect("checked in block")).is_err(), + self, + "outbound update broke" + ); + } + } + todo!() + } + } + .instrument(span), + ) + } +} diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 99d21053..476e5bbb 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -154,7 +154,7 @@ impl Domain { ); let network = &self.network; let event = "relay"; - tracing::info!( + tracing::debug!( network = self.name(), replica = emitter, event, @@ -183,7 +183,7 @@ impl Domain { ); let network = &self.network; let event = "process"; - tracing::info!( + tracing::debug!( network = self.name(), replica = emitter, event, diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 7a0659ac..b17080f5 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -63,7 +63,7 @@ pub(crate) fn provider_for(config: &NomadConfig, network: &str) -> eyre::Result< "Missing Url. Please specify by config or env var." ); - let url = url.unwrap(); + let url = url.expect("checked on previous line"); let provider = EthersProvider::::try_from(&url)?; let timelag = config @@ -102,6 +102,7 @@ impl Monitor { self.metrics.clone().run_http_server() } + #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_dispatch( &self, ) -> HashMap<&str, BetweenHandle>> { @@ -121,6 +122,7 @@ impl Monitor { .collect() } + #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_update( &self, ) -> HashMap<&str, BetweenHandle>> { @@ -140,6 +142,7 @@ impl Monitor { .collect() } + #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_relay( &self, ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { @@ -149,6 +152,7 @@ impl Monitor { .collect() } + #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_process( &self, ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index cd1edb6f..bde25f5e 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -1,6 +1,9 @@ // bails a restartable task #[macro_export] -macro_rules! task_bail_if { +macro_rules! bail_task_if { + ($cond:expr, $self:ident, $err:expr,) => { + task_bail_if!($cond, $self, $err) + }; ($cond:expr, $self:ident, $err:expr) => { if $cond { let err = eyre::eyre!($err); diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index a199ffb7..a247fb37 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,12 +1,13 @@ use annotate::WithMeta; use std::{panic, sync::Arc}; use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; -use tracing::{debug_span, info_span, instrument::Instrumented, Instrument}; +use tracing::{debug_span, info_span, Instrument}; use ethers::prelude::{Http, Provider as EthersProvider}; pub(crate) mod annotate; pub(crate) mod between; +pub(crate) mod dispatch_wait; pub(crate) mod domain; pub(crate) mod init; pub(crate) mod macros; @@ -33,13 +34,15 @@ async fn main() -> eyre::Result<()> { let _relay_counters = monitor.run_between_relay(); let _process_counters = monitor.run_between_process(); + tracing::info!("counters started"); + // should cause it to run until crashes occur let _ = dispatch_trackers.into_iter().next().unwrap().1.handle.await; } Ok(()) } -pub type Restartable = Instrumented>; +pub type Restartable = JoinHandle<(Task, eyre::Report)>; pub(crate) struct StepHandle { handle: Restartable, @@ -96,14 +99,16 @@ where { fn spawn(mut self) -> TerminalHandle { let span = debug_span!("Terminal Handler"); - tokio::spawn(async move { - loop { - if self.rx.recv().await.is_none() { - tracing::info!("Upstream broke, shutting down"); - return (self, eyre::eyre!("")); + tokio::spawn( + async move { + loop { + if self.rx.recv().await.is_none() { + tracing::info!("Upstream broke, shutting down"); + return (self, eyre::eyre!("")); + } } } - }) - .instrument(span) + .instrument(span), + ) } } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 4983d346..4027178b 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -1,4 +1,4 @@ -use crate::{annotate::WithMeta, task_bail_if, ProcessStep, Restartable, StepHandle}; +use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; use ethers::prelude::Middleware; use nomad_ethereum::bindings::{ @@ -41,36 +41,41 @@ impl ProcessStep> for DispatchProducer { event = "dispatch", ); - tokio::spawn(async move { - let provider = self.home.client(); - let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; - loop { - if from < to { - let res = self - .home - .dispatch_filter() - .from_block(from) - .to_block(to) - .query_with_meta() - .await; - - task_bail_if!(res.is_err(), self, res.unwrap_err()); - - for event in res.unwrap().into_iter() { - let res = self.tx.send(event.into()); - task_bail_if!(res.is_err(), self, res.unwrap_err()); + tokio::spawn( + async move { + let provider = self.home.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + if from < to { + let res = self + .home + .dispatch_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + bail_task_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + bail_task_if!(res.is_err(), self, res.unwrap_err()); + } } - } - let tip = provider.get_block_number().await.unwrap() - 5; - from = to; - to = std::cmp::min(to, tip); + let tip_res = provider.get_block_number().await; + bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); + + let tip = tip_res.unwrap() - 5; + from = to; + to = std::cmp::min(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } } - }) - .instrument(span) + .instrument(span), + ) } } @@ -107,36 +112,41 @@ impl ProcessStep> for UpdateProducer { event = "update", ); - tokio::spawn(async move { - let provider = self.home.client(); - let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; - loop { - if from < to { - let res = self - .home - .update_filter() - .from_block(from) - .to_block(to) - .query_with_meta() - .await; - - task_bail_if!(res.is_err(), self, res.unwrap_err()); - - for event in res.unwrap().into_iter() { - let res = self.tx.send(event.into()); - task_bail_if!(res.is_err(), self, res.unwrap_err()); + tokio::spawn( + async move { + let provider = self.home.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + if from < to { + let res = self + .home + .update_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + bail_task_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + bail_task_if!(res.is_err(), self, res.unwrap_err()); + } } - } - let tip = provider.get_block_number().await.unwrap() - 5; - from = to; - to = std::cmp::min(to, tip); + let tip_res = provider.get_block_number().await; + bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); + + let tip = tip_res.unwrap() - 5; + from = to; + to = std::cmp::min(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } } - }) - .instrument(span) + .instrument(span), + ) } } @@ -177,37 +187,41 @@ impl ProcessStep> for RelayProducer { replica_of = self.replica_of.as_str() ); - tokio::spawn(async move { - let provider = self.replica.client(); - let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; - loop { - tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); - if from < to { - let res = self - .replica - .update_filter() - .from_block(from) - .to_block(to) - .query_with_meta() - .await; - - task_bail_if!(res.is_err(), self, res.unwrap_err()); - - for event in res.unwrap().into_iter() { - let res = self.tx.send(event.into()); - task_bail_if!(res.is_err(), self, res.unwrap_err()); + tokio::spawn( + async move { + let provider = self.replica.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); + if from < to { + let res = self + .replica + .update_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + bail_task_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + bail_task_if!(res.is_err(), self, res.unwrap_err()); + } } - } - let tip = provider.get_block_number().await.unwrap() - 5; - from = to; - to = std::cmp::max(to, tip); + let tip_res = provider.get_block_number().await; + bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); + let tip = tip_res.unwrap() - 5; + from = to; + to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } } - }) - .instrument(span) + .instrument(span), + ) } } @@ -248,35 +262,40 @@ impl ProcessStep> for ProcessProducer { replica_of = self.replica_of.as_str(), ); - tokio::spawn(async move { - let provider = self.replica.client(); - let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; - loop { - if from < to { - let res = self - .replica - .process_filter() - .from_block(from) - .to_block(to) - .query_with_meta() - .await; - - task_bail_if!(res.is_err(), self, res.unwrap_err()); - - for event in res.unwrap().into_iter() { - let res = self.tx.send(event.into()); - task_bail_if!(res.is_err(), self, res.unwrap_err()); + tokio::spawn( + async move { + let provider = self.replica.client(); + let height = provider.get_block_number().await.unwrap(); + let mut from = height - 10; + let mut to = height - 5; + loop { + if from < to { + let res = self + .replica + .process_filter() + .from_block(from) + .to_block(to) + .query_with_meta() + .await; + + bail_task_if!(res.is_err(), self, res.unwrap_err()); + + for event in res.unwrap().into_iter() { + let res = self.tx.send(event.into()); + bail_task_if!(res.is_err(), self, res.unwrap_err()); + } } - } - let tip = provider.get_block_number().await.unwrap() - 5; - from = to; - to = std::cmp::max(to, tip); + let tip_res = provider.get_block_number().await; + bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let tip = tip_res.unwrap() - 5; + from = to; + to = std::cmp::max(to, tip); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } } - }) - .instrument(span) + .instrument(span), + ) } } From d7de8624f746a9c932ac25fed233b3da472a1dc0 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 19 Jul 2022 10:01:00 -0700 Subject: [PATCH 14/80] fix: broken event loops --- agents/monitor/src/producer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 4027178b..d00777af 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -69,7 +69,7 @@ impl ProcessStep> for DispatchProducer { let tip = tip_res.unwrap() - 5; from = to; - to = std::cmp::min(to, tip); + to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(5)).await; } @@ -140,7 +140,7 @@ impl ProcessStep> for UpdateProducer { let tip = tip_res.unwrap() - 5; from = to; - to = std::cmp::min(to, tip); + to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(5)).await; } From b17b8acc113b1a9baf2a15eb4b639657b58d5f0f Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 09:21:17 -0700 Subject: [PATCH 15/80] feature: update-to-relay wait metrics --- agents/monitor/Cargo.toml | 1 + agents/monitor/src/between.rs | 21 +++- agents/monitor/src/dispatch_wait.rs | 97 +++++++++++++-- agents/monitor/src/domain.rs | 119 ++++++++++++++----- agents/monitor/src/init.rs | 108 +++++++++++++++-- agents/monitor/src/macros.rs | 4 +- agents/monitor/src/main.rs | 99 ++++++++-------- agents/monitor/src/metrics.rs | 54 ++++++++- agents/monitor/src/producer.rs | 85 ++++++++++++-- agents/monitor/src/terminal.rs | 47 ++++++++ agents/monitor/src/update_wait.rs | 175 ++++++++++++++++++++++++++++ agents/monitor/src/utils.rs | 118 +++++++++++++++++++ 12 files changed, 812 insertions(+), 116 deletions(-) create mode 100644 agents/monitor/src/terminal.rs create mode 100644 agents/monitor/src/update_wait.rs create mode 100644 agents/monitor/src/utils.rs diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index fe34f060..4a136274 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -19,3 +19,4 @@ eyre = "0.6.8" nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } warp = "0.3.2" async-trait = "0.1.56" +futures-util = "0.3.21" diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 2ccb755d..0ac17d2d 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -1,6 +1,6 @@ use tracing::{info_span, Instrument}; -use tokio::sync::mpsc; +use tokio::sync::mpsc::{self, UnboundedReceiver}; use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; @@ -16,9 +16,20 @@ pub(crate) struct BetweenEvents { pub(crate) metrics: BetweenMetrics, pub(crate) network: String, pub(crate) event: String, + pub(crate) emitter: String, pub(crate) outgoing: mpsc::UnboundedSender, } +impl std::fmt::Display for BetweenEvents { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "BetweenEvents - network {}'s {} @ {}", + self.network, self.event, self.emitter + ) + } +} + impl std::fmt::Debug for BetweenEvents { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("BetweenEvents") @@ -38,6 +49,7 @@ where metrics: BetweenMetrics, network: impl AsRef, event: impl AsRef, + emitter: impl AsRef, outgoing: mpsc::UnboundedSender, ) -> Self { Self { @@ -45,18 +57,21 @@ where metrics, network: network.as_ref().to_owned(), event: event.as_ref().to_owned(), + emitter: emitter.as_ref().to_owned(), outgoing, } } } -pub(crate) type BetweenHandle = StepHandle, T>; +pub(crate) type BetweenHandle = StepHandle>; pub(crate) type BetweenTask = Restartable>; -impl ProcessStep> for BetweenEvents> +impl ProcessStep for BetweenEvents> where T: 'static + Send + Sync + std::fmt::Debug, { + type Output = UnboundedReceiver>; + fn spawn(mut self) -> BetweenTask> { let span = info_span!( target: "monitor::between", diff --git a/agents/monitor/src/dispatch_wait.rs b/agents/monitor/src/dispatch_wait.rs index 7d80a887..49f795d9 100644 --- a/agents/monitor/src/dispatch_wait.rs +++ b/agents/monitor/src/dispatch_wait.rs @@ -1,4 +1,5 @@ use nomad_ethereum::bindings::home::{DispatchFilter, UpdateFilter}; +use prometheus::Histogram; use tokio::{ select, sync::mpsc::{UnboundedReceiver, UnboundedSender}, @@ -7,19 +8,67 @@ use tracing::{info_span, Instrument}; use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; +#[derive(Debug)] +pub(crate) struct DispatchWaitMetrics { + pub(crate) timer: Histogram, + pub(crate) blocks: Histogram, +} + #[derive(Debug)] pub(crate) struct DispatchWait { incoming_dispatch: UnboundedReceiver>, incoming_update: UnboundedReceiver>, network: String, emitter: String, + metrics: DispatchWaitMetrics, outgoing_update: UnboundedSender>, + outgoing_dispatch: UnboundedSender>, +} + +impl std::fmt::Display for DispatchWait { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "DispatchToUpdate latency for {}'s home @ {}", + self.network, self.emitter, + ) + } +} + +impl DispatchWait { + pub(crate) fn new( + incoming_dispatch: UnboundedReceiver>, + incoming_update: UnboundedReceiver>, + network: String, + emitter: String, + metrics: DispatchWaitMetrics, + outgoing_update: UnboundedSender>, + outgoing_dispatch: UnboundedSender>, + ) -> Self { + Self { + incoming_dispatch, + incoming_update, + network, + emitter, + metrics, + outgoing_update, + outgoing_dispatch, + } + } } pub(crate) type DispatchWaitTask = Restartable; -pub(crate) type DispatchWaitHandle = StepHandle>; +pub(crate) type DispatchWaitHandle = StepHandle; + +#[derive(Debug)] +pub struct DispatchWaitOutput { + pub(crate) dispatches: UnboundedReceiver>, + pub(crate) updates: UnboundedReceiver>, +} + +impl ProcessStep for DispatchWait { + type Output = DispatchWaitOutput; -impl ProcessStep> for DispatchWait { fn spawn(mut self) -> DispatchWaitTask where Self: 'static + Send + Sync + Sized, @@ -29,20 +78,43 @@ impl ProcessStep> for DispatchWait { network = self.network.as_str(), emitter = self.emitter.as_str(), ); - let mut dispatches = vec![]; - // TODO: timers vec here + + let mut timers = vec![]; + let mut blocks = vec![]; + tokio::spawn( async move { loop { + // how this works: + // For each dispatch, we mark its block height and start a + // timer. + // Every time an update comes in, we observe all timers, and + // then observe all the interblock periods. + // + // We always send the event onwards before making local + // observations, to ensure that the next step gets it + // reasonably early + select! { + // cause the select block to always poll dispatches + // first. i.e. ready dispatches will arrive first + biased; + dispatch_next = self.incoming_dispatch.recv() => { bail_task_if!( dispatch_next.is_none(), self, "inbound dispatch broke" ); - // TODO: push timer - dispatches.push(dispatch_next.expect("checked in block")); + let dispatch = dispatch_next.expect("checked in block"); + let block_number = dispatch.meta.block_number; + bail_task_if!( + self.outgoing_dispatch.send(dispatch).is_err(), + self, + "outbound dispatch broke" + ); + timers.push(self.metrics.timer.start_timer()); + blocks.push(block_number); } update_opt = self.incoming_update.recv() => { bail_task_if!( @@ -50,15 +122,22 @@ impl ProcessStep> for DispatchWait { self, "inbound update broke" ); - // TODO: close out all timers here + let update = update_opt.expect("checked in block"); + let block_number = update.meta.block_number; + bail_task_if!( - self.outgoing_update.send(update_opt.expect("checked in block")).is_err(), + self.outgoing_update.send(update).is_err(), self, "outbound update broke" ); + // drain the entire vec + timers.drain(0..).for_each(|timer| timer.observe_duration()); + blocks.drain(0..).for_each(|dispatch_height| { + let diff = block_number.saturating_sub(dispatch_height); + self.metrics.blocks.observe(diff.as_u64() as f64); + }); } } - todo!() } } .instrument(span), diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 476e5bbb..fc990d48 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use nomad_ethereum::bindings::{ - home::Home, + home::{DispatchFilter, Home, UpdateFilter}, replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; @@ -10,6 +10,7 @@ use tokio::sync::mpsc; use crate::{ annotate::WithMeta, between::{BetweenEvents, BetweenHandle, BetweenMetrics}, + dispatch_wait::{DispatchWait, DispatchWaitHandle, DispatchWaitOutput}, init::provider_for, metrics::Metrics, producer::{ @@ -27,6 +28,10 @@ pub(crate) struct Domain { } impl Domain { + pub(crate) fn home_address(&self) -> String { + format!("{:?}", self.home.address()) + } + pub(crate) fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { let network = network.to_owned(); let provider = provider_for(config, &network)?; @@ -135,67 +140,125 @@ impl Domain { { let network = self.network.clone(); let (tx, rx) = mpsc::unbounded_channel(); + let emitter = self.home_address(); + + tracing::debug!( + network = network, + home = emitter, + event = event.as_ref(), + "starting counter", + ); let handle = - BetweenEvents::>::new(incoming, metrics, network, event, tx).spawn(); + BetweenEvents::>::new(incoming, metrics, network, event, emitter, tx) + .spawn(); StepHandle { handle, rx } } + pub(crate) fn count_dispatches( + &self, + incoming: mpsc::UnboundedReceiver>, + metrics: BetweenMetrics, + event: impl AsRef, + ) -> BetweenHandle> { + self.count(incoming, metrics, event) + } + + pub(crate) fn count_updates( + &self, + incoming: mpsc::UnboundedReceiver>, + metrics: BetweenMetrics, + event: impl AsRef, + ) -> BetweenHandle> { + self.count(incoming, metrics, event) + } + pub(crate) fn count_relays( &self, + mut incomings: HashMap<&str, mpsc::UnboundedReceiver>>, metrics: Arc, ) -> HashMap<&str, BetweenHandle>> { - self.relay_producers() - .into_iter() - .map(|(replica_of, producer)| { - let emitter = format!( - "{:?}", - self.replicas.get(&replica_of.to_owned()).unwrap().address() - ); - let network = &self.network; + self.replicas + .iter() + .map(|(replica_of, replica)| { + let emitter = format!("{:?}", replica.address()); + let network = self.name(); let event = "relay"; tracing::debug!( - network = self.name(), + network = network, replica = emitter, event, - "starting relay counter", + "starting counter", ); - + let incoming = incomings + .remove(replica_of.as_str()) + .expect("Missing channel"); let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(producer.rx, metrics, event); + let between = self.count(incoming, metrics, event); - (replica_of, between) + (replica_of.as_str(), between) }) .collect() } pub(crate) fn count_processes( &self, + mut incomings: HashMap<&str, mpsc::UnboundedReceiver>>, metrics: Arc, ) -> HashMap<&str, BetweenHandle>> { - self.process_producers() - .into_iter() - .map(|(replica_of, producer)| { - let emitter = format!( - "{:?}", - self.replicas.get(&replica_of.to_owned()).unwrap().address() - ); - let network = &self.network; + self.replicas + .iter() + .map(|(replica_of, replica)| { + let emitter = format!("{:?}", replica.address()); + let network = self.name(); let event = "process"; tracing::debug!( - network = self.name(), + network = network, replica = emitter, event, - "starting process counter", + "starting counter", ); - + let incoming = incomings + .remove(replica_of.as_str()) + .expect("Missing channel"); let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(producer.rx, metrics, event); + let between = self.count(incoming, metrics, event); - (replica_of, between) + (replica_of.as_str(), between) }) .collect() } + + pub(crate) fn dispatch_to_update( + &self, + incoming_dispatch: mpsc::UnboundedReceiver>, + incoming_update: mpsc::UnboundedReceiver>, + metrics: Arc, + ) -> DispatchWaitHandle { + let metrics = metrics.dispatch_wait_metrics(&self.network, &self.home_address()); + + let (outgoing_update, updates) = mpsc::unbounded_channel(); + let (outgoing_dispatch, dispatches) = mpsc::unbounded_channel(); + + let handle = DispatchWait::new( + incoming_dispatch, + incoming_update, + self.name().to_owned(), + self.home_address(), + metrics, + outgoing_update, + outgoing_dispatch, + ) + .spawn(); + + DispatchWaitHandle { + handle, + rx: DispatchWaitOutput { + dispatches, + updates, + }, + } + } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index b17080f5..378d2ff4 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -10,11 +10,19 @@ use nomad_ethereum::bindings::{ replica::{ProcessFilter, UpdateFilter as RelayFilter}, }; use nomad_xyz_configuration::{get_builtin, NomadConfig}; -use tokio::task::JoinHandle; +use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; use tracing_subscriber::EnvFilter; use crate::{ - annotate::WithMeta, between::BetweenHandle, domain::Domain, metrics::Metrics, ArcProvider, + annotate::WithMeta, + between::BetweenHandle, + dispatch_wait::DispatchWaitHandle, + domain::Domain, + metrics::Metrics, + producer::{ + DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, + }, + ArcProvider, HomeReplicaMap, }; pub(crate) fn config_from_file() -> Option { @@ -38,7 +46,7 @@ pub(crate) fn config() -> eyre::Result { pub(crate) fn init_tracing() { tracing_subscriber::FmtSubscriber::builder() - .json() + .pretty() .with_env_filter(EnvFilter::from_default_env()) .with_level(true) .init(); @@ -102,20 +110,50 @@ impl Monitor { self.metrics.clone().run_http_server() } + pub(crate) fn run_dispatch_producers(&self) -> HashMap<&str, DispatchProducerHandle> { + self.networks + .iter() + .map(|(network, domain)| (network.as_str(), domain.dispatch_producer())) + .collect() + } + + pub(crate) fn run_update_producers(&self) -> HashMap<&str, UpdateProducerHandle> { + self.networks + .iter() + .map(|(network, domain)| (network.as_str(), domain.update_producer())) + .collect() + } + + pub(crate) fn run_relay_producers(&self) -> HomeReplicaMap { + self.networks + .iter() + .map(|(network, domain)| (network.as_str(), domain.relay_producers())) + .collect() + } + + pub(crate) fn run_process_producers(&self) -> HomeReplicaMap { + self.networks + .iter() + .map(|(network, domain)| (network.as_str(), domain.process_producers())) + .collect() + } + #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_dispatch( &self, + mut incomings: HashMap<&str, UnboundedReceiver>>, ) -> HashMap<&str, BetweenHandle>> { self.networks .iter() .map(|(chain, domain)| { - let emitter = format!("{:?}", domain.home().address()); + let emitter = domain.home_address(); let event = "dispatch"; let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = domain.dispatch_producer(); - let between = domain.count(producer.rx, metrics, event); + let producer = incomings.remove(chain.as_str()).expect("missing producer"); + + let between = domain.count_dispatches(producer, metrics, event); (chain.as_str(), between) }) @@ -125,6 +163,7 @@ impl Monitor { #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_update( &self, + mut incomings: HashMap<&str, UnboundedReceiver>>, ) -> HashMap<&str, BetweenHandle>> { self.networks .iter() @@ -134,8 +173,9 @@ impl Monitor { let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = domain.update_producer(); - let between = domain.count(producer.rx, metrics, event); + let producer = incomings.remove(chain.as_str()).expect("missing producer"); + + let between = domain.count_updates(producer, metrics, event); (chain.as_str(), between) }) @@ -145,25 +185,69 @@ impl Monitor { #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_relay( &self, - ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { + mut incomings: HomeReplicaMap>>, + ) -> HomeReplicaMap>> { self.networks .iter() - .map(|(network, domain)| (network.as_str(), domain.count_relays(self.metrics.clone()))) + .map(|(network, domain)| { + let incomings = incomings + .remove(network.as_str()) + .expect("missing producer") + .into_iter() + .map(|(k, v)| (k, v)) + .collect(); + ( + network.as_str(), + domain.count_relays(incomings, self.metrics.clone()), + ) + }) .collect() } #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_between_process( &self, - ) -> HashMap<&str, HashMap<&str, BetweenHandle>>> { + mut incomings: HomeReplicaMap>>, + ) -> HomeReplicaMap>> { self.networks .iter() .map(|(network, domain)| { + let incomings = incomings + .remove(network.as_str()) + .expect("missing producer") + .into_iter() + .collect(); ( network.as_str(), - domain.count_processes(self.metrics.clone()), + domain.count_processes(incomings, self.metrics.clone()), ) }) .collect() } + + #[tracing::instrument(skip_all, level = "debug")] + pub(crate) fn run_dispatch_to_update( + &self, + mut incoming_dispatches: HashMap<&str, UnboundedReceiver>>, + mut incoming_updates: HashMap<&str, UnboundedReceiver>>, + ) -> HashMap<&str, DispatchWaitHandle> { + self.networks + .iter() + .map(|(network, domain)| { + let incoming_dispatch = incoming_dispatches + .remove(network.as_str()) + .expect("missing incoming dispatch"); + let incoming_update = incoming_updates + .remove(network.as_str()) + .expect("missing incoming update"); + + let d_to_r = domain.dispatch_to_update( + incoming_dispatch, + incoming_update, + self.metrics.clone(), + ); + (network.as_str(), d_to_r) + }) + .collect() + } } diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index bde25f5e..3d0d1d6a 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -2,12 +2,12 @@ #[macro_export] macro_rules! bail_task_if { ($cond:expr, $self:ident, $err:expr,) => { - task_bail_if!($cond, $self, $err) + bail_task_if!($cond, $self, $err) }; ($cond:expr, $self:ident, $err:expr) => { if $cond { let err = eyre::eyre!($err); - tracing::error!(task = ?$self, err = %err, "Task failed"); + tracing::error!(task = %$self, err = %err, "Task failed"); return ($self, err); } }; diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index a247fb37..90c7d753 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,7 +1,6 @@ -use annotate::WithMeta; -use std::{panic, sync::Arc}; -use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; -use tracing::{debug_span, info_span, Instrument}; +use std::{collections::HashMap, panic, sync::Arc}; +use tokio::task::JoinHandle; +use tracing::info_span; use ethers::prelude::{Http, Provider as EthersProvider}; @@ -13,11 +12,16 @@ pub(crate) mod init; pub(crate) mod macros; pub(crate) mod metrics; pub(crate) mod producer; +pub(crate) mod terminal; +pub(crate) mod update_wait; +pub(crate) mod utils; pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; // pub(crate) type ProviderError = ContractError; +pub(crate) type HomeReplicaMap<'a, T> = HashMap<&'a str, HashMap<&'a str, T>>; + #[tokio::main] async fn main() -> eyre::Result<()> { init::init_tracing(); @@ -29,35 +33,63 @@ async fn main() -> eyre::Result<()> { tracing::info!("setup complete!"); let _http = monitor.run_http_server(); - let dispatch_trackers = monitor.run_between_dispatch(); - let _update_counters = monitor.run_between_update(); - let _relay_counters = monitor.run_between_relay(); - let _process_counters = monitor.run_between_process(); + let dispatch_producer = monitor.run_dispatch_producers(); + let update_producer = monitor.run_update_producers(); + let relay_producers = monitor.run_relay_producers(); + let process_producers = monitor.run_process_producers(); + + let (dispatch_handles, dispatch_producer) = utils::split(dispatch_producer); + let (update_handles, update_producer) = utils::split(update_producer); + let (relay_handles, relay_producers) = utils::nested_split(relay_producers); + let (process_handles, process_producers) = utils::nested_split(process_producers); + + let dispatch_counters = monitor.run_between_dispatch(dispatch_producer); + let update_counters = monitor.run_between_update(update_producer); + let relay_counters = monitor.run_between_relay(relay_producers); + let process_counters = monitor.run_between_process(process_producers); + + let (dispatch_count_handles, dispatch_producer) = utils::split(dispatch_counters); + let (update_count_handles, update_producer) = utils::split(update_counters); + let (relay_count_handles, relay_producer) = utils::nested_split(relay_counters); + let (process_count_handles, process_producer) = utils::nested_split(process_counters); + + let d_to_u = monitor.run_dispatch_to_update(dispatch_producer, update_producer); + + let (d_to_u_handles, d_and_u_producers) = utils::split(d_to_u); tracing::info!("counters started"); // should cause it to run until crashes occur - let _ = dispatch_trackers.into_iter().next().unwrap().1.handle.await; + let _ = update_handles.into_iter().next().unwrap().await; } Ok(()) } pub type Restartable = JoinHandle<(Task, eyre::Report)>; -pub(crate) struct StepHandle { +/// A step handle is the handle to the process, and its outbound channels. +/// +/// Task creation reutns a step handle so that we can +/// - track the +pub(crate) struct StepHandle +where + Task: ProcessStep, +{ handle: Restartable, - rx: UnboundedReceiver, + rx: ::Output, } -pub(crate) trait ProcessStep -where - T: 'static + Send + Sync + std::fmt::Debug, -{ +pub(crate) trait ProcessStep: std::fmt::Display { + type Output: 'static + Send + Sync + std::fmt::Debug; + fn spawn(self) -> Restartable where Self: 'static + Send + Sync + Sized; - fn forever(self) -> JoinHandle<()> + /// Run the task until it panics. Errors result in a task restart with the + /// same channels. This means that an error causes the task to lose only + /// the data that is in-scope when it faults. + fn run_until_panic(self) -> Restartable where Self: 'static + Send + Sync + Sized, { @@ -72,8 +104,8 @@ where handle } Err(e) => { - tracing::error!(err = %e, "JoinError in forever"); - panic!("JoinError in forever"); + tracing::error!(err = %e, "JoinError in forever. Internal task panicked"); + panic!("JoinError in forever. Internal task panicked"); } }; handle = again.spawn() @@ -81,34 +113,3 @@ where }) } } - -/// A process step that just drains its input and drops everything -/// Its [`StepHandle`] will never produce values. -pub(crate) struct Terminal -where - T: std::fmt::Debug, -{ - rx: UnboundedReceiver>, -} - -pub(crate) type TerminalHandle = Restartable>; - -impl ProcessStep for Terminal -where - T: 'static + Send + Sync + std::fmt::Debug, -{ - fn spawn(mut self) -> TerminalHandle { - let span = debug_span!("Terminal Handler"); - tokio::spawn( - async move { - loop { - if self.rx.recv().await.is_none() { - tracing::info!("Upstream broke, shutting down"); - return (self, eyre::eyre!("")); - } - } - } - .instrument(span), - ) - } -} diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index ac89c2f6..dfd37af2 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -4,13 +4,16 @@ use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, In use tokio::task::JoinHandle; use warp::Filter; -use crate::between::BetweenMetrics; +use crate::{between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics}; #[derive(Debug)] pub(crate) struct Metrics { wallclock_times: prometheus::HistogramVec, event_blocks: prometheus::HistogramVec, - counts: prometheus::IntCounterVec, + event_counts: prometheus::IntCounterVec, + + dispatch_to_update_timers: prometheus::HistogramVec, + dispatch_to_update_blocks: prometheus::HistogramVec, } fn u16_from_env(s: impl AsRef) -> Option { @@ -27,6 +30,26 @@ fn gather() -> prometheus::Result> { impl Metrics { pub(crate) fn new() -> eyre::Result { + let dispatch_to_update_timers = HistogramVec::new( + HistogramOpts::new( + "dispatch_to_update_ms", + "Ms between dispatch and update, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "emitter"], + )?; + + let dispatch_to_update_blocks = HistogramVec::new( + HistogramOpts::new( + "dispatch_to_update_blocks", + "Blocks between dispatch and update, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "emitter"], + )?; + let wallclock_times = HistogramVec::new( HistogramOpts::new( "inter_event_period_wallclock_ms", @@ -67,11 +90,19 @@ impl Metrics { registry .register(Box::new(counts.clone())) .expect("unable to register metric"); + registry + .register(Box::new(dispatch_to_update_timers.clone())) + .expect("unable to register metric"); + registry + .register(Box::new(dispatch_to_update_blocks.clone())) + .expect("unable to register metric"); Ok(Self { wallclock_times, event_blocks, - counts, + event_counts: counts, + dispatch_to_update_blocks, + dispatch_to_update_timers, }) } @@ -116,7 +147,7 @@ impl Metrics { emitter: &str, replica_of: Option<&str>, ) -> IntCounter { - self.counts + self.event_counts .with_label_values(&[network, event, emitter, replica_of.unwrap_or("n/a")]) } @@ -159,4 +190,19 @@ impl Metrics { block_latency: self.block_latency(network, event, emitter, replica_of), } } + + pub(crate) fn dispatch_wait_metrics( + &self, + network: &str, + emitter: &str, + ) -> DispatchWaitMetrics { + DispatchWaitMetrics { + timer: self + .dispatch_to_update_timers + .with_label_values(&[network, emitter]), + blocks: self + .dispatch_to_update_blocks + .with_label_values(&[network, emitter]), + } + } } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index d00777af..2e36f55b 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -5,7 +5,7 @@ use nomad_ethereum::bindings::{ home::{DispatchFilter, Home, UpdateFilter}, replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; -use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; #[derive(Debug)] @@ -15,7 +15,22 @@ pub(crate) struct DispatchProducer { tx: UnboundedSender>, } +impl std::fmt::Display for DispatchProducer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "DispatchProducer - {}'s home @ {}", + self.network, + self.home_address() + ) + } +} + impl DispatchProducer { + pub(crate) fn home_address(&self) -> String { + format!("{:?}", self.home.address()) + } + pub(crate) fn new( home: Home, network: impl AsRef, @@ -30,9 +45,11 @@ impl DispatchProducer { } pub(crate) type DispatchProducerTask = Restartable; -pub(crate) type DispatchProducerHandle = StepHandle>; +pub(crate) type DispatchProducerHandle = StepHandle; + +impl ProcessStep for DispatchProducer { + type Output = UnboundedReceiver>; -impl ProcessStep> for DispatchProducer { fn spawn(self) -> DispatchProducerTask { let span = info_span!( "DispatchProducer", @@ -86,7 +103,22 @@ pub(crate) struct UpdateProducer { tx: UnboundedSender>, } +impl std::fmt::Display for UpdateProducer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "UpdateProducer - {}'s home @ {}", + self.network, + self.home_address() + ) + } +} + impl UpdateProducer { + pub(crate) fn home_address(&self) -> String { + format!("{:?}", self.home.address()) + } + pub(crate) fn new( home: Home, network: impl AsRef, @@ -101,9 +133,10 @@ impl UpdateProducer { } pub(crate) type UpdateProducerTask = Restartable; -pub(crate) type UpdateProducerHandle = StepHandle>; +pub(crate) type UpdateProducerHandle = StepHandle; -impl ProcessStep> for UpdateProducer { +impl ProcessStep for UpdateProducer { + type Output = UnboundedReceiver>; fn spawn(self) -> UpdateProducerTask { let span = info_span!( "UpdateProducer", @@ -158,7 +191,23 @@ pub(crate) struct RelayProducer { tx: UnboundedSender>, } +impl std::fmt::Display for RelayProducer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "RelayProducer - {}'s replica of {} @ {}", + self.network, + self.replica_of, + self.replica_address() + ) + } +} + impl RelayProducer { + pub(crate) fn replica_address(&self) -> String { + format!("{:?}", self.replica.address()) + } + pub(crate) fn new( replica: Replica, network: impl AsRef, @@ -175,9 +224,10 @@ impl RelayProducer { } pub(crate) type RelayProducerTask = Restartable; -pub(crate) type RelayProducerHandle = StepHandle>; +pub(crate) type RelayProducerHandle = StepHandle; -impl ProcessStep> for RelayProducer { +impl ProcessStep for RelayProducer { + type Output = UnboundedReceiver>; fn spawn(self) -> RelayProducerTask { let span = info_span!( "RelayProducer", @@ -233,6 +283,18 @@ pub(crate) struct ProcessProducer { tx: UnboundedSender>, } +impl std::fmt::Display for ProcessProducer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "ProcessProducer for {}'s replica of {} @ {}", + self.network, + self.replica_of, + self.replica_address() + ) + } +} + impl ProcessProducer { pub(crate) fn new( replica: Replica, @@ -247,12 +309,17 @@ impl ProcessProducer { tx, } } + + pub(crate) fn replica_address(&self) -> String { + format!("{:?}", self.replica.address()) + } } pub(crate) type ProcessProducerTask = Restartable; -pub(crate) type ProcessProducerHandle = StepHandle>; +pub(crate) type ProcessProducerHandle = StepHandle; -impl ProcessStep> for ProcessProducer { +impl ProcessStep for ProcessProducer { + type Output = UnboundedReceiver>; fn spawn(self) -> ProcessProducerTask { let span = info_span!( "ProcessProducer", diff --git a/agents/monitor/src/terminal.rs b/agents/monitor/src/terminal.rs new file mode 100644 index 00000000..e0f7f544 --- /dev/null +++ b/agents/monitor/src/terminal.rs @@ -0,0 +1,47 @@ +use tokio::sync::mpsc::UnboundedReceiver; +use tracing::{debug_span, Instrument}; + +use crate::{ProcessStep, Restartable}; + +#[derive(Debug)] +/// A process step that just drains its input and drops everything +/// Its [`StepHandle`] will never produce values. +pub(crate) struct Terminal +where + T: std::fmt::Debug, +{ + pub(crate) rx: UnboundedReceiver, +} + +impl std::fmt::Display for Terminal +where + T: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Terminal") + } +} + +pub(crate) type TerminalHandle = Restartable>; + +impl ProcessStep for Terminal +where + T: std::fmt::Debug + Send + Sync + 'static, +{ + type Output = (); + + fn spawn(mut self) -> TerminalHandle { + let span = debug_span!("Terminal Handler"); + tokio::spawn( + async move { + loop { + if self.rx.recv().await.is_none() { + tracing::info!("Upstream broke, shutting down"); + return (self, eyre::eyre!("")); + } + } + } + .instrument(span), + ) + } +} diff --git a/agents/monitor/src/update_wait.rs b/agents/monitor/src/update_wait.rs new file mode 100644 index 00000000..3b7afca8 --- /dev/null +++ b/agents/monitor/src/update_wait.rs @@ -0,0 +1,175 @@ +use ethers::prelude::H256; +use futures_util::future::select_all; +use prometheus::{Histogram, HistogramTimer}; +use std::{collections::HashMap, fmt::Display, time::Instant}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use tracing::{info_span, Instrument}; + +use nomad_ethereum::bindings::{home::UpdateFilter, replica::UpdateFilter as RelayFilter}; + +use crate::{ + annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, Restartable, StepHandle, +}; + +#[derive(Debug)] +pub(crate) struct UpdateWaitMetrics { + pub(crate) times: Histogram, +} + +#[derive(Debug)] +pub(crate) struct UpdateWait { + incoming_update: UnboundedReceiver>, + incoming_relays: > as ProcessStep>::Output, + + network: String, + metrics: UpdateWaitMetrics, + + updates: HashMap, + relays: HashMap>, + + outgoing_update: UnboundedSender>, + outgoing_relays: HashMap>>, +} + +impl UpdateWait { + pub(crate) fn new( + incoming_update: UnboundedReceiver>, + incoming_relays: HashMap>>, + network: String, + metrics: UpdateWaitMetrics, + outgoing_update: UnboundedSender>, + outgoing_relays: HashMap>>, + ) -> Self { + let (tx, rx) = unbounded_channel(); + + let _ = SelectChannels::new(incoming_relays, tx).spawn(); + Self { + incoming_update, + incoming_relays: rx, + network, + metrics, + updates: Default::default(), + relays: Default::default(), + outgoing_update, + outgoing_relays, + } + } +} + +impl std::fmt::Display for UpdateWait { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "UpdateToRelay - {}", self.network) + } +} + +pub(crate) type UpdateWaitTask = Restartable; +pub(crate) type UpdateWaitHandle = StepHandle; + +#[derive(Debug)] +pub struct UpdateWaitOutput { + pub(crate) updates: UnboundedReceiver>, + pub(crate) relays: HashMap>>, +} + +impl UpdateWait { + fn handle_relay(&mut self, net: &str, root: H256) { + let now = std::time::Instant::now(); + + // mem optimization: don't need to store the relay time + // if we observe immediately + if let Some(update_time) = self.updates.get(&root) { + self.record(now, *update_time) + } else { + self.relays + .entry(root) + .or_default() + .insert(net.to_owned(), now); + } + } + + fn handle_update(&mut self, root: H256) { + let now = std::time::Instant::now(); + self.updates.insert(root, now); + + // mem optimization: remove the hashmap as we no longer need to store + // any future relay times for this root + if let Some(mut relays) = self.relays.remove(&root) { + // times + relays.drain().for_each(|(_, relay)| { + self.record(relay, now); + }) + } + } + + fn record(&self, relay: Instant, update: Instant) { + let v = relay.saturating_duration_since(update); + self.metrics.times.observe(v.as_millis() as f64) + } +} + +impl ProcessStep for UpdateWait { + type Output = UpdateWaitOutput; + + fn spawn(mut self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!("UpdateWait"); + tokio::spawn( + async move { + loop { + tokio::select! { + // how this works: + // For each update, we insert it into the updates map + // We then check the relays map and record differences + // For each relay, we insert it into the relays map + // We then check the updates map and record + + biased; + + update_opt = self.incoming_update.recv() => { + bail_task_if!( + update_opt.is_none(), + self, + "Inbound updates broke", + ); + let update = update_opt.unwrap(); + let root: H256 = update.log.new_root.into(); + bail_task_if!{ + // send onwards + self.outgoing_update.send(update).is_err(), + self, + "Outbound updates broke", + }; + + self.handle_update(root); + } + relay_opt = self.incoming_relays.recv() => { + bail_task_if!( + relay_opt.is_none(), + self, + format!("Inbound relays broke"), + ); + let (net, relay) = relay_opt.unwrap(); + let root: H256 = relay.log.new_root.into(); + + bail_task_if!( + // send onward + self.outgoing_relays + .get(&net) + .expect("missing outgoing") + .send(relay) + .is_err(), + self, + format!("outgoing relay for {} broke", &net) + ); + + self.handle_relay(&net, root); + } + } + } + } + .instrument(span), + ) + } +} diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs new file mode 100644 index 00000000..e88f7eda --- /dev/null +++ b/agents/monitor/src/utils.rs @@ -0,0 +1,118 @@ +use std::{collections::HashMap, fmt::Display, pin::Pin}; + +use futures_util::future::{select_all, Select}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tracing::{info_span, Instrument}; + +use crate::bail_task_if; + +use super::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; + +pub(crate) fn split( + map: HashMap<&str, StepHandle>, +) -> ( + Vec>, + HashMap<&str, ::Output>, +) +where + T: ProcessStep, +{ + let mut handles = vec![]; + let map = map + .into_iter() + .map(|(name, out)| { + handles.push(out.handle); + (name, out.rx) + }) + .collect(); + (handles, map) +} + +pub(crate) fn nested_split( + map: HomeReplicaMap>, +) -> ( + Vec>, + HomeReplicaMap<::Output>, +) +where + T: ProcessStep, +{ + let mut handles = vec![]; + let map = map + .into_iter() + .map(|(name, map)| { + let (mut h, m) = split(map); + handles.append(&mut h); + (name, m) + }) + .collect(); + (handles, map) +} + +// polls all channels in a hashmap +pub(crate) fn nexts( + map: &mut HashMap>, +) -> Vec::Owned, Option)> + '_>>> { + map.iter_mut() + .map(|(k, rx)| { + let k = k.to_owned(); + let fut = rx.recv(); + async move { (k, fut.await) } + }) + .map(Box::pin) + .collect() +} + +#[derive(Debug)] +pub(crate) struct SelectChannels { + channels: HashMap>, + outbound: UnboundedSender<(String, T)>, +} + +impl SelectChannels { + pub(crate) fn new( + channels: HashMap>, + outbound: UnboundedSender<(String, T)>, + ) -> Self { + Self { channels, outbound } + } +} + +impl Display for SelectChannels { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SelectChannels") + } +} + +impl ProcessStep for SelectChannels +where + T: 'static + Send + Sync + std::fmt::Debug, +{ + type Output = UnboundedReceiver<(String, T)>; + + fn spawn(mut self) -> Restartable + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!("SelectChannels"); + tokio::spawn( + async move { + loop { + let ((net, next_opt), _, _) = select_all(nexts(&mut self.channels)).await; + bail_task_if!( + next_opt.is_none(), + self, + format!("Inbound from {} broke", net), + ); + let next = next_opt.expect("checked"); + bail_task_if!( + self.outbound.send((net, next)).is_err(), + self, + "outbound channel broke" + ); + } + } + .instrument(span), + ) + } +} From a5c4bc82ff978703a29d220114413c18e38786dc Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 09:21:59 -0700 Subject: [PATCH 16/80] chore: lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 7fa1bdea..49bea57e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2282,6 +2282,7 @@ dependencies = [ "async-trait", "ethers", "eyre", + "futures-util", "nomad-core", "nomad-ethereum", "nomad-xyz-configuration", From b8c0cb0d226c2099f4b5966d4fb3a3d07d0521ad Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 09:56:58 -0700 Subject: [PATCH 17/80] refactor: more functions, more fun --- agents/monitor/src/dispatch_wait.rs | 40 ++++++++++++++++++++--------- agents/monitor/src/update_wait.rs | 3 ++- agents/monitor/src/utils.rs | 2 ++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/agents/monitor/src/dispatch_wait.rs b/agents/monitor/src/dispatch_wait.rs index 49f795d9..dc175efc 100644 --- a/agents/monitor/src/dispatch_wait.rs +++ b/agents/monitor/src/dispatch_wait.rs @@ -1,5 +1,6 @@ +use ethers::prelude::U64; use nomad_ethereum::bindings::home::{DispatchFilter, UpdateFilter}; -use prometheus::Histogram; +use prometheus::{Histogram, HistogramTimer}; use tokio::{ select, sync::mpsc::{UnboundedReceiver, UnboundedSender}, @@ -18,9 +19,15 @@ pub(crate) struct DispatchWaitMetrics { pub(crate) struct DispatchWait { incoming_dispatch: UnboundedReceiver>, incoming_update: UnboundedReceiver>, + network: String, emitter: String, + metrics: DispatchWaitMetrics, + + timers: Vec, + blocks: Vec, + outgoing_update: UnboundedSender>, outgoing_dispatch: UnboundedSender>, } @@ -51,10 +58,28 @@ impl DispatchWait { network, emitter, metrics, + timers: vec![], + blocks: vec![], outgoing_update, outgoing_dispatch, } } + + fn handle_dispatch(&mut self, block_number: U64) { + self.timers.push(self.metrics.timer.start_timer()); + self.blocks.push(block_number); + } + + fn handle_update(&mut self, block_number: U64) { + // drain the entire vec + self.timers + .drain(0..) + .for_each(|timer| timer.observe_duration()); + self.blocks.drain(0..).for_each(|dispatch_height| { + let diff = block_number.saturating_sub(dispatch_height); + self.metrics.blocks.observe(diff.as_u64() as f64); + }); + } } pub(crate) type DispatchWaitTask = Restartable; @@ -79,9 +104,6 @@ impl ProcessStep for DispatchWait { emitter = self.emitter.as_str(), ); - let mut timers = vec![]; - let mut blocks = vec![]; - tokio::spawn( async move { loop { @@ -113,8 +135,7 @@ impl ProcessStep for DispatchWait { self, "outbound dispatch broke" ); - timers.push(self.metrics.timer.start_timer()); - blocks.push(block_number); + self.handle_dispatch(block_number); } update_opt = self.incoming_update.recv() => { bail_task_if!( @@ -130,12 +151,7 @@ impl ProcessStep for DispatchWait { self, "outbound update broke" ); - // drain the entire vec - timers.drain(0..).for_each(|timer| timer.observe_duration()); - blocks.drain(0..).for_each(|dispatch_height| { - let diff = block_number.saturating_sub(dispatch_height); - self.metrics.blocks.observe(diff.as_u64() as f64); - }); + self.handle_update(block_number); } } } diff --git a/agents/monitor/src/update_wait.rs b/agents/monitor/src/update_wait.rs index 3b7afca8..5c7d3db6 100644 --- a/agents/monitor/src/update_wait.rs +++ b/agents/monitor/src/update_wait.rs @@ -42,7 +42,8 @@ impl UpdateWait { ) -> Self { let (tx, rx) = unbounded_channel(); - let _ = SelectChannels::new(incoming_relays, tx).spawn(); + SelectChannels::new(incoming_relays, tx).spawn(); + Self { incoming_update, incoming_relays: rx, diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs index e88f7eda..038a2fe0 100644 --- a/agents/monitor/src/utils.rs +++ b/agents/monitor/src/utils.rs @@ -8,6 +8,7 @@ use crate::bail_task_if; use super::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; +// split handles from outputs pub(crate) fn split( map: HashMap<&str, StepHandle>, ) -> ( @@ -28,6 +29,7 @@ where (handles, map) } +// split handles from outputs in a nested map pub(crate) fn nested_split( map: HomeReplicaMap>, ) -> ( From 5c513aeffb7e0302644fd8944145ca6a1e7ac886 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 11:09:27 -0700 Subject: [PATCH 18/80] refactor: Faucets<'a> --- agents/monitor/src/between.rs | 8 ++ agents/monitor/src/dispatch_wait.rs | 31 +++-- agents/monitor/src/domain.rs | 141 ++++++++++----------- agents/monitor/src/init.rs | 188 +++++++++++++--------------- agents/monitor/src/main.rs | 58 +++++---- agents/monitor/src/update_wait.rs | 30 ++--- agents/monitor/src/utils.rs | 19 ++- 7 files changed, 239 insertions(+), 236 deletions(-) diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/between.rs index 0ac17d2d..f6f42beb 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/between.rs @@ -1,3 +1,7 @@ +use nomad_ethereum::bindings::{ + home::{DispatchFilter, UpdateFilter}, + replica::{ProcessFilter, UpdateFilter as RelayFilter}, +}; use tracing::{info_span, Instrument}; use tokio::sync::mpsc::{self, UnboundedReceiver}; @@ -65,6 +69,10 @@ where pub(crate) type BetweenHandle = StepHandle>; pub(crate) type BetweenTask = Restartable>; +pub(crate) type BetweenDispatch = BetweenHandle>; +pub(crate) type BetweenUpdate = BetweenHandle>; +pub(crate) type BetweenRelay = BetweenHandle>; +pub(crate) type BetweenProcess = BetweenHandle>; impl ProcessStep for BetweenEvents> where diff --git a/agents/monitor/src/dispatch_wait.rs b/agents/monitor/src/dispatch_wait.rs index dc175efc..3e73eab2 100644 --- a/agents/monitor/src/dispatch_wait.rs +++ b/agents/monitor/src/dispatch_wait.rs @@ -1,13 +1,12 @@ use ethers::prelude::U64; -use nomad_ethereum::bindings::home::{DispatchFilter, UpdateFilter}; use prometheus::{Histogram, HistogramTimer}; -use tokio::{ - select, - sync::mpsc::{UnboundedReceiver, UnboundedSender}, -}; +use tokio::select; use tracing::{info_span, Instrument}; -use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; +use crate::{ + bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, StepHandle, UpdateFaucet, + UpdateSink, +}; #[derive(Debug)] pub(crate) struct DispatchWaitMetrics { @@ -17,8 +16,8 @@ pub(crate) struct DispatchWaitMetrics { #[derive(Debug)] pub(crate) struct DispatchWait { - incoming_dispatch: UnboundedReceiver>, - incoming_update: UnboundedReceiver>, + incoming_dispatch: DispatchFaucet, + incoming_update: UpdateFaucet, network: String, emitter: String, @@ -28,8 +27,8 @@ pub(crate) struct DispatchWait { timers: Vec, blocks: Vec, - outgoing_update: UnboundedSender>, - outgoing_dispatch: UnboundedSender>, + outgoing_update: UpdateSink, + outgoing_dispatch: DispatchSink, } impl std::fmt::Display for DispatchWait { @@ -44,13 +43,13 @@ impl std::fmt::Display for DispatchWait { impl DispatchWait { pub(crate) fn new( - incoming_dispatch: UnboundedReceiver>, - incoming_update: UnboundedReceiver>, + incoming_dispatch: DispatchFaucet, + incoming_update: UpdateFaucet, network: String, emitter: String, metrics: DispatchWaitMetrics, - outgoing_update: UnboundedSender>, - outgoing_dispatch: UnboundedSender>, + outgoing_update: UpdateSink, + outgoing_dispatch: DispatchSink, ) -> Self { Self { incoming_dispatch, @@ -87,8 +86,8 @@ pub(crate) type DispatchWaitHandle = StepHandle; #[derive(Debug)] pub struct DispatchWaitOutput { - pub(crate) dispatches: UnboundedReceiver>, - pub(crate) updates: UnboundedReceiver>, + pub(crate) dispatches: DispatchFaucet, + pub(crate) updates: UpdateFaucet, } impl ProcessStep for DispatchWait { diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index fc990d48..3de51cd6 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -1,15 +1,12 @@ use std::{collections::HashMap, sync::Arc}; -use nomad_ethereum::bindings::{ - home::{DispatchFilter, Home, UpdateFilter}, - replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, -}; +use nomad_ethereum::bindings::{home::Home, replica::Replica}; use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; -use tokio::sync::mpsc; +use tokio::sync::mpsc::unbounded_channel; use crate::{ annotate::WithMeta, - between::{BetweenEvents, BetweenHandle, BetweenMetrics}, + between::{BetweenDispatch, BetweenEvents, BetweenHandle, BetweenMetrics, BetweenUpdate}, dispatch_wait::{DispatchWait, DispatchWaitHandle, DispatchWaitOutput}, init::provider_for, metrics::Metrics, @@ -17,7 +14,8 @@ use crate::{ DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, }, - ProcessStep, Provider, StepHandle, + DispatchFaucet, Faucet, NetworkMap, ProcessFaucet, ProcessStep, Provider, RelayFaucet, + StepHandle, UpdateFaucet, }; #[derive(Debug)] @@ -72,7 +70,7 @@ impl Domain { } pub(crate) fn dispatch_producer(&self) -> DispatchProducerHandle { - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = unbounded_channel(); let handle = DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); @@ -80,7 +78,7 @@ impl Domain { } pub(crate) fn update_producer(&self) -> UpdateProducerHandle { - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = unbounded_channel(); let handle = UpdateProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); @@ -92,7 +90,7 @@ impl Domain { replica: &Replica, replica_of: &str, ) -> RelayProducerHandle { - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = unbounded_channel(); let handle = RelayProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); StepHandle { handle, rx } @@ -113,7 +111,7 @@ impl Domain { replica: &Replica, replica_of: &str, ) -> ProcessProducerHandle { - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = unbounded_channel(); let handle = ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); StepHandle { handle, rx } @@ -131,7 +129,7 @@ impl Domain { pub(crate) fn count( &self, - incoming: mpsc::UnboundedReceiver>, + incoming: Faucet, metrics: BetweenMetrics, event: impl AsRef, ) -> BetweenHandle> @@ -139,7 +137,7 @@ impl Domain { T: 'static + Send + Sync + std::fmt::Debug, { let network = self.network.clone(); - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = unbounded_channel(); let emitter = self.home_address(); tracing::debug!( @@ -157,90 +155,83 @@ impl Domain { pub(crate) fn count_dispatches( &self, - incoming: mpsc::UnboundedReceiver>, + incoming: DispatchFaucet, metrics: BetweenMetrics, event: impl AsRef, - ) -> BetweenHandle> { + ) -> BetweenDispatch { self.count(incoming, metrics, event) } pub(crate) fn count_updates( &self, - incoming: mpsc::UnboundedReceiver>, + incoming: UpdateFaucet, metrics: BetweenMetrics, event: impl AsRef, - ) -> BetweenHandle> { + ) -> BetweenUpdate { self.count(incoming, metrics, event) } - pub(crate) fn count_relays( - &self, - mut incomings: HashMap<&str, mpsc::UnboundedReceiver>>, + pub(crate) fn count_relays<'a>( + &'a self, + incomings: &mut NetworkMap<'a, RelayFaucet>, metrics: Arc, - ) -> HashMap<&str, BetweenHandle>> { - self.replicas - .iter() - .map(|(replica_of, replica)| { - let emitter = format!("{:?}", replica.address()); - let network = self.name(); - let event = "relay"; - tracing::debug!( - network = network, - replica = emitter, - event, - "starting counter", - ); - let incoming = incomings - .remove(replica_of.as_str()) - .expect("Missing channel"); - let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - - let between = self.count(incoming, metrics, event); - - (replica_of.as_str(), between) - }) - .collect() - } - - pub(crate) fn count_processes( - &self, - mut incomings: HashMap<&str, mpsc::UnboundedReceiver>>, + ) { + self.replicas.iter().for_each(|(replica_of, replica)| { + let emitter = format!("{:?}", replica.address()); + let network = self.name(); + let event = "relay"; + tracing::debug!( + network = network, + replica = emitter, + event, + "starting counter", + ); + let incoming = incomings + .remove(replica_of.as_str()) + .expect("Missing channel"); + let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); + + let between = self.count(incoming, metrics, event); + incomings.insert(replica_of, between.rx); + }); + } + + pub(crate) fn count_processes<'a>( + &'a self, + incomings: &mut NetworkMap<'a, ProcessFaucet>, metrics: Arc, - ) -> HashMap<&str, BetweenHandle>> { - self.replicas - .iter() - .map(|(replica_of, replica)| { - let emitter = format!("{:?}", replica.address()); - let network = self.name(); - let event = "process"; - tracing::debug!( - network = network, - replica = emitter, - event, - "starting counter", - ); - let incoming = incomings - .remove(replica_of.as_str()) - .expect("Missing channel"); - let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - - let between = self.count(incoming, metrics, event); - - (replica_of.as_str(), between) - }) - .collect() + ) { + self.replicas.iter().for_each(|(replica_of, replica)| { + let emitter = format!("{:?}", replica.address()); + let network = self.name(); + let event = "process"; + tracing::debug!( + network = network, + replica = emitter, + event, + "starting counter", + ); + let incoming = incomings + .remove(replica_of.as_str()) + .expect("Missing channel"); + + let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); + let between = self.count(incoming, metrics, event); + + incomings.insert(replica_of, between.rx); + }); } pub(crate) fn dispatch_to_update( &self, - incoming_dispatch: mpsc::UnboundedReceiver>, - incoming_update: mpsc::UnboundedReceiver>, + incoming_dispatch: DispatchFaucet, + incoming_update: UpdateFaucet, metrics: Arc, ) -> DispatchWaitHandle { let metrics = metrics.dispatch_wait_metrics(&self.network, &self.home_address()); - let (outgoing_update, updates) = mpsc::unbounded_channel(); - let (outgoing_dispatch, dispatches) = mpsc::unbounded_channel(); + let (outgoing_update, updates) = unbounded_channel(); + let (outgoing_dispatch, dispatches) = unbounded_channel(); let handle = DispatchWait::new( incoming_dispatch, diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 378d2ff4..fc3ae16e 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -5,24 +5,20 @@ use ethers::{ prelude::{Http, Provider as EthersProvider}, }; -use nomad_ethereum::bindings::{ - home::{DispatchFilter, UpdateFilter}, - replica::{ProcessFilter, UpdateFilter as RelayFilter}, -}; use nomad_xyz_configuration::{get_builtin, NomadConfig}; -use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; +use tokio::task::JoinHandle; use tracing_subscriber::EnvFilter; use crate::{ - annotate::WithMeta, - between::BetweenHandle, + between::{BetweenDispatch, BetweenProcess, BetweenRelay, BetweenUpdate}, dispatch_wait::DispatchWaitHandle, domain::Domain, metrics::Metrics, producer::{ DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, }, - ArcProvider, HomeReplicaMap, + utils, ArcProvider, DispatchFaucet, Faucets, HomeReplicaMap, NetworkMap, ProcessFaucet, + RelayFaucet, UpdateFaucet, }; pub(crate) fn config_from_file() -> Option { @@ -110,144 +106,132 @@ impl Monitor { self.metrics.clone().run_http_server() } - pub(crate) fn run_dispatch_producers(&self) -> HashMap<&str, DispatchProducerHandle> { + fn run_dispatch_producers(&self) -> HashMap<&str, DispatchProducerHandle> { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.dispatch_producer())) .collect() } - pub(crate) fn run_update_producers(&self) -> HashMap<&str, UpdateProducerHandle> { + fn run_update_producers(&self) -> HashMap<&str, UpdateProducerHandle> { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.update_producer())) .collect() } - pub(crate) fn run_relay_producers(&self) -> HomeReplicaMap { + fn run_relay_producers(&self) -> HomeReplicaMap { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.relay_producers())) .collect() } - pub(crate) fn run_process_producers(&self) -> HomeReplicaMap { + fn run_process_producers(&self) -> HomeReplicaMap { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.process_producers())) .collect() } + pub(crate) fn producers(&self) -> Faucets { + Faucets { + dispatches: utils::split(self.run_dispatch_producers()).1, + updates: utils::split(self.run_update_producers()).1, + relays: utils::nested_split(self.run_relay_producers()).1, + processes: utils::nested_split(self.run_process_producers()).1, + } + } #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run_between_dispatch( - &self, - mut incomings: HashMap<&str, UnboundedReceiver>>, - ) -> HashMap<&str, BetweenHandle>> { - self.networks - .iter() - .map(|(chain, domain)| { - let emitter = domain.home_address(); - let event = "dispatch"; + fn run_between_dispatch<'a>(&'a self, incomings: &mut NetworkMap<'a, DispatchFaucet>) { + self.networks.iter().for_each(|(chain, domain)| { + let emitter = domain.home_address(); + let event = "dispatch"; - let metrics = self.metrics.between_metrics(chain, event, &emitter, None); + let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = incomings.remove(chain.as_str()).expect("missing producer"); + let producer = incomings.remove(chain.as_str()).expect("missing producer"); - let between = domain.count_dispatches(producer, metrics, event); + let between = domain.count_dispatches(producer, metrics, event); - (chain.as_str(), between) - }) - .collect() + incomings.insert(chain, between.rx); + }) } #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run_between_update( - &self, - mut incomings: HashMap<&str, UnboundedReceiver>>, - ) -> HashMap<&str, BetweenHandle>> { - self.networks - .iter() - .map(|(chain, domain)| { - let emitter = format!("{:?}", domain.home().address()); - let event = "update"; + fn run_between_update<'a>(&'a self, incomings: &mut NetworkMap<'a, UpdateFaucet>) { + self.networks.iter().for_each(|(chain, domain)| { + let emitter = format!("{:?}", domain.home().address()); + let event = "update"; - let metrics = self.metrics.between_metrics(chain, event, &emitter, None); + let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = incomings.remove(chain.as_str()).expect("missing producer"); + let producer = incomings.remove(chain.as_str()).expect("missing producer"); - let between = domain.count_updates(producer, metrics, event); + let between = domain.count_updates(producer, metrics, event); - (chain.as_str(), between) - }) - .collect() + incomings.insert(chain, between.rx); + }) } #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run_between_relay( - &self, - mut incomings: HomeReplicaMap>>, - ) -> HomeReplicaMap>> { - self.networks - .iter() - .map(|(network, domain)| { - let incomings = incomings - .remove(network.as_str()) - .expect("missing producer") - .into_iter() - .map(|(k, v)| (k, v)) - .collect(); - ( - network.as_str(), - domain.count_relays(incomings, self.metrics.clone()), - ) - }) - .collect() + fn run_between_relay<'a>(&'a self, incomings: &mut HomeReplicaMap<'a, RelayFaucet>) { + self.networks.iter().for_each(|(network, domain)| { + let inner = incomings + .get_mut(network.as_str()) + .expect("missing producer"); + + domain.count_relays(inner, self.metrics.clone()); + }); } #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run_between_process( - &self, - mut incomings: HomeReplicaMap>>, - ) -> HomeReplicaMap>> { - self.networks - .iter() - .map(|(network, domain)| { - let incomings = incomings - .remove(network.as_str()) - .expect("missing producer") - .into_iter() - .collect(); - ( - network.as_str(), - domain.count_processes(incomings, self.metrics.clone()), - ) - }) - .collect() + fn run_between_process<'a>(&'a self, incomings: &mut HomeReplicaMap<'a, ProcessFaucet>) { + self.networks.iter().for_each(|(network, domain)| { + let inner = incomings + .get_mut(network.as_str()) + .expect("missing producer"); + + domain.count_processes(inner, self.metrics.clone()); + }); + } + + pub(crate) fn run_betweens<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.run_between_dispatch(&mut faucets.dispatches); + self.run_between_update(&mut faucets.updates); + self.run_between_relay(&mut faucets.relays); + self.run_between_process(&mut faucets.processes); } #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run_dispatch_to_update( - &self, - mut incoming_dispatches: HashMap<&str, UnboundedReceiver>>, - mut incoming_updates: HashMap<&str, UnboundedReceiver>>, - ) -> HashMap<&str, DispatchWaitHandle> { - self.networks - .iter() - .map(|(network, domain)| { - let incoming_dispatch = incoming_dispatches - .remove(network.as_str()) - .expect("missing incoming dispatch"); - let incoming_update = incoming_updates - .remove(network.as_str()) - .expect("missing incoming update"); - - let d_to_r = domain.dispatch_to_update( - incoming_dispatch, - incoming_update, - self.metrics.clone(), - ); - (network.as_str(), d_to_r) - }) - .collect() + pub(crate) fn run_dispatch_to_update<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.networks.iter().for_each(|(network, domain)| { + let incoming_dispatch = faucets + .dispatches + .remove(network.as_str()) + .expect("missing incoming dispatch"); + let incoming_update = faucets + .updates + .remove(network.as_str()) + .expect("missing incoming update"); + + let d_to_r = + domain.dispatch_to_update(incoming_dispatch, incoming_update, self.metrics.clone()); + + faucets.dispatches.insert(network, d_to_r.rx.dispatches); + faucets.updates.insert(network, d_to_r.rx.updates); + }); } + + // TODO + // pub(crate) fn run_update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { + // self.networks.iter().for_each(|(k,v)| { + // let incoming_update = faucets + // .updates + // .remove(network.as_str()) + // .expect("missing incoming update"); + + // todo!() + // }); + // } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 90c7d753..463e1cd5 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,5 +1,13 @@ +use annotate::WithMeta; +use nomad_ethereum::bindings::{ + home::{DispatchFilter, UpdateFilter}, + replica::{ProcessFilter, UpdateFilter as RelayFilter}, +}; use std::{collections::HashMap, panic, sync::Arc}; -use tokio::task::JoinHandle; +use tokio::{ + sync::mpsc::{UnboundedReceiver, UnboundedSender}, + task::JoinHandle, +}; use tracing::info_span; use ethers::prelude::{Http, Provider as EthersProvider}; @@ -20,8 +28,28 @@ pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; // pub(crate) type ProviderError = ContractError; +pub(crate) type Faucet = UnboundedReceiver>; +pub(crate) type Sink = UnboundedSender>; + +pub(crate) type DispatchFaucet = Faucet; +pub(crate) type UpdateFaucet = Faucet; +pub(crate) type RelayFaucet = Faucet; +pub(crate) type ProcessFaucet = Faucet; +pub(crate) type DispatchSink = Sink; +pub(crate) type UpdateSink = Sink; +pub(crate) type RelaySink = Sink; +pub(crate) type ProcessSink = Sink; + +pub(crate) type NetworkMap<'a, T> = HashMap<&'a str, T>; pub(crate) type HomeReplicaMap<'a, T> = HashMap<&'a str, HashMap<&'a str, T>>; +pub(crate) struct Faucets<'a> { + pub(crate) dispatches: NetworkMap<'a, DispatchFaucet>, + pub(crate) updates: NetworkMap<'a, UpdateFaucet>, + pub(crate) relays: HomeReplicaMap<'a, RelayFaucet>, + pub(crate) processes: HomeReplicaMap<'a, ProcessFaucet>, +} + #[tokio::main] async fn main() -> eyre::Result<()> { init::init_tracing(); @@ -33,34 +61,12 @@ async fn main() -> eyre::Result<()> { tracing::info!("setup complete!"); let _http = monitor.run_http_server(); - let dispatch_producer = monitor.run_dispatch_producers(); - let update_producer = monitor.run_update_producers(); - let relay_producers = monitor.run_relay_producers(); - let process_producers = monitor.run_process_producers(); - - let (dispatch_handles, dispatch_producer) = utils::split(dispatch_producer); - let (update_handles, update_producer) = utils::split(update_producer); - let (relay_handles, relay_producers) = utils::nested_split(relay_producers); - let (process_handles, process_producers) = utils::nested_split(process_producers); + let mut faucets = monitor.producers(); - let dispatch_counters = monitor.run_between_dispatch(dispatch_producer); - let update_counters = monitor.run_between_update(update_producer); - let relay_counters = monitor.run_between_relay(relay_producers); - let process_counters = monitor.run_between_process(process_producers); - - let (dispatch_count_handles, dispatch_producer) = utils::split(dispatch_counters); - let (update_count_handles, update_producer) = utils::split(update_counters); - let (relay_count_handles, relay_producer) = utils::nested_split(relay_counters); - let (process_count_handles, process_producer) = utils::nested_split(process_counters); - - let d_to_u = monitor.run_dispatch_to_update(dispatch_producer, update_producer); - - let (d_to_u_handles, d_and_u_producers) = utils::split(d_to_u); + monitor.run_betweens(&mut faucets); + monitor.run_dispatch_to_update(&mut faucets); tracing::info!("counters started"); - - // should cause it to run until crashes occur - let _ = update_handles.into_iter().next().unwrap().await; } Ok(()) } diff --git a/agents/monitor/src/update_wait.rs b/agents/monitor/src/update_wait.rs index 5c7d3db6..5e647c08 100644 --- a/agents/monitor/src/update_wait.rs +++ b/agents/monitor/src/update_wait.rs @@ -1,14 +1,14 @@ use ethers::prelude::H256; -use futures_util::future::select_all; -use prometheus::{Histogram, HistogramTimer}; -use std::{collections::HashMap, fmt::Display, time::Instant}; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use prometheus::Histogram; +use std::{collections::HashMap, time::Instant}; +use tokio::sync::mpsc::unbounded_channel; use tracing::{info_span, Instrument}; -use nomad_ethereum::bindings::{home::UpdateFilter, replica::UpdateFilter as RelayFilter}; +use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ - annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, Restartable, StepHandle, + annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, RelayFaucet, RelaySink, + Restartable, StepHandle, UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -18,7 +18,7 @@ pub(crate) struct UpdateWaitMetrics { #[derive(Debug)] pub(crate) struct UpdateWait { - incoming_update: UnboundedReceiver>, + incoming_update: UpdateFaucet, incoming_relays: > as ProcessStep>::Output, network: String, @@ -27,18 +27,18 @@ pub(crate) struct UpdateWait { updates: HashMap, relays: HashMap>, - outgoing_update: UnboundedSender>, - outgoing_relays: HashMap>>, + outgoing_update: UpdateSink, + outgoing_relays: HashMap, } impl UpdateWait { pub(crate) fn new( - incoming_update: UnboundedReceiver>, - incoming_relays: HashMap>>, + incoming_update: UpdateFaucet, + incoming_relays: HashMap, network: String, metrics: UpdateWaitMetrics, - outgoing_update: UnboundedSender>, - outgoing_relays: HashMap>>, + outgoing_update: UpdateSink, + outgoing_relays: HashMap, ) -> Self { let (tx, rx) = unbounded_channel(); @@ -68,8 +68,8 @@ pub(crate) type UpdateWaitHandle = StepHandle; #[derive(Debug)] pub struct UpdateWaitOutput { - pub(crate) updates: UnboundedReceiver>, - pub(crate) relays: HashMap>>, + pub(crate) updates: UpdateFaucet, + pub(crate) relays: HashMap, } impl UpdateWait { diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs index 038a2fe0..8d72eb6e 100644 --- a/agents/monitor/src/utils.rs +++ b/agents/monitor/src/utils.rs @@ -1,10 +1,12 @@ use std::{collections::HashMap, fmt::Display, pin::Pin}; -use futures_util::future::{select_all, Select}; +use futures_util::future::select_all; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; -use crate::bail_task_if; +use crate::{ + bail_task_if, dispatch_wait::DispatchWaitOutput, DispatchFaucet, NetworkMap, UpdateFaucet, +}; use super::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; @@ -118,3 +120,16 @@ where ) } } + +pub(crate) fn split_dispatch_wait_output( + mut map: HashMap<&str, DispatchWaitOutput>, +) -> (NetworkMap, NetworkMap) { + let mut dispatches = HashMap::new(); + let mut updates = HashMap::new(); + + map.drain().for_each(|(k, v)| { + dispatches.insert(k, v.dispatches); + updates.insert(k, v.updates); + }); + (dispatches, updates) +} From 611b3fa8d517f28cf1ca5aadd2e5373b3e7238b0 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 12:18:30 -0700 Subject: [PATCH 19/80] fix: one less panic in producer --- agents/monitor/src/domain.rs | 52 +++++++++++++++++++++++++++++-- agents/monitor/src/init.rs | 22 ++++++------- agents/monitor/src/main.rs | 16 +++++++--- agents/monitor/src/metrics.rs | 25 ++++++++++++++- agents/monitor/src/producer.rs | 19 +++++++---- agents/monitor/src/update_wait.rs | 38 +++++++++++----------- 6 files changed, 126 insertions(+), 46 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 3de51cd6..eb7b5e48 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -14,8 +14,9 @@ use crate::{ DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, }, - DispatchFaucet, Faucet, NetworkMap, ProcessFaucet, ProcessStep, Provider, RelayFaucet, - StepHandle, UpdateFaucet, + update_wait::UpdateWait, + DispatchFaucet, Faucet, HomeReplicaMap, NetworkMap, ProcessFaucet, ProcessStep, Provider, + RelayFaucet, StepHandle, UpdateFaucet, }; #[derive(Debug)] @@ -252,4 +253,51 @@ impl Domain { }, } } + + pub(crate) fn update_to_relay<'a>( + &'a self, + global_updates: &mut NetworkMap<'a, UpdateFaucet>, + global_relays: &mut HomeReplicaMap<'a, RelayFaucet>, + metrics: Arc, + ) { + let mut relay_faucets = HashMap::new(); + let mut relay_sinks = HashMap::new(); + + // we want to go through each network that is NOT this network + // get the relay sink FOR THIS NETWORK'S REPLICA + global_relays + .iter_mut() + // does not match this network + .filter(|(k, _)| **k != self.network) + .for_each(|(k, v)| { + // create a new channel + let (sink, faucet) = unbounded_channel(); + // insert this in the map we'll give to the metrics task + relay_sinks.insert(k.to_string(), sink); + + // replace the faucet in the global producers map + let faucet = v + .insert(self.name(), faucet) + .expect("missing relay producer"); + // insert upstream faucet into the map we'll give to the + // metrics task + relay_faucets.insert(k.to_string(), faucet); + }); + + let (update_sink, update_faucet) = unbounded_channel(); + + let update_faucet = global_updates + .insert(self.name(), update_faucet) + .expect("missing producer"); + + UpdateWait::new( + update_faucet, + relay_faucets, + self.name(), + metrics.update_wait_metrics(self.name(), &self.home_address()), + update_sink, + relay_sinks, + ) + .spawn(); + } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index fc3ae16e..5b60038e 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -10,8 +10,6 @@ use tokio::task::JoinHandle; use tracing_subscriber::EnvFilter; use crate::{ - between::{BetweenDispatch, BetweenProcess, BetweenRelay, BetweenUpdate}, - dispatch_wait::DispatchWaitHandle, domain::Domain, metrics::Metrics, producer::{ @@ -223,15 +221,13 @@ impl Monitor { }); } - // TODO - // pub(crate) fn run_update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { - // self.networks.iter().for_each(|(k,v)| { - // let incoming_update = faucets - // .updates - // .remove(network.as_str()) - // .expect("missing incoming update"); - - // todo!() - // }); - // } + pub(crate) fn run_update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.networks.iter().for_each(|(_, v)| { + v.update_to_relay( + &mut faucets.updates, + &mut faucets.relays, + self.metrics.clone(), + ) + }); + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 463e1cd5..309ef987 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -54,21 +54,27 @@ pub(crate) struct Faucets<'a> { async fn main() -> eyre::Result<()> { init::init_tracing(); { - let span = info_span!("MonitorBootup"); - let _span = span.enter(); + let monitor = info_span!("MonitorBootup").in_scope(|| { + let monitor = init::monitor()?; + tracing::info!("setup complete!"); + Ok::<_, eyre::Report>(monitor) + })?; - let monitor = init::monitor()?; - tracing::info!("setup complete!"); let _http = monitor.run_http_server(); let mut faucets = monitor.producers(); monitor.run_betweens(&mut faucets); monitor.run_dispatch_to_update(&mut faucets); + monitor.run_update_to_relay(&mut faucets); tracing::info!("counters started"); + + // just run forever + loop { + tokio::time::sleep(std::time::Duration::from_secs(10000)).await + } } - Ok(()) } pub type Restartable = JoinHandle<(Task, eyre::Report)>; diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index dfd37af2..ce27d9a3 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -4,7 +4,9 @@ use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, In use tokio::task::JoinHandle; use warp::Filter; -use crate::{between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics}; +use crate::{ + between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, update_wait::UpdateWaitMetrics, +}; #[derive(Debug)] pub(crate) struct Metrics { @@ -14,6 +16,8 @@ pub(crate) struct Metrics { dispatch_to_update_timers: prometheus::HistogramVec, dispatch_to_update_blocks: prometheus::HistogramVec, + + update_to_relay_timers: prometheus::HistogramVec, } fn u16_from_env(s: impl AsRef) -> Option { @@ -30,6 +34,16 @@ fn gather() -> prometheus::Result> { impl Metrics { pub(crate) fn new() -> eyre::Result { + let update_to_relay_timers = HistogramVec::new( + HistogramOpts::new( + "update_to_relay_ms", + "Ms between update and relay, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "emitter"], + )?; + let dispatch_to_update_timers = HistogramVec::new( HistogramOpts::new( "dispatch_to_update_ms", @@ -103,6 +117,7 @@ impl Metrics { event_counts: counts, dispatch_to_update_blocks, dispatch_to_update_timers, + update_to_relay_timers, }) } @@ -205,4 +220,12 @@ impl Metrics { .with_label_values(&[network, emitter]), } } + + pub(crate) fn update_wait_metrics(&self, network: &str, emitter: &str) -> UpdateWaitMetrics { + UpdateWaitMetrics { + times: self + .update_to_relay_timers + .with_label_values(&[network, emitter]), + } + } } diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 2e36f55b..483777d7 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -1,5 +1,3 @@ -use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; - use ethers::prelude::Middleware; use nomad_ethereum::bindings::{ home::{DispatchFilter, Home, UpdateFilter}, @@ -8,11 +6,16 @@ use nomad_ethereum::bindings::{ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; +use crate::{ + annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, + StepHandle, +}; + #[derive(Debug)] pub(crate) struct DispatchProducer { home: Home, network: String, - tx: UnboundedSender>, + tx: DispatchSink, } impl std::fmt::Display for DispatchProducer { @@ -48,7 +51,7 @@ pub(crate) type DispatchProducerTask = Restartable; pub(crate) type DispatchProducerHandle = StepHandle; impl ProcessStep for DispatchProducer { - type Output = UnboundedReceiver>; + type Output = DispatchFaucet; fn spawn(self) -> DispatchProducerTask { let span = info_span!( @@ -148,7 +151,11 @@ impl ProcessStep for UpdateProducer { tokio::spawn( async move { let provider = self.home.client(); - let height = provider.get_block_number().await.unwrap(); + let height = provider.get_block_number().await; + + bail_task_if!(height.is_err(), self, "Err retrieving height"); + let height = height.expect("checked"); + let mut from = height - 10; let mut to = height - 5; loop { @@ -171,7 +178,7 @@ impl ProcessStep for UpdateProducer { let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - 5; + let tip = tip_res.expect("checked") - 5; from = to; to = std::cmp::max(to, tip); diff --git a/agents/monitor/src/update_wait.rs b/agents/monitor/src/update_wait.rs index 5e647c08..53ba0978 100644 --- a/agents/monitor/src/update_wait.rs +++ b/agents/monitor/src/update_wait.rs @@ -18,8 +18,8 @@ pub(crate) struct UpdateWaitMetrics { #[derive(Debug)] pub(crate) struct UpdateWait { - incoming_update: UpdateFaucet, - incoming_relays: > as ProcessStep>::Output, + update_faucet: UpdateFaucet, + relay_faucets: > as ProcessStep>::Output, network: String, metrics: UpdateWaitMetrics, @@ -27,32 +27,32 @@ pub(crate) struct UpdateWait { updates: HashMap, relays: HashMap>, - outgoing_update: UpdateSink, - outgoing_relays: HashMap, + update_sink: UpdateSink, + relay_sinks: HashMap, } impl UpdateWait { pub(crate) fn new( - incoming_update: UpdateFaucet, - incoming_relays: HashMap, - network: String, + update_faucet: UpdateFaucet, + relay_faucets: HashMap, + network: impl AsRef, metrics: UpdateWaitMetrics, - outgoing_update: UpdateSink, - outgoing_relays: HashMap, + update_sink: UpdateSink, + relay_sinks: HashMap, ) -> Self { let (tx, rx) = unbounded_channel(); - SelectChannels::new(incoming_relays, tx).spawn(); + SelectChannels::new(relay_faucets, tx).spawn(); Self { - incoming_update, - incoming_relays: rx, - network, + update_faucet, + relay_faucets: rx, + network: network.as_ref().to_owned(), metrics, updates: Default::default(), relays: Default::default(), - outgoing_update, - outgoing_relays, + update_sink, + relay_sinks, } } } @@ -128,7 +128,7 @@ impl ProcessStep for UpdateWait { biased; - update_opt = self.incoming_update.recv() => { + update_opt = self.update_faucet.recv() => { bail_task_if!( update_opt.is_none(), self, @@ -138,14 +138,14 @@ impl ProcessStep for UpdateWait { let root: H256 = update.log.new_root.into(); bail_task_if!{ // send onwards - self.outgoing_update.send(update).is_err(), + self.update_sink.send(update).is_err(), self, "Outbound updates broke", }; self.handle_update(root); } - relay_opt = self.incoming_relays.recv() => { + relay_opt = self.relay_faucets.recv() => { bail_task_if!( relay_opt.is_none(), self, @@ -156,7 +156,7 @@ impl ProcessStep for UpdateWait { bail_task_if!( // send onward - self.outgoing_relays + self.relay_sinks .get(&net) .expect("missing outgoing") .send(relay) From 0865966d60640033eb78bc5ea19239d4bf1c950e Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 13:37:42 -0700 Subject: [PATCH 20/80] refactor: use faucets in more places --- agents/monitor/src/domain.rs | 146 ++++++++---------- agents/monitor/src/init.rs | 74 +++------ agents/monitor/src/main.rs | 91 ++++++++--- agents/monitor/src/metrics.rs | 2 +- agents/monitor/src/producer.rs | 19 +-- agents/monitor/src/{ => steps}/between.rs | 24 ++- .../monitor/src/{ => steps}/dispatch_wait.rs | 36 ++--- agents/monitor/src/steps/mod.rs | 3 + agents/monitor/src/{ => steps}/update_wait.rs | 15 +- agents/monitor/src/terminal.rs | 2 - agents/monitor/src/utils.rs | 39 +---- 11 files changed, 204 insertions(+), 247 deletions(-) rename agents/monitor/src/{ => steps}/between.rs (86%) rename agents/monitor/src/{ => steps}/dispatch_wait.rs (84%) create mode 100644 agents/monitor/src/steps/mod.rs rename agents/monitor/src/{ => steps}/update_wait.rs (92%) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index eb7b5e48..d5ae47e6 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -5,18 +5,18 @@ use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; use tokio::sync::mpsc::unbounded_channel; use crate::{ - annotate::WithMeta, - between::{BetweenDispatch, BetweenEvents, BetweenHandle, BetweenMetrics, BetweenUpdate}, - dispatch_wait::{DispatchWait, DispatchWaitHandle, DispatchWaitOutput}, init::provider_for, metrics::Metrics, producer::{ DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, }, - update_wait::UpdateWait, - DispatchFaucet, Faucet, HomeReplicaMap, NetworkMap, ProcessFaucet, ProcessStep, Provider, - RelayFaucet, StepHandle, UpdateFaucet, + steps::{ + between::{BetweenEvents, BetweenMetrics}, + dispatch_wait::DispatchWait, + update_wait::UpdateWait, + }, + Faucets, ProcessStep, Provider, StepHandle, }; #[derive(Debug)] @@ -128,55 +128,49 @@ impl Domain { .collect() } - pub(crate) fn count( - &self, - incoming: Faucet, + pub(crate) fn count_dispatches<'a>( + &'a self, + faucets: &mut Faucets<'a>, metrics: BetweenMetrics, event: impl AsRef, - ) -> BetweenHandle> - where - T: 'static + Send + Sync + std::fmt::Debug, - { + ) { let network = self.network.clone(); - let (tx, rx) = unbounded_channel(); let emitter = self.home_address(); + let (sink, faucet) = unbounded_channel(); + let faucet = faucets.swap_dispatch(self.name(), faucet); + tracing::debug!( network = network, home = emitter, event = event.as_ref(), "starting counter", ); - let handle = - BetweenEvents::>::new(incoming, metrics, network, event, emitter, tx) - .spawn(); - - StepHandle { handle, rx } + BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); } - pub(crate) fn count_dispatches( - &self, - incoming: DispatchFaucet, + pub(crate) fn count_updates<'a>( + &'a self, + faucets: &mut Faucets<'a>, metrics: BetweenMetrics, event: impl AsRef, - ) -> BetweenDispatch { - self.count(incoming, metrics, event) - } + ) { + let network = self.network.clone(); + let emitter = self.home_address(); - pub(crate) fn count_updates( - &self, - incoming: UpdateFaucet, - metrics: BetweenMetrics, - event: impl AsRef, - ) -> BetweenUpdate { - self.count(incoming, metrics, event) + let (sink, faucet) = unbounded_channel(); + let faucet = faucets.swap_update(self.name(), faucet); + + tracing::debug!( + network = network, + home = emitter, + event = event.as_ref(), + "starting counter", + ); + BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); } - pub(crate) fn count_relays<'a>( - &'a self, - incomings: &mut NetworkMap<'a, RelayFaucet>, - metrics: Arc, - ) { + pub(crate) fn count_relays<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { self.replicas.iter().for_each(|(replica_of, replica)| { let emitter = format!("{:?}", replica.address()); let network = self.name(); @@ -187,21 +181,18 @@ impl Domain { event, "starting counter", ); - let incoming = incomings - .remove(replica_of.as_str()) - .expect("Missing channel"); + + let (sink, faucet) = unbounded_channel(); + + let faucet = faucets.swap_relay(self.name(), replica_of, faucet); + let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(incoming, metrics, event); - incomings.insert(replica_of, between.rx); + BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); }); } - pub(crate) fn count_processes<'a>( - &'a self, - incomings: &mut NetworkMap<'a, ProcessFaucet>, - metrics: Arc, - ) { + pub(crate) fn count_processes<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { self.replicas.iter().for_each(|(replica_of, replica)| { let emitter = format!("{:?}", replica.address()); let network = self.name(); @@ -212,60 +203,50 @@ impl Domain { event, "starting counter", ); - let incoming = incomings - .remove(replica_of.as_str()) - .expect("Missing channel"); + + let (sink, faucet) = unbounded_channel(); + + let faucet = faucets.swap_process(self.name(), replica_of, faucet); let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - let between = self.count(incoming, metrics, event); - incomings.insert(replica_of, between.rx); + BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); }); } - pub(crate) fn dispatch_to_update( - &self, - incoming_dispatch: DispatchFaucet, - incoming_update: UpdateFaucet, + pub(crate) fn dispatch_to_update<'a>( + &'a self, + faucets: &mut Faucets<'a>, metrics: Arc, - ) -> DispatchWaitHandle { + ) { let metrics = metrics.dispatch_wait_metrics(&self.network, &self.home_address()); - let (outgoing_update, updates) = unbounded_channel(); - let (outgoing_dispatch, dispatches) = unbounded_channel(); + let (update_sink, update_faucet) = unbounded_channel(); + let (dispatch_sink, dispatch_faucet) = unbounded_channel(); + + let dispatch_faucet = faucets.swap_dispatch(self.name(), dispatch_faucet); + let update_faucet = faucets.swap_update(self.name(), update_faucet); - let handle = DispatchWait::new( - incoming_dispatch, - incoming_update, + DispatchWait::new( + dispatch_faucet, + update_faucet, self.name().to_owned(), self.home_address(), metrics, - outgoing_update, - outgoing_dispatch, + dispatch_sink, + update_sink, ) .spawn(); - - DispatchWaitHandle { - handle, - rx: DispatchWaitOutput { - dispatches, - updates, - }, - } } - pub(crate) fn update_to_relay<'a>( - &'a self, - global_updates: &mut NetworkMap<'a, UpdateFaucet>, - global_relays: &mut HomeReplicaMap<'a, RelayFaucet>, - metrics: Arc, - ) { + pub(crate) fn update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { let mut relay_faucets = HashMap::new(); let mut relay_sinks = HashMap::new(); // we want to go through each network that is NOT this network // get the relay sink FOR THIS NETWORK'S REPLICA - global_relays + faucets + .relays .iter_mut() // does not match this network .filter(|(k, _)| **k != self.network) @@ -275,7 +256,8 @@ impl Domain { // insert this in the map we'll give to the metrics task relay_sinks.insert(k.to_string(), sink); - // replace the faucet in the global producers map + // replace the faucet for the replica matching this network + // in the global producers map let faucet = v .insert(self.name(), faucet) .expect("missing relay producer"); @@ -286,9 +268,7 @@ impl Domain { let (update_sink, update_faucet) = unbounded_channel(); - let update_faucet = global_updates - .insert(self.name(), update_faucet) - .expect("missing producer"); + let update_faucet = faucets.swap_update(self.name(), update_faucet); UpdateWait::new( update_faucet, diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 5b60038e..deef9ecf 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -15,8 +15,7 @@ use crate::{ producer::{ DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, }, - utils, ArcProvider, DispatchFaucet, Faucets, HomeReplicaMap, NetworkMap, ProcessFaucet, - RelayFaucet, UpdateFaucet, + utils, ArcProvider, Faucets, HomeReplicaMap, }; pub(crate) fn config_from_file() -> Option { @@ -141,93 +140,60 @@ impl Monitor { } #[tracing::instrument(skip_all, level = "debug")] - fn run_between_dispatch<'a>(&'a self, incomings: &mut NetworkMap<'a, DispatchFaucet>) { + fn run_between_dispatch<'a>(&'a self, faucets: &mut Faucets<'a>) { self.networks.iter().for_each(|(chain, domain)| { let emitter = domain.home_address(); let event = "dispatch"; let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = incomings.remove(chain.as_str()).expect("missing producer"); - - let between = domain.count_dispatches(producer, metrics, event); - - incomings.insert(chain, between.rx); + domain.count_dispatches(faucets, metrics, event); }) } #[tracing::instrument(skip_all, level = "debug")] - fn run_between_update<'a>(&'a self, incomings: &mut NetworkMap<'a, UpdateFaucet>) { + fn run_between_update<'a>(&'a self, faucets: &mut Faucets<'a>) { self.networks.iter().for_each(|(chain, domain)| { let emitter = format!("{:?}", domain.home().address()); let event = "update"; let metrics = self.metrics.between_metrics(chain, event, &emitter, None); - let producer = incomings.remove(chain.as_str()).expect("missing producer"); - - let between = domain.count_updates(producer, metrics, event); - - incomings.insert(chain, between.rx); + domain.count_updates(faucets, metrics, event); }) } #[tracing::instrument(skip_all, level = "debug")] - fn run_between_relay<'a>(&'a self, incomings: &mut HomeReplicaMap<'a, RelayFaucet>) { - self.networks.iter().for_each(|(network, domain)| { - let inner = incomings - .get_mut(network.as_str()) - .expect("missing producer"); - - domain.count_relays(inner, self.metrics.clone()); + fn run_between_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.networks.values().for_each(|domain| { + domain.count_relays(faucets, self.metrics.clone()); }); } #[tracing::instrument(skip_all, level = "debug")] - fn run_between_process<'a>(&'a self, incomings: &mut HomeReplicaMap<'a, ProcessFaucet>) { - self.networks.iter().for_each(|(network, domain)| { - let inner = incomings - .get_mut(network.as_str()) - .expect("missing producer"); - - domain.count_processes(inner, self.metrics.clone()); + fn run_between_process<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.networks.values().for_each(|domain| { + domain.count_processes(faucets, self.metrics.clone()); }); } pub(crate) fn run_betweens<'a>(&'a self, faucets: &mut Faucets<'a>) { - self.run_between_dispatch(&mut faucets.dispatches); - self.run_between_update(&mut faucets.updates); - self.run_between_relay(&mut faucets.relays); - self.run_between_process(&mut faucets.processes); + self.run_between_dispatch(faucets); + self.run_between_update(faucets); + self.run_between_relay(faucets); + self.run_between_process(faucets); } #[tracing::instrument(skip_all, level = "debug")] pub(crate) fn run_dispatch_to_update<'a>(&'a self, faucets: &mut Faucets<'a>) { - self.networks.iter().for_each(|(network, domain)| { - let incoming_dispatch = faucets - .dispatches - .remove(network.as_str()) - .expect("missing incoming dispatch"); - let incoming_update = faucets - .updates - .remove(network.as_str()) - .expect("missing incoming update"); - - let d_to_r = - domain.dispatch_to_update(incoming_dispatch, incoming_update, self.metrics.clone()); - - faucets.dispatches.insert(network, d_to_r.rx.dispatches); - faucets.updates.insert(network, d_to_r.rx.updates); + self.networks.values().for_each(|domain| { + domain.dispatch_to_update(faucets, self.metrics.clone()); }); } pub(crate) fn run_update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { - self.networks.iter().for_each(|(_, v)| { - v.update_to_relay( - &mut faucets.updates, - &mut faucets.relays, - self.metrics.clone(), - ) - }); + self.networks + .iter() + .for_each(|(_, v)| v.update_to_relay(faucets, self.metrics.clone())); } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 309ef987..4117ce43 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -13,21 +13,21 @@ use tracing::info_span; use ethers::prelude::{Http, Provider as EthersProvider}; pub(crate) mod annotate; -pub(crate) mod between; -pub(crate) mod dispatch_wait; pub(crate) mod domain; pub(crate) mod init; pub(crate) mod macros; pub(crate) mod metrics; pub(crate) mod producer; +pub(crate) mod steps; pub(crate) mod terminal; -pub(crate) mod update_wait; pub(crate) mod utils; pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; // pub(crate) type ProviderError = ContractError; +pub(crate) type Restartable = JoinHandle<(Task, eyre::Report)>; + pub(crate) type Faucet = UnboundedReceiver>; pub(crate) type Sink = UnboundedSender>; @@ -50,6 +50,70 @@ pub(crate) struct Faucets<'a> { pub(crate) processes: HomeReplicaMap<'a, ProcessFaucet>, } +impl<'a> Faucets<'a> { + pub(crate) fn swap_dispatch( + &mut self, + network: &'a str, + mut dispatch_faucet: DispatchFaucet, + ) -> DispatchFaucet { + self.dispatches + .get_mut(network) + .map(|old| { + std::mem::swap(old, &mut dispatch_faucet); + dispatch_faucet + }) + .expect("missing dispatch faucet") + } + + pub(crate) fn swap_update( + &mut self, + network: &'a str, + mut update_faucet: UpdateFaucet, + ) -> UpdateFaucet { + self.updates + .get_mut(network) + .map(|old| { + std::mem::swap(old, &mut update_faucet); + update_faucet + }) + .expect("missing dispatch faucet") + } + + pub(crate) fn swap_relay( + &mut self, + network: &'a str, + replica_of: &'a str, + mut relay_faucet: RelayFaucet, + ) -> RelayFaucet { + self.relays + .get_mut(network) + .expect("missing network") + .get_mut(replica_of) + .map(|old| { + std::mem::swap(old, &mut relay_faucet); + relay_faucet + }) + .expect("missing faucet") + } + + pub(crate) fn swap_process( + &mut self, + network: &'a str, + replica_of: &'a str, + mut process_faucet: ProcessFaucet, + ) -> ProcessFaucet { + self.processes + .get_mut(network) + .expect("missing network") + .get_mut(replica_of) + .map(|old| { + std::mem::swap(old, &mut process_faucet); + process_faucet + }) + .expect("missing faucet") + } +} + #[tokio::main] async fn main() -> eyre::Result<()> { init::init_tracing(); @@ -77,23 +141,7 @@ async fn main() -> eyre::Result<()> { } } -pub type Restartable = JoinHandle<(Task, eyre::Report)>; - -/// A step handle is the handle to the process, and its outbound channels. -/// -/// Task creation reutns a step handle so that we can -/// - track the -pub(crate) struct StepHandle -where - Task: ProcessStep, -{ - handle: Restartable, - rx: ::Output, -} - pub(crate) trait ProcessStep: std::fmt::Display { - type Output: 'static + Send + Sync + std::fmt::Debug; - fn spawn(self) -> Restartable where Self: 'static + Send + Sync + Sized; @@ -125,3 +173,8 @@ pub(crate) trait ProcessStep: std::fmt::Display { }) } } + +pub(crate) struct StepHandle { + handle: Restartable, + rx: Output, +} diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index ce27d9a3..27cd1448 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -4,7 +4,7 @@ use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, In use tokio::task::JoinHandle; use warp::Filter; -use crate::{ +use crate::steps::{ between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, update_wait::UpdateWaitMetrics, }; diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/producer.rs index 483777d7..2105c9b5 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/producer.rs @@ -3,12 +3,12 @@ use nomad_ethereum::bindings::{ home::{DispatchFilter, Home, UpdateFilter}, replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, Instrument}; use crate::{ - annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, - StepHandle, + annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessStep, + RelayFaucet, Restartable, StepHandle, UpdateFaucet, }; #[derive(Debug)] @@ -48,11 +48,9 @@ impl DispatchProducer { } pub(crate) type DispatchProducerTask = Restartable; -pub(crate) type DispatchProducerHandle = StepHandle; +pub(crate) type DispatchProducerHandle = StepHandle; impl ProcessStep for DispatchProducer { - type Output = DispatchFaucet; - fn spawn(self) -> DispatchProducerTask { let span = info_span!( "DispatchProducer", @@ -136,10 +134,9 @@ impl UpdateProducer { } pub(crate) type UpdateProducerTask = Restartable; -pub(crate) type UpdateProducerHandle = StepHandle; +pub(crate) type UpdateProducerHandle = StepHandle; impl ProcessStep for UpdateProducer { - type Output = UnboundedReceiver>; fn spawn(self) -> UpdateProducerTask { let span = info_span!( "UpdateProducer", @@ -231,10 +228,9 @@ impl RelayProducer { } pub(crate) type RelayProducerTask = Restartable; -pub(crate) type RelayProducerHandle = StepHandle; +pub(crate) type RelayProducerHandle = StepHandle; impl ProcessStep for RelayProducer { - type Output = UnboundedReceiver>; fn spawn(self) -> RelayProducerTask { let span = info_span!( "RelayProducer", @@ -323,10 +319,9 @@ impl ProcessProducer { } pub(crate) type ProcessProducerTask = Restartable; -pub(crate) type ProcessProducerHandle = StepHandle; +pub(crate) type ProcessProducerHandle = StepHandle; impl ProcessStep for ProcessProducer { - type Output = UnboundedReceiver>; fn spawn(self) -> ProcessProducerTask { let span = info_span!( "ProcessProducer", diff --git a/agents/monitor/src/between.rs b/agents/monitor/src/steps/between.rs similarity index 86% rename from agents/monitor/src/between.rs rename to agents/monitor/src/steps/between.rs index f6f42beb..c5895a31 100644 --- a/agents/monitor/src/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -4,9 +4,9 @@ use nomad_ethereum::bindings::{ }; use tracing::{info_span, Instrument}; -use tokio::sync::mpsc::{self, UnboundedReceiver}; +use tokio::sync::mpsc::{self}; -use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable, StepHandle}; +use crate::{annotate::WithMeta, bail_task_if, Faucet, ProcessStep, Restartable, StepHandle}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, @@ -16,12 +16,12 @@ pub(crate) struct BetweenMetrics { // Track time between events of the same kind pub(crate) struct BetweenEvents { - pub(crate) incoming: mpsc::UnboundedReceiver, + pub(crate) faucet: mpsc::UnboundedReceiver, pub(crate) metrics: BetweenMetrics, pub(crate) network: String, pub(crate) event: String, pub(crate) emitter: String, - pub(crate) outgoing: mpsc::UnboundedSender, + pub(crate) sink: mpsc::UnboundedSender, } impl std::fmt::Display for BetweenEvents { @@ -49,25 +49,25 @@ where T: 'static + std::fmt::Debug, { pub(crate) fn new( - incoming: mpsc::UnboundedReceiver, + faucet: mpsc::UnboundedReceiver, metrics: BetweenMetrics, network: impl AsRef, event: impl AsRef, emitter: impl AsRef, - outgoing: mpsc::UnboundedSender, + sink: mpsc::UnboundedSender, ) -> Self { Self { - incoming, + faucet, metrics, network: network.as_ref().to_owned(), event: event.as_ref().to_owned(), emitter: emitter.as_ref().to_owned(), - outgoing, + sink, } } } -pub(crate) type BetweenHandle = StepHandle>; +pub(crate) type BetweenHandle = StepHandle, Faucet>; pub(crate) type BetweenTask = Restartable>; pub(crate) type BetweenDispatch = BetweenHandle>; pub(crate) type BetweenUpdate = BetweenHandle>; @@ -78,8 +78,6 @@ impl ProcessStep for BetweenEvents> where T: 'static + Send + Sync + std::fmt::Debug, { - type Output = UnboundedReceiver>; - fn spawn(mut self) -> BetweenTask> { let span = info_span!( target: "monitor::between", @@ -95,7 +93,7 @@ where loop { // get the next event from the channel - let incoming = self.incoming.recv().await; + let incoming = self.faucet.recv().await; bail_task_if!(incoming.is_none(), self, "inbound channel broke"); let incoming = incoming.expect("checked on previous line"); @@ -122,7 +120,7 @@ where // send the next event out bail_task_if!( - self.outgoing.send(incoming).is_err(), + self.sink.send(incoming).is_err(), self, "outbound channel broke" ); diff --git a/agents/monitor/src/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs similarity index 84% rename from agents/monitor/src/dispatch_wait.rs rename to agents/monitor/src/steps/dispatch_wait.rs index 3e73eab2..7d17fb2d 100644 --- a/agents/monitor/src/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -16,8 +16,8 @@ pub(crate) struct DispatchWaitMetrics { #[derive(Debug)] pub(crate) struct DispatchWait { - incoming_dispatch: DispatchFaucet, - incoming_update: UpdateFaucet, + dispatch_faucet: DispatchFaucet, + update_faucet: UpdateFaucet, network: String, emitter: String, @@ -27,8 +27,8 @@ pub(crate) struct DispatchWait { timers: Vec, blocks: Vec, - outgoing_update: UpdateSink, - outgoing_dispatch: DispatchSink, + dispatch_sink: DispatchSink, + update_sink: UpdateSink, } impl std::fmt::Display for DispatchWait { @@ -43,24 +43,24 @@ impl std::fmt::Display for DispatchWait { impl DispatchWait { pub(crate) fn new( - incoming_dispatch: DispatchFaucet, - incoming_update: UpdateFaucet, + dispatch_faucet: DispatchFaucet, + update_faucet: UpdateFaucet, network: String, emitter: String, metrics: DispatchWaitMetrics, - outgoing_update: UpdateSink, - outgoing_dispatch: DispatchSink, + dispatch_sink: DispatchSink, + update_sink: UpdateSink, ) -> Self { Self { - incoming_dispatch, - incoming_update, + dispatch_faucet, + update_faucet, network, emitter, metrics, timers: vec![], blocks: vec![], - outgoing_update, - outgoing_dispatch, + dispatch_sink, + update_sink, } } @@ -82,7 +82,7 @@ impl DispatchWait { } pub(crate) type DispatchWaitTask = Restartable; -pub(crate) type DispatchWaitHandle = StepHandle; +pub(crate) type DispatchWaitHandle = StepHandle; #[derive(Debug)] pub struct DispatchWaitOutput { @@ -91,8 +91,6 @@ pub struct DispatchWaitOutput { } impl ProcessStep for DispatchWait { - type Output = DispatchWaitOutput; - fn spawn(mut self) -> DispatchWaitTask where Self: 'static + Send + Sync + Sized, @@ -121,7 +119,7 @@ impl ProcessStep for DispatchWait { // first. i.e. ready dispatches will arrive first biased; - dispatch_next = self.incoming_dispatch.recv() => { + dispatch_next = self.dispatch_faucet.recv() => { bail_task_if!( dispatch_next.is_none(), self, @@ -130,13 +128,13 @@ impl ProcessStep for DispatchWait { let dispatch = dispatch_next.expect("checked in block"); let block_number = dispatch.meta.block_number; bail_task_if!( - self.outgoing_dispatch.send(dispatch).is_err(), + self.dispatch_sink.send(dispatch).is_err(), self, "outbound dispatch broke" ); self.handle_dispatch(block_number); } - update_opt = self.incoming_update.recv() => { + update_opt = self.update_faucet.recv() => { bail_task_if!( update_opt.is_none(), self, @@ -146,7 +144,7 @@ impl ProcessStep for DispatchWait { let block_number = update.meta.block_number; bail_task_if!( - self.outgoing_update.send(update).is_err(), + self.update_sink.send(update).is_err(), self, "outbound update broke" ); diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs new file mode 100644 index 00000000..069654a3 --- /dev/null +++ b/agents/monitor/src/steps/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod between; +pub(crate) mod dispatch_wait; +pub(crate) mod update_wait; diff --git a/agents/monitor/src/update_wait.rs b/agents/monitor/src/steps/update_wait.rs similarity index 92% rename from agents/monitor/src/update_wait.rs rename to agents/monitor/src/steps/update_wait.rs index 53ba0978..5145343d 100644 --- a/agents/monitor/src/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -1,14 +1,14 @@ use ethers::prelude::H256; use prometheus::Histogram; use std::{collections::HashMap, time::Instant}; -use tokio::sync::mpsc::unbounded_channel; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tracing::{info_span, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, RelayFaucet, RelaySink, - Restartable, StepHandle, UpdateFaucet, UpdateSink, + Restartable, UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -19,7 +19,7 @@ pub(crate) struct UpdateWaitMetrics { #[derive(Debug)] pub(crate) struct UpdateWait { update_faucet: UpdateFaucet, - relay_faucets: > as ProcessStep>::Output, + relay_faucets: UnboundedReceiver<(String, WithMeta)>, network: String, metrics: UpdateWaitMetrics, @@ -64,13 +64,6 @@ impl std::fmt::Display for UpdateWait { } pub(crate) type UpdateWaitTask = Restartable; -pub(crate) type UpdateWaitHandle = StepHandle; - -#[derive(Debug)] -pub struct UpdateWaitOutput { - pub(crate) updates: UpdateFaucet, - pub(crate) relays: HashMap, -} impl UpdateWait { fn handle_relay(&mut self, net: &str, root: H256) { @@ -109,8 +102,6 @@ impl UpdateWait { } impl ProcessStep for UpdateWait { - type Output = UpdateWaitOutput; - fn spawn(mut self) -> crate::Restartable where Self: 'static + Send + Sync + Sized, diff --git a/agents/monitor/src/terminal.rs b/agents/monitor/src/terminal.rs index e0f7f544..f0a4953f 100644 --- a/agents/monitor/src/terminal.rs +++ b/agents/monitor/src/terminal.rs @@ -28,8 +28,6 @@ impl ProcessStep for Terminal where T: std::fmt::Debug + Send + Sync + 'static, { - type Output = (); - fn spawn(mut self) -> TerminalHandle { let span = debug_span!("Terminal Handler"); tokio::spawn( diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs index 8d72eb6e..62709a08 100644 --- a/agents/monitor/src/utils.rs +++ b/agents/monitor/src/utils.rs @@ -4,19 +4,12 @@ use futures_util::future::select_all; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; -use crate::{ - bail_task_if, dispatch_wait::DispatchWaitOutput, DispatchFaucet, NetworkMap, UpdateFaucet, -}; - -use super::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; +use crate::{bail_task_if, HomeReplicaMap, ProcessStep, Restartable, StepHandle}; // split handles from outputs -pub(crate) fn split( - map: HashMap<&str, StepHandle>, -) -> ( - Vec>, - HashMap<&str, ::Output>, -) +pub(crate) fn split( + map: HashMap<&str, StepHandle>, +) -> (Vec>, HashMap<&str, U>) where T: ProcessStep, { @@ -32,12 +25,9 @@ where } // split handles from outputs in a nested map -pub(crate) fn nested_split( - map: HomeReplicaMap>, -) -> ( - Vec>, - HomeReplicaMap<::Output>, -) +pub(crate) fn nested_split( + map: HomeReplicaMap>, +) -> (Vec>, HomeReplicaMap) where T: ProcessStep, { @@ -92,8 +82,6 @@ impl ProcessStep for SelectChannels where T: 'static + Send + Sync + std::fmt::Debug, { - type Output = UnboundedReceiver<(String, T)>; - fn spawn(mut self) -> Restartable where Self: 'static + Send + Sync + Sized, @@ -120,16 +108,3 @@ where ) } } - -pub(crate) fn split_dispatch_wait_output( - mut map: HashMap<&str, DispatchWaitOutput>, -) -> (NetworkMap, NetworkMap) { - let mut dispatches = HashMap::new(); - let mut updates = HashMap::new(); - - map.drain().for_each(|(k, v)| { - dispatches.insert(k, v.dispatches); - updates.insert(k, v.updates); - }); - (dispatches, updates) -} From 51aa0c1470f66680acae59d35e1559e30b405963 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 13:57:03 -0700 Subject: [PATCH 21/80] feature: relay to process metrics --- agents/monitor/src/steps/mod.rs | 1 + agents/monitor/src/steps/relay_wait.rs | 98 ++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 agents/monitor/src/steps/relay_wait.rs diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index 069654a3..9614a4c7 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod between; pub(crate) mod dispatch_wait; +pub(crate) mod relay_wait; pub(crate) mod update_wait; diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs new file mode 100644 index 00000000..26cb3192 --- /dev/null +++ b/agents/monitor/src/steps/relay_wait.rs @@ -0,0 +1,98 @@ +use ethers::prelude::U64; +use prometheus::Histogram; +use tokio::time::Instant; + +use tracing::{info_span, Instrument}; + +use crate::{bail_task_if, ProcessFaucet, ProcessSink, ProcessStep, RelayFaucet, RelaySink}; + +pub struct RelayWaitMetrics { + timers: Histogram, + blocks: Histogram, +} + +pub(crate) struct RelayWait { + relay_faucet: RelayFaucet, + process_faucet: ProcessFaucet, + + network: String, + replica_of: String, + emitter: String, + metrics: RelayWaitMetrics, + + relay_instant: Instant, + relay_block: U64, + + relay_sink: RelaySink, + process_sink: ProcessSink, +} + +impl std::fmt::Display for RelayWait { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "RelayToProcess latency for {}'s replica of {} @ {}", + self.network, self.replica_of, self.emitter, + ) + } +} + +impl ProcessStep for RelayWait { + fn spawn(mut self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!( + "ProcessWait", + network = self.network.as_str(), + emitter = self.emitter.as_str(), + ); + tokio::spawn( + async move { + loop { + tokio::select! { + biased; + + process_next = self.process_faucet.recv() => { + bail_task_if!( + process_next.is_none(), + self, + "inbound relay broke" + ); + let process = process_next.expect("checked"); + let process_instant = tokio::time::Instant::now(); + let process_block = process.meta.block_number; + bail_task_if!( + self.process_sink.send(process).is_err(), + self, + "outbound relay broke", + ); + let elapsed_ms = process_instant.saturating_duration_since(self.relay_instant).as_millis() as f64; + let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; + + self.metrics.timers.observe(elapsed_ms); + self.metrics.blocks.observe(elapsed_blocks); + } + relay_next = self.relay_faucet.recv() => { + bail_task_if!( + relay_next.is_none(), + self, + "inbound relay broke" + ); + let relay = relay_next.expect("checked"); + self.relay_instant = tokio::time::Instant::now(); + self.relay_block = relay.meta.block_number; + bail_task_if!( + self.relay_sink.send(relay).is_err(), + self, + "outbound relay broke", + ); + } + + } + } + } + .instrument(span), + ) + } +} From dcccbd6d4d60b627bbb19ca05900efda52a2f398 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Jul 2022 13:58:43 -0700 Subject: [PATCH 22/80] chore: remove unused imports --- agents/monitor/src/steps/between.rs | 11 +---------- agents/monitor/src/steps/dispatch_wait.rs | 10 +--------- agents/monitor/src/steps/update_wait.rs | 4 +--- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index c5895a31..b4233f6e 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,12 +1,8 @@ -use nomad_ethereum::bindings::{ - home::{DispatchFilter, UpdateFilter}, - replica::{ProcessFilter, UpdateFilter as RelayFilter}, -}; use tracing::{info_span, Instrument}; use tokio::sync::mpsc::{self}; -use crate::{annotate::WithMeta, bail_task_if, Faucet, ProcessStep, Restartable, StepHandle}; +use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, @@ -67,12 +63,7 @@ where } } -pub(crate) type BetweenHandle = StepHandle, Faucet>; pub(crate) type BetweenTask = Restartable>; -pub(crate) type BetweenDispatch = BetweenHandle>; -pub(crate) type BetweenUpdate = BetweenHandle>; -pub(crate) type BetweenRelay = BetweenHandle>; -pub(crate) type BetweenProcess = BetweenHandle>; impl ProcessStep for BetweenEvents> where diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 7d17fb2d..4bfe53da 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -4,8 +4,7 @@ use tokio::select; use tracing::{info_span, Instrument}; use crate::{ - bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, StepHandle, UpdateFaucet, - UpdateSink, + bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -82,13 +81,6 @@ impl DispatchWait { } pub(crate) type DispatchWaitTask = Restartable; -pub(crate) type DispatchWaitHandle = StepHandle; - -#[derive(Debug)] -pub struct DispatchWaitOutput { - pub(crate) dispatches: DispatchFaucet, - pub(crate) updates: UpdateFaucet, -} impl ProcessStep for DispatchWait { fn spawn(mut self) -> DispatchWaitTask diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 5145343d..60c65636 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -8,7 +8,7 @@ use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, RelayFaucet, RelaySink, - Restartable, UpdateFaucet, UpdateSink, + UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -63,8 +63,6 @@ impl std::fmt::Display for UpdateWait { } } -pub(crate) type UpdateWaitTask = Restartable; - impl UpdateWait { fn handle_relay(&mut self, net: &str, root: H256) { let now = std::time::Instant::now(); From f60c2b4fab0d7f59f1f46f5a644c5cc4854236ba Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 08:34:17 -0700 Subject: [PATCH 23/80] wip --- agents/monitor/src/domain.rs | 20 ++- agents/monitor/src/init.rs | 12 +- agents/monitor/src/main.rs | 5 +- agents/monitor/src/steps/between.rs | 1 + agents/monitor/src/steps/combine.rs | 78 +++++++++ agents/monitor/src/steps/dispatch_wait.rs | 1 + agents/monitor/src/steps/e2e.rs | 186 +++++++++++++++++++++ agents/monitor/src/steps/mod.rs | 4 + agents/monitor/src/{ => steps}/producer.rs | 4 + agents/monitor/src/steps/relay_wait.rs | 31 ++++ agents/monitor/src/{ => steps}/terminal.rs | 1 + agents/monitor/src/steps/update_wait.rs | 7 +- agents/monitor/src/utils.rs | 60 +------ 13 files changed, 340 insertions(+), 70 deletions(-) create mode 100644 agents/monitor/src/steps/combine.rs create mode 100644 agents/monitor/src/steps/e2e.rs rename agents/monitor/src/{ => steps}/producer.rs (97%) rename agents/monitor/src/{ => steps}/terminal.rs (94%) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index d5ae47e6..7bd6b176 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -7,13 +7,14 @@ use tokio::sync::mpsc::unbounded_channel; use crate::{ init::provider_for, metrics::Metrics, - producer::{ - DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, - RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, - }, steps::{ between::{BetweenEvents, BetweenMetrics}, dispatch_wait::DispatchWait, + producer::{ + DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, + RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, + }, + relay_wait::RelayWait, update_wait::UpdateWait, }, Faucets, ProcessStep, Provider, StepHandle, @@ -280,4 +281,15 @@ impl Domain { ) .spawn(); } + + // TODO + // pub(crate) fn relay_to_process<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { + + // let relay_faucet = faucets.relays.get(self.) + + // RelayWait::new( + // faucet + // ); + + // } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index deef9ecf..66702e64 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -12,7 +12,7 @@ use tracing_subscriber::EnvFilter; use crate::{ domain::Domain, metrics::Metrics, - producer::{ + steps::producer::{ DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, }, utils, ArcProvider, Faucets, HomeReplicaMap, @@ -193,7 +193,13 @@ impl Monitor { pub(crate) fn run_update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>) { self.networks - .iter() - .for_each(|(_, v)| v.update_to_relay(faucets, self.metrics.clone())); + .values() + .for_each(|v| v.update_to_relay(faucets, self.metrics.clone())); + } + + pub(crate) fn run_relay_to_process<'a>(&'a self, faucets: &mut Faucets<'a>) { + self.networks + .values() + .for_each(|domain| domain.relay_to_process(faucets, self.metrics.clone())); } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 4117ce43..daf5bc0c 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -17,9 +17,7 @@ pub(crate) mod domain; pub(crate) mod init; pub(crate) mod macros; pub(crate) mod metrics; -pub(crate) mod producer; pub(crate) mod steps; -pub(crate) mod terminal; pub(crate) mod utils; pub(crate) type Provider = ethers::prelude::TimeLag>; @@ -131,8 +129,9 @@ async fn main() -> eyre::Result<()> { monitor.run_betweens(&mut faucets); monitor.run_dispatch_to_update(&mut faucets); monitor.run_update_to_relay(&mut faucets); + monitor.run_relay_to_process(&mut faucets); - tracing::info!("counters started"); + tracing::info!("tasks started"); // just run forever loop { diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index b4233f6e..790eed23 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -11,6 +11,7 @@ pub(crate) struct BetweenMetrics { } // Track time between events of the same kind +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct BetweenEvents { pub(crate) faucet: mpsc::UnboundedReceiver, pub(crate) metrics: BetweenMetrics, diff --git a/agents/monitor/src/steps/combine.rs b/agents/monitor/src/steps/combine.rs new file mode 100644 index 00000000..48a938a9 --- /dev/null +++ b/agents/monitor/src/steps/combine.rs @@ -0,0 +1,78 @@ +use futures_util::future::select_all; +use std::{collections::HashMap, fmt::Display}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use tracing::{info_span, Instrument}; + +use crate::{bail_task_if, utils::nexts, ProcessStep, Restartable}; + +#[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +pub(crate) struct CombineChannels { + pub(crate) faucets: HashMap>, + pub(crate) sink: UnboundedSender<(String, T)>, +} + +impl CombineChannels { + pub(crate) fn new( + faucets: HashMap>, + sink: UnboundedSender<(String, T)>, + ) -> Self { + Self { faucets, sink } + } + + pub(crate) fn nested( + faucets: HashMap>>, + sink: UnboundedSender<(String, (String, T))>, + ) -> CombineChannels<(String, T)> + where + T: 'static + Send + Sync + std::fmt::Debug, + { + let faucets = faucets + .into_iter() + .map(|(k, v)| { + let (sink, faucet) = unbounded_channel(); + CombineChannels::::new(v, sink).spawn(); + (k, faucet) + }) + .collect(); + + CombineChannels::<(String, T)>::new(faucets, sink) + } +} + +impl Display for CombineChannels { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "CombineChannels") + } +} + +impl ProcessStep for CombineChannels +where + T: 'static + Send + Sync + std::fmt::Debug, +{ + fn spawn(mut self) -> Restartable + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!("CombineChannels"); + tokio::spawn( + async move { + loop { + let ((net, next_opt), _, _) = select_all(nexts(&mut self.faucets)).await; + bail_task_if!( + next_opt.is_none(), + self, + format!("Inbound from {} broke", net), + ); + let next = next_opt.expect("checked"); + bail_task_if!( + self.sink.send((net, next)).is_err(), + self, + "sink channel broke" + ); + } + } + .instrument(span), + ) + } +} diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 4bfe53da..692edb6c 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -14,6 +14,7 @@ pub(crate) struct DispatchWaitMetrics { } #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct DispatchWait { dispatch_faucet: DispatchFaucet, update_faucet: UpdateFaucet, diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs new file mode 100644 index 00000000..b6709b7d --- /dev/null +++ b/agents/monitor/src/steps/e2e.rs @@ -0,0 +1,186 @@ +use std::collections::HashMap; + +use nomad_ethereum::bindings::{home::DispatchFilter, replica::ProcessFilter}; +use prometheus::Histogram; +use tokio::{ + sync::mpsc::{unbounded_channel, UnboundedReceiver}, + time::Instant, +}; + +use ethers::prelude::H256; +use tracing::{info_span, Instrument}; + +use crate::{ + annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink, + ProcessStep, +}; + +use super::combine::CombineChannels; + +pub(crate) struct E2EMetrics { + timers: HashMap, +} + +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +pub struct E2ELatency { + dispatch_faucet: UnboundedReceiver<(String, WithMeta)>, + process_faucet: UnboundedReceiver<(String, (String, WithMeta))>, + + domain_to_network: HashMap, + metrics: E2EMetrics, + + // home -> destination -> message hash -> time + dispatches: HashMap>>, + + // replica_of -> message hash -> time + processes: HashMap>, + + dispatch_sinks: HashMap, + process_sinks: HashMap>, +} + +impl std::fmt::Display for E2ELatency { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "E2E latency for the whole network") + } +} + +impl E2ELatency { + pub(crate) fn new( + dispatch_faucets: HashMap, + process_faucets: HashMap>, + domain_to_network: HashMap, + metrics: E2EMetrics, + dispatch_sinks: HashMap, + process_sinks: HashMap>, + ) -> Self { + let (process_sink, process_faucet) = unbounded_channel(); + let (dispatch_sink, dispatch_faucet) = unbounded_channel(); + + CombineChannels::new(dispatch_faucets, dispatch_sink).spawn(); + CombineChannels::nested(process_faucets, process_sink).spawn(); + + Self { + dispatch_faucet, + process_faucet, + domain_to_network, + metrics, + dispatches: Default::default(), + processes: Default::default(), + dispatch_sinks, + process_sinks, + } + } + + fn record_dispatch(&mut self, network: String, destination: u32, message_hash: H256) { + // ignore unknown destinations + if let Some(destination) = self.domain_to_network.get(&destination) { + let now = Instant::now(); + + // if we know of a matching process on the appropriate destination + // mark it + // otherwise store in dispatch map + if self + .processes + .get_mut(&network) + .and_then(|entry| entry.remove(&message_hash)) + .is_some() + { + self.metrics + .timers + .get_mut(&network) + .unwrap() + .observe(0 as f64); + } else { + self.dispatches + .entry(network.clone()) + .or_default() + .entry(destination.to_owned()) + .or_default() + .insert(message_hash, now); + } + } + + todo!() + } + + fn record_process(&mut self, network: String, replica_of: String, message_hash: H256) { + let now = Instant::now(); + + // if we know of a matching dispatch, mark it and remove from map + if let Some(dispatch) = self + .dispatches + .get_mut(&replica_of) + .and_then(|inner| inner.get_mut(&network)) + .and_then(|inner| inner.remove(&message_hash)) + { + let time = now.saturating_duration_since(dispatch).as_secs_f64(); + self.metrics + .timers + .get_mut(&replica_of) + .unwrap() + .observe(time); + } else { + // record it for later + self.processes + .entry(replica_of) + .or_default() + .insert(message_hash, now); + } + } +} + +impl ProcessStep for E2ELatency { + fn spawn(mut self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + let span = info_span!("E2ELatency"); + tokio::spawn( + async move { + loop { + tokio::select! { + dispatch_opt = self.dispatch_faucet.recv() => { + bail_task_if!( + dispatch_opt.is_none(), + self, + "inbound dispatch broke", + ); + let (network, dispatch) = dispatch_opt.expect("checked"); + let message_hash: H256 = dispatch.log.message_hash.into(); + let destination: u32 = (dispatch.log.destination_and_nonce >> 32) as u32; + + let outbound = self.dispatch_sinks.get(&network).expect("missing sink"); + + bail_task_if!( + outbound.send(dispatch).is_err(), self, "outbound dispatch broke", + ); + self.record_dispatch(network,destination, message_hash); + } + process_opt = self.process_faucet.recv() => { + bail_task_if!( + process_opt.is_none(), + self, + "inbound process broke", + ); + + let (network, (replica_of, process)) = process_opt.expect("checked"); + + let message_hash: H256 = process.log.message_hash.into(); + + let outbound = self.process_sinks.get(&network).expect("missing network").get(&replica_of).expect("missing sink"); + + bail_task_if!( + outbound.send(process).is_err(), + self, + "outbound dispatch broke", + ); + self.record_process(network, replica_of, message_hash); + } + } + } + } + .instrument(span), + ) + } +} diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index 9614a4c7..44cbb238 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -1,4 +1,8 @@ pub(crate) mod between; +pub(crate) mod combine; pub(crate) mod dispatch_wait; +pub(crate) mod e2e; +pub(crate) mod producer; pub(crate) mod relay_wait; +pub(crate) mod terminal; pub(crate) mod update_wait; diff --git a/agents/monitor/src/producer.rs b/agents/monitor/src/steps/producer.rs similarity index 97% rename from agents/monitor/src/producer.rs rename to agents/monitor/src/steps/producer.rs index 2105c9b5..a693eb69 100644 --- a/agents/monitor/src/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -12,6 +12,7 @@ use crate::{ }; #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct DispatchProducer { home: Home, network: String, @@ -98,6 +99,7 @@ impl ProcessStep for DispatchProducer { } #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct UpdateProducer { home: Home, network: String, @@ -188,6 +190,7 @@ impl ProcessStep for UpdateProducer { } #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct RelayProducer { replica: Replica, network: String, @@ -279,6 +282,7 @@ impl ProcessStep for RelayProducer { } #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct ProcessProducer { replica: Replica, network: String, diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 26cb3192..e7dea96d 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use ethers::prelude::U64; use prometheus::Histogram; use tokio::time::Instant; @@ -6,11 +8,14 @@ use tracing::{info_span, Instrument}; use crate::{bail_task_if, ProcessFaucet, ProcessSink, ProcessStep, RelayFaucet, RelaySink}; +#[derive(Debug)] pub struct RelayWaitMetrics { timers: Histogram, blocks: Histogram, } +#[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct RelayWait { relay_faucet: RelayFaucet, process_faucet: ProcessFaucet, @@ -27,6 +32,32 @@ pub(crate) struct RelayWait { process_sink: ProcessSink, } +impl RelayWait { + pub(crate) fn new( + relay_faucet: RelayFaucet, + process_faucet: ProcessFaucet, + network: String, + replica_of: String, + emitter: String, + metrics: RelayWaitMetrics, + relay_sink: RelaySink, + process_sink: ProcessSink, + ) -> Self { + Self { + relay_faucet, + process_faucet, + network, + replica_of, + emitter, + metrics, + relay_instant: Instant::now() + Duration::from_secs(86400 * 30 * 12 * 30), + relay_block: U64::zero(), + relay_sink, + process_sink, + } + } +} + impl std::fmt::Display for RelayWait { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/agents/monitor/src/terminal.rs b/agents/monitor/src/steps/terminal.rs similarity index 94% rename from agents/monitor/src/terminal.rs rename to agents/monitor/src/steps/terminal.rs index f0a4953f..1a48359a 100644 --- a/agents/monitor/src/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -4,6 +4,7 @@ use tracing::{debug_span, Instrument}; use crate::{ProcessStep, Restartable}; #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] /// A process step that just drains its input and drops everything /// Its [`StepHandle`] will never produce values. pub(crate) struct Terminal diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 60c65636..519d35d7 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -7,8 +7,8 @@ use tracing::{info_span, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ - annotate::WithMeta, bail_task_if, utils::SelectChannels, ProcessStep, RelayFaucet, RelaySink, - UpdateFaucet, UpdateSink, + annotate::WithMeta, bail_task_if, steps::combine::CombineChannels, ProcessStep, RelayFaucet, + RelaySink, UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -17,6 +17,7 @@ pub(crate) struct UpdateWaitMetrics { } #[derive(Debug)] +#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct UpdateWait { update_faucet: UpdateFaucet, relay_faucets: UnboundedReceiver<(String, WithMeta)>, @@ -42,7 +43,7 @@ impl UpdateWait { ) -> Self { let (tx, rx) = unbounded_channel(); - SelectChannels::new(relay_faucets, tx).spawn(); + CombineChannels::new(relay_faucets, tx).spawn(); Self { update_faucet, diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs index 62709a08..bdcc4cb6 100644 --- a/agents/monitor/src/utils.rs +++ b/agents/monitor/src/utils.rs @@ -1,10 +1,8 @@ -use std::{collections::HashMap, fmt::Display, pin::Pin}; +use std::{collections::HashMap, pin::Pin}; -use futures_util::future::select_all; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tracing::{info_span, Instrument}; +use tokio::sync::mpsc::UnboundedReceiver; -use crate::{bail_task_if, HomeReplicaMap, ProcessStep, Restartable, StepHandle}; +use crate::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; // split handles from outputs pub(crate) fn split( @@ -56,55 +54,3 @@ pub(crate) fn nexts( .map(Box::pin) .collect() } - -#[derive(Debug)] -pub(crate) struct SelectChannels { - channels: HashMap>, - outbound: UnboundedSender<(String, T)>, -} - -impl SelectChannels { - pub(crate) fn new( - channels: HashMap>, - outbound: UnboundedSender<(String, T)>, - ) -> Self { - Self { channels, outbound } - } -} - -impl Display for SelectChannels { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "SelectChannels") - } -} - -impl ProcessStep for SelectChannels -where - T: 'static + Send + Sync + std::fmt::Debug, -{ - fn spawn(mut self) -> Restartable - where - Self: 'static + Send + Sync + Sized, - { - let span = info_span!("SelectChannels"); - tokio::spawn( - async move { - loop { - let ((net, next_opt), _, _) = select_all(nexts(&mut self.channels)).await; - bail_task_if!( - next_opt.is_none(), - self, - format!("Inbound from {} broke", net), - ); - let next = next_opt.expect("checked"); - bail_task_if!( - self.outbound.send((net, next)).is_err(), - self, - "outbound channel broke" - ); - } - } - .instrument(span), - ) - } -} From 5702cda6e982aba4bc09dbbcef57fb660897c9bb Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 10:01:15 -0700 Subject: [PATCH 24/80] feature: pipe process-once semantics --- agents/monitor/src/domain.rs | 27 +++++-- agents/monitor/src/faucets.rs | 103 +++++++++++++++++++++++++ agents/monitor/src/init.rs | 7 +- agents/monitor/src/macros.rs | 15 ++++ agents/monitor/src/main.rs | 93 +++------------------- agents/monitor/src/metrics.rs | 79 ++++++++++++++++++- agents/monitor/src/pipe.rs | 47 +++++++++++ agents/monitor/src/steps/e2e.rs | 3 +- agents/monitor/src/steps/relay_wait.rs | 62 +++++---------- 9 files changed, 303 insertions(+), 133 deletions(-) create mode 100644 agents/monitor/src/faucets.rs create mode 100644 agents/monitor/src/pipe.rs diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 7bd6b176..051b8ee2 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -5,6 +5,7 @@ use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; use tokio::sync::mpsc::unbounded_channel; use crate::{ + faucets::Faucets, init::provider_for, metrics::Metrics, steps::{ @@ -17,7 +18,7 @@ use crate::{ relay_wait::RelayWait, update_wait::UpdateWait, }, - Faucets, ProcessStep, Provider, StepHandle, + ProcessStep, Provider, StepHandle, }; #[derive(Debug)] @@ -282,14 +283,24 @@ impl Domain { .spawn(); } - // TODO - // pub(crate) fn relay_to_process<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { + pub(crate) fn relay_to_process<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { + self.replicas.iter().for_each(|(replica_of, replica)| { + let emitter = format!("{:?}", replica.address()); - // let relay_faucet = faucets.relays.get(self.) + let relay_pipe = faucets.relay_pipe(self.name(), replica_of); + let process_pipe = faucets.process_pipe(self.name(), replica_of); - // RelayWait::new( - // faucet - // ); + let metrics = metrics.relay_wait_metrics(self.name(), replica_of, &emitter); - // } + RelayWait::new( + relay_pipe, + process_pipe, + self.name().to_owned(), + replica_of.to_owned(), + emitter, + metrics, + ) + .spawn(); + }); + } } diff --git a/agents/monitor/src/faucets.rs b/agents/monitor/src/faucets.rs new file mode 100644 index 00000000..8bba3f6e --- /dev/null +++ b/agents/monitor/src/faucets.rs @@ -0,0 +1,103 @@ +use nomad_ethereum::bindings::home::UpdateFilter; +use tokio::sync::mpsc::unbounded_channel; + +use crate::{ + annotate::WithMeta, + pipe::{DispatchPipe, Pipe, ProcessPipe, RelayPipe, UpdatePipe}, + DispatchFaucet, HomeReplicaMap, NetworkMap, ProcessFaucet, RelayFaucet, UpdateFaucet, +}; + +pub(crate) struct Faucets<'a> { + pub(crate) dispatches: NetworkMap<'a, DispatchFaucet>, + pub(crate) updates: NetworkMap<'a, UpdateFaucet>, + pub(crate) relays: HomeReplicaMap<'a, RelayFaucet>, + pub(crate) processes: HomeReplicaMap<'a, ProcessFaucet>, +} + +impl<'a> Faucets<'a> { + pub(crate) fn swap_dispatch( + &mut self, + network: &'a str, + mut dispatch_faucet: DispatchFaucet, + ) -> DispatchFaucet { + self.dispatches + .get_mut(network) + .map(|old| { + std::mem::swap(old, &mut dispatch_faucet); + dispatch_faucet + }) + .expect("missing dispatch faucet") + } + + pub(crate) fn swap_update( + &mut self, + network: &'a str, + mut update_faucet: UpdateFaucet, + ) -> UpdateFaucet { + self.updates + .get_mut(network) + .map(|old| { + std::mem::swap(old, &mut update_faucet); + update_faucet + }) + .expect("missing dispatch faucet") + } + + pub(crate) fn swap_relay( + &mut self, + network: &'a str, + replica_of: &'a str, + mut relay_faucet: RelayFaucet, + ) -> RelayFaucet { + self.relays + .get_mut(network) + .expect("missing network") + .get_mut(replica_of) + .map(|old| { + std::mem::swap(old, &mut relay_faucet); + relay_faucet + }) + .expect("missing faucet") + } + + pub(crate) fn swap_process( + &mut self, + network: &'a str, + replica_of: &'a str, + mut process_faucet: ProcessFaucet, + ) -> ProcessFaucet { + self.processes + .get_mut(network) + .expect("missing network") + .get_mut(replica_of) + .map(|old| { + std::mem::swap(old, &mut process_faucet); + process_faucet + }) + .expect("missing faucet") + } + + pub(crate) fn dispatch_pipe(&mut self, network: &'a str) -> DispatchPipe { + let (tx, rx) = unbounded_channel(); + let rx = self.swap_dispatch(network, rx); + Pipe::new(rx, tx, None) + } + + pub(crate) fn update_pipe(&mut self, network: &'a str) -> UpdatePipe { + let (tx, rx) = unbounded_channel(); + let rx = self.swap_update(network, rx); + Pipe::new(rx, tx, None) + } + + pub(crate) fn relay_pipe(&mut self, network: &'a str, replica_of: &'a str) -> RelayPipe { + let (tx, rx) = unbounded_channel(); + let rx = self.swap_relay(network, replica_of, rx); + Pipe::new(rx, tx, None) + } + + pub(crate) fn process_pipe(&mut self, network: &'a str, replica_of: &'a str) -> ProcessPipe { + let (tx, rx) = unbounded_channel(); + let rx = self.swap_process(network, replica_of, rx); + Pipe::new(rx, tx, None) + } +} diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 66702e64..07b1d8eb 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -11,11 +11,12 @@ use tracing_subscriber::EnvFilter; use crate::{ domain::Domain, + faucets::Faucets, metrics::Metrics, steps::producer::{ DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, }, - utils, ArcProvider, Faucets, HomeReplicaMap, + utils, ArcProvider, HomeReplicaMap, }; pub(crate) fn config_from_file() -> Option { @@ -202,4 +203,8 @@ impl Monitor { .values() .for_each(|domain| domain.relay_to_process(faucets, self.metrics.clone())); } + + pub(crate) fn run_e2e<'a>(&'a self, faucets: &mut Faucets<'a>) { + todo!() + } } diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index 3d0d1d6a..aa5c00b6 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -12,3 +12,18 @@ macro_rules! bail_task_if { } }; } + +#[macro_export] +macro_rules! unwrap_pipe_item { + ($pipe_output:ident, $self:ident,) => {{ + unwrap_pipe_output!($pipe_output, $self) + }}; + ($pipe_output:ident, $self:ident) => {{ + bail_task_if!($pipe_output.is_err(), $self, $pipe_output.unwrap_err(),); + + let item_opt = $pipe_output.unwrap(); + bail_task_if!(item_opt.is_none(), $self, "inbound pipe failed",); + + item_opt.unwrap() + }}; +} diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index daf5bc0c..847a6242 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -14,9 +14,11 @@ use ethers::prelude::{Http, Provider as EthersProvider}; pub(crate) mod annotate; pub(crate) mod domain; +pub(crate) mod faucets; pub(crate) mod init; pub(crate) mod macros; pub(crate) mod metrics; +pub(crate) mod pipe; pub(crate) mod steps; pub(crate) mod utils; @@ -26,92 +28,21 @@ pub(crate) type ArcProvider = Arc; pub(crate) type Restartable = JoinHandle<(Task, eyre::Report)>; -pub(crate) type Faucet = UnboundedReceiver>; -pub(crate) type Sink = UnboundedSender>; +pub(crate) type Faucet = UnboundedReceiver; +pub(crate) type Sink = UnboundedSender; -pub(crate) type DispatchFaucet = Faucet; -pub(crate) type UpdateFaucet = Faucet; -pub(crate) type RelayFaucet = Faucet; -pub(crate) type ProcessFaucet = Faucet; -pub(crate) type DispatchSink = Sink; -pub(crate) type UpdateSink = Sink; -pub(crate) type RelaySink = Sink; -pub(crate) type ProcessSink = Sink; +pub(crate) type DispatchFaucet = Faucet>; +pub(crate) type UpdateFaucet = Faucet>; +pub(crate) type RelayFaucet = Faucet>; +pub(crate) type ProcessFaucet = Faucet>; +pub(crate) type DispatchSink = Sink>; +pub(crate) type UpdateSink = Sink>; +pub(crate) type RelaySink = Sink>; +pub(crate) type ProcessSink = Sink>; pub(crate) type NetworkMap<'a, T> = HashMap<&'a str, T>; pub(crate) type HomeReplicaMap<'a, T> = HashMap<&'a str, HashMap<&'a str, T>>; -pub(crate) struct Faucets<'a> { - pub(crate) dispatches: NetworkMap<'a, DispatchFaucet>, - pub(crate) updates: NetworkMap<'a, UpdateFaucet>, - pub(crate) relays: HomeReplicaMap<'a, RelayFaucet>, - pub(crate) processes: HomeReplicaMap<'a, ProcessFaucet>, -} - -impl<'a> Faucets<'a> { - pub(crate) fn swap_dispatch( - &mut self, - network: &'a str, - mut dispatch_faucet: DispatchFaucet, - ) -> DispatchFaucet { - self.dispatches - .get_mut(network) - .map(|old| { - std::mem::swap(old, &mut dispatch_faucet); - dispatch_faucet - }) - .expect("missing dispatch faucet") - } - - pub(crate) fn swap_update( - &mut self, - network: &'a str, - mut update_faucet: UpdateFaucet, - ) -> UpdateFaucet { - self.updates - .get_mut(network) - .map(|old| { - std::mem::swap(old, &mut update_faucet); - update_faucet - }) - .expect("missing dispatch faucet") - } - - pub(crate) fn swap_relay( - &mut self, - network: &'a str, - replica_of: &'a str, - mut relay_faucet: RelayFaucet, - ) -> RelayFaucet { - self.relays - .get_mut(network) - .expect("missing network") - .get_mut(replica_of) - .map(|old| { - std::mem::swap(old, &mut relay_faucet); - relay_faucet - }) - .expect("missing faucet") - } - - pub(crate) fn swap_process( - &mut self, - network: &'a str, - replica_of: &'a str, - mut process_faucet: ProcessFaucet, - ) -> ProcessFaucet { - self.processes - .get_mut(network) - .expect("missing network") - .get_mut(replica_of) - .map(|old| { - std::mem::swap(old, &mut process_faucet); - process_faucet - }) - .expect("missing faucet") - } -} - #[tokio::main] async fn main() -> eyre::Result<()> { init::init_tracing(); diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 27cd1448..fb69b3ce 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -5,7 +5,8 @@ use tokio::task::JoinHandle; use warp::Filter; use crate::steps::{ - between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, update_wait::UpdateWaitMetrics, + between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, e2e::E2EMetrics, + relay_wait::RelayWaitMetrics, update_wait::UpdateWaitMetrics, }; #[derive(Debug)] @@ -18,6 +19,11 @@ pub(crate) struct Metrics { dispatch_to_update_blocks: prometheus::HistogramVec, update_to_relay_timers: prometheus::HistogramVec, + + relay_to_process_timers: prometheus::HistogramVec, + relay_to_process_blocks: prometheus::HistogramVec, + + e2e_timers: prometheus::HistogramVec, } fn u16_from_env(s: impl AsRef) -> Option { @@ -34,6 +40,16 @@ fn gather() -> prometheus::Result> { impl Metrics { pub(crate) fn new() -> eyre::Result { + let e2e_timers = HistogramVec::new( + HistogramOpts::new( + "e2e_ms", + "Ms between dispatch and associated process, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain"], + )?; + let update_to_relay_timers = HistogramVec::new( HistogramOpts::new( "update_to_relay_ms", @@ -64,6 +80,26 @@ impl Metrics { &["chain", "emitter"], )?; + let relay_to_process_timers = HistogramVec::new( + HistogramOpts::new( + "relay_to_process_ms", + "Ms between relay and process, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "replica_of", "emitter"], + )?; + + let relay_to_process_blocks = HistogramVec::new( + HistogramOpts::new( + "relay_to_process_blocks", + "Blocks between dispatch and update, as observed by this agent", + ) + .namespace("nomad") + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "replica_of", "emitter"], + )?; + let wallclock_times = HistogramVec::new( HistogramOpts::new( "inter_event_period_wallclock_ms", @@ -110,6 +146,15 @@ impl Metrics { registry .register(Box::new(dispatch_to_update_blocks.clone())) .expect("unable to register metric"); + registry + .register(Box::new(update_to_relay_timers.clone())) + .expect("unable to register metric"); + registry + .register(Box::new(relay_to_process_timers.clone())) + .expect("unable to register metric"); + registry + .register(Box::new(relay_to_process_blocks.clone())) + .expect("unable to register metric"); Ok(Self { wallclock_times, @@ -118,6 +163,9 @@ impl Metrics { dispatch_to_update_blocks, dispatch_to_update_timers, update_to_relay_timers, + relay_to_process_timers, + relay_to_process_blocks, + e2e_timers, }) } @@ -228,4 +276,33 @@ impl Metrics { .with_label_values(&[network, emitter]), } } + + pub(crate) fn relay_wait_metrics( + &self, + network: &str, + replica_of: &str, + emitter: &str, + ) -> RelayWaitMetrics { + RelayWaitMetrics { + timers: self + .relay_to_process_timers + .with_label_values(&[network, replica_of, emitter]), + blocks: self + .relay_to_process_blocks + .with_label_values(&[network, replica_of, emitter]), + } + } + + pub(crate) fn e2e_metrics<'a>(&self, networks: impl Iterator) -> E2EMetrics { + let timers = networks + .map(|network| { + ( + network.to_owned(), + self.e2e_timers.with_label_values(&[network]), + ) + }) + .collect(); + + E2EMetrics { timers } + } } diff --git a/agents/monitor/src/pipe.rs b/agents/monitor/src/pipe.rs new file mode 100644 index 00000000..662f325d --- /dev/null +++ b/agents/monitor/src/pipe.rs @@ -0,0 +1,47 @@ +use std::fmt::Debug; + +use eyre::bail; +use nomad_ethereum::bindings::{ + home::{DispatchFilter, UpdateFilter}, + replica::{ProcessFilter, UpdateFilter as RelayFilter}, +}; + +use crate::{annotate::WithMeta, Faucet, Sink}; + +#[derive(Debug)] +pub(crate) struct Pipe { + rx: Faucet, + tx: Sink, + contents: Option, +} + +pub(crate) type DispatchPipe = Pipe>; +pub(crate) type UpdatePipe = Pipe>; +pub(crate) type RelayPipe = Pipe>; +pub(crate) type ProcessPipe = Pipe>; + +impl Pipe +where + T: Debug + Send + Sync + 'static, +{ + pub(crate) fn new(rx: Faucet, tx: Sink, contents: Option) -> Self { + Self { rx, tx, contents } + } + + pub(crate) fn read(&self) -> Option<&T> { + self.contents.as_ref() + } + + /// Get the + pub(crate) async fn next(&mut self) -> eyre::Result> { + if let Some(contents) = self.contents.take() { + self.tx.send(contents)?; + } + + self.contents = self.rx.recv().await; + if self.contents.is_none() { + bail!("Rx Broke") + } + Ok(self.read()) + } +} diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index b6709b7d..202f8ec2 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -18,7 +18,8 @@ use crate::{ use super::combine::CombineChannels; pub(crate) struct E2EMetrics { - timers: HashMap, + // home to times + pub(crate) timers: HashMap, } #[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index e7dea96d..39eac126 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -6,19 +6,23 @@ use tokio::time::Instant; use tracing::{info_span, Instrument}; -use crate::{bail_task_if, ProcessFaucet, ProcessSink, ProcessStep, RelayFaucet, RelaySink}; +use crate::{ + bail_task_if, + pipe::{ProcessPipe, RelayPipe}, + unwrap_pipe_item, ProcessStep, +}; #[derive(Debug)] -pub struct RelayWaitMetrics { - timers: Histogram, - blocks: Histogram, +pub(crate) struct RelayWaitMetrics { + pub(crate) timers: Histogram, + pub(crate) blocks: Histogram, } #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct RelayWait { - relay_faucet: RelayFaucet, - process_faucet: ProcessFaucet, + relay_pipe: RelayPipe, + process_pipe: ProcessPipe, network: String, replica_of: String, @@ -27,33 +31,28 @@ pub(crate) struct RelayWait { relay_instant: Instant, relay_block: U64, - - relay_sink: RelaySink, - process_sink: ProcessSink, } impl RelayWait { + #[allow(clippy::too_many_arguments)] pub(crate) fn new( - relay_faucet: RelayFaucet, - process_faucet: ProcessFaucet, + relay_pipe: RelayPipe, + process_pipe: ProcessPipe, + network: String, replica_of: String, emitter: String, metrics: RelayWaitMetrics, - relay_sink: RelaySink, - process_sink: ProcessSink, ) -> Self { Self { - relay_faucet, - process_faucet, + relay_pipe, + process_pipe, network, replica_of, emitter, metrics, relay_instant: Instant::now() + Duration::from_secs(86400 * 30 * 12 * 30), relay_block: U64::zero(), - relay_sink, - process_sink, } } } @@ -84,40 +83,21 @@ impl ProcessStep for RelayWait { tokio::select! { biased; - process_next = self.process_faucet.recv() => { - bail_task_if!( - process_next.is_none(), - self, - "inbound relay broke" - ); - let process = process_next.expect("checked"); + process_next = self.process_pipe.next() => { + let process = unwrap_pipe_item!(process_next, self); let process_instant = tokio::time::Instant::now(); let process_block = process.meta.block_number; - bail_task_if!( - self.process_sink.send(process).is_err(), - self, - "outbound relay broke", - ); + let elapsed_ms = process_instant.saturating_duration_since(self.relay_instant).as_millis() as f64; let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; self.metrics.timers.observe(elapsed_ms); self.metrics.blocks.observe(elapsed_blocks); } - relay_next = self.relay_faucet.recv() => { - bail_task_if!( - relay_next.is_none(), - self, - "inbound relay broke" - ); - let relay = relay_next.expect("checked"); + relay_next = self.relay_pipe.next() => { + let relay = unwrap_pipe_item!(relay_next, self); self.relay_instant = tokio::time::Instant::now(); self.relay_block = relay.meta.block_number; - bail_task_if!( - self.relay_sink.send(relay).is_err(), - self, - "outbound relay broke", - ); } } From 77154ecab8c2f12e1da48bd592fac1a251029e4e Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 10:11:07 -0700 Subject: [PATCH 25/80] refactor: between uses pipes --- agents/monitor/src/domain.rs | 30 ++++++++++++++--------------- agents/monitor/src/faucets.rs | 2 -- agents/monitor/src/steps/between.rs | 25 ++++++++---------------- 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 051b8ee2..8064ff7e 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -139,16 +139,20 @@ impl Domain { let network = self.network.clone(); let emitter = self.home_address(); - let (sink, faucet) = unbounded_channel(); - let faucet = faucets.swap_dispatch(self.name(), faucet); - tracing::debug!( network = network, home = emitter, event = event.as_ref(), "starting counter", ); - BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); + BetweenEvents::new( + faucets.dispatch_pipe(self.name()), + metrics, + network, + event, + emitter, + ) + .spawn(); } pub(crate) fn count_updates<'a>( @@ -160,16 +164,14 @@ impl Domain { let network = self.network.clone(); let emitter = self.home_address(); - let (sink, faucet) = unbounded_channel(); - let faucet = faucets.swap_update(self.name(), faucet); - + let pipe = faucets.update_pipe(&self.network); tracing::debug!( network = network, home = emitter, event = event.as_ref(), "starting counter", ); - BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); } pub(crate) fn count_relays<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { @@ -184,13 +186,11 @@ impl Domain { "starting counter", ); - let (sink, faucet) = unbounded_channel(); - - let faucet = faucets.swap_relay(self.name(), replica_of, faucet); + let pipe = faucets.relay_pipe(&self.network, replica_of); let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); }); } @@ -206,13 +206,11 @@ impl Domain { "starting counter", ); - let (sink, faucet) = unbounded_channel(); - - let faucet = faucets.swap_process(self.name(), replica_of, faucet); + let pipe = faucets.process_pipe(&self.network, replica_of); let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - BetweenEvents::new(faucet, metrics, network, event, emitter, sink).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); }); } diff --git a/agents/monitor/src/faucets.rs b/agents/monitor/src/faucets.rs index 8bba3f6e..59a6603b 100644 --- a/agents/monitor/src/faucets.rs +++ b/agents/monitor/src/faucets.rs @@ -1,8 +1,6 @@ -use nomad_ethereum::bindings::home::UpdateFilter; use tokio::sync::mpsc::unbounded_channel; use crate::{ - annotate::WithMeta, pipe::{DispatchPipe, Pipe, ProcessPipe, RelayPipe, UpdatePipe}, DispatchFaucet, HomeReplicaMap, NetworkMap, ProcessFaucet, RelayFaucet, UpdateFaucet, }; diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 790eed23..45e3b82a 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -2,7 +2,9 @@ use tracing::{info_span, Instrument}; use tokio::sync::mpsc::{self}; -use crate::{annotate::WithMeta, bail_task_if, ProcessStep, Restartable}; +use crate::{ + annotate::WithMeta, bail_task_if, pipe::Pipe, unwrap_pipe_item, ProcessStep, Restartable, +}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, @@ -13,12 +15,11 @@ pub(crate) struct BetweenMetrics { // Track time between events of the same kind #[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct BetweenEvents { - pub(crate) faucet: mpsc::UnboundedReceiver, + pub(crate) pipe: Pipe, pub(crate) metrics: BetweenMetrics, pub(crate) network: String, pub(crate) event: String, pub(crate) emitter: String, - pub(crate) sink: mpsc::UnboundedSender, } impl std::fmt::Display for BetweenEvents { @@ -46,20 +47,18 @@ where T: 'static + std::fmt::Debug, { pub(crate) fn new( - faucet: mpsc::UnboundedReceiver, + pipe: Pipe, metrics: BetweenMetrics, network: impl AsRef, event: impl AsRef, emitter: impl AsRef, - sink: mpsc::UnboundedSender, ) -> Self { Self { - faucet, + pipe, metrics, network: network.as_ref().to_owned(), event: event.as_ref().to_owned(), emitter: emitter.as_ref().to_owned(), - sink, } } } @@ -85,10 +84,9 @@ where loop { // get the next event from the channel - let incoming = self.faucet.recv().await; + let incoming = self.pipe.next().await; - bail_task_if!(incoming.is_none(), self, "inbound channel broke"); - let incoming = incoming.expect("checked on previous line"); + let incoming = unwrap_pipe_item!(incoming, self); tracing::debug!( target: "monitor::between", @@ -110,13 +108,6 @@ where self.metrics.count.inc(); wallclock_latency.observe_duration(); - // send the next event out - bail_task_if!( - self.sink.send(incoming).is_err(), - self, - "outbound channel broke" - ); - // restart the timer wallclock_latency = self.metrics.wallclock_latency.start_timer(); } From 08587f6f6830c9b55f8355651178e3d05d7e0e58 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 10:55:09 -0700 Subject: [PATCH 26/80] feature: e2e bootup --- agents/monitor/src/domain.rs | 7 +++ agents/monitor/src/faucets.rs | 73 ++++++++++++++++++++++++++++- agents/monitor/src/init.rs | 33 +++++++++++-- agents/monitor/src/main.rs | 1 + agents/monitor/src/steps/between.rs | 2 - configuration/CHANGELOG.md | 1 + configuration/src/network.rs | 5 ++ 7 files changed, 115 insertions(+), 7 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 8064ff7e..b63d9dc0 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -24,6 +24,7 @@ use crate::{ #[derive(Debug)] pub(crate) struct Domain { pub(crate) network: String, + pub(crate) domain_number: u32, pub(crate) home: Home, pub(crate) replicas: HashMap>, } @@ -37,6 +38,11 @@ impl Domain { let network = network.to_owned(); let provider = provider_for(config, &network)?; + let domain_number = config + .protocol() + .resolve_name_to_domain(&network) + .expect("invalid config"); + let CoreContracts::Evm(core) = config.core().get(&network).expect("invalid config"); let home = Home::new(core.home.proxy.as_ethereum_address()?, provider.clone()); @@ -57,6 +63,7 @@ impl Domain { network, home, replicas, + domain_number, }) } diff --git a/agents/monitor/src/faucets.rs b/agents/monitor/src/faucets.rs index 59a6603b..97ca3e35 100644 --- a/agents/monitor/src/faucets.rs +++ b/agents/monitor/src/faucets.rs @@ -1,8 +1,11 @@ +use std::collections::HashMap; + use tokio::sync::mpsc::unbounded_channel; use crate::{ pipe::{DispatchPipe, Pipe, ProcessPipe, RelayPipe, UpdatePipe}, - DispatchFaucet, HomeReplicaMap, NetworkMap, ProcessFaucet, RelayFaucet, UpdateFaucet, + DispatchFaucet, DispatchSink, HomeReplicaMap, NetworkMap, ProcessFaucet, ProcessSink, + RelayFaucet, UpdateFaucet, }; pub(crate) struct Faucets<'a> { @@ -98,4 +101,72 @@ impl<'a> Faucets<'a> { let rx = self.swap_process(network, replica_of, rx); Pipe::new(rx, tx, None) } + + // used for e2e + pub(crate) fn swap_all_dispatches( + &mut self, + ) -> ( + HashMap, + HashMap, + ) { + let mut sinks = HashMap::new(); + + let mut dispatches = HashMap::new(); + + self.dispatches.keys().for_each(|key| { + let (sink, faucet) = unbounded_channel(); + sinks.insert(key.to_string(), sink); + dispatches.insert(*key, faucet); + }); + + std::mem::swap(&mut self.dispatches, &mut dispatches); + + let faucets = dispatches + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + + (sinks, faucets) + } + + #[allow(clippy::type_complexity)] + pub(crate) fn swap_all_processes( + &mut self, + ) -> ( + HashMap>, + HashMap>, + ) { + let mut sinks: HashMap> = HashMap::new(); + let mut processes: HomeReplicaMap = HashMap::new(); + + self.processes.iter().for_each(|(network, map)| { + map.keys().for_each(|replica_of| { + let (sink, faucet) = unbounded_channel(); + + processes + .entry(network) + .or_default() + .insert(replica_of, faucet); + + sinks + .entry(network.to_string()) + .or_default() + .insert(replica_of.to_string(), sink); + }); + }); + + std::mem::swap(&mut self.processes, &mut processes); + + let faucets = processes + .into_iter() + .map(|(network, map)| { + let map = map + .into_iter() + .map(|(replica_of, faucet)| (replica_of.to_string(), faucet)) + .collect(); + (network.to_string(), map) + }) + .collect(); + (sinks, faucets) + } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 07b1d8eb..a41e5df3 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -13,10 +13,14 @@ use crate::{ domain::Domain, faucets::Faucets, metrics::Metrics, - steps::producer::{ - DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, UpdateProducerHandle, + steps::{ + e2e::E2ELatency, + producer::{ + DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, + UpdateProducerHandle, + }, }, - utils, ArcProvider, HomeReplicaMap, + utils, ArcProvider, HomeReplicaMap, ProcessStep, }; pub(crate) fn config_from_file() -> Option { @@ -205,6 +209,27 @@ impl Monitor { } pub(crate) fn run_e2e<'a>(&'a self, faucets: &mut Faucets<'a>) { - todo!() + let (process_sinks, process_faucets) = faucets.swap_all_processes(); + let (dispatch_sinks, dispatch_faucets) = faucets.swap_all_dispatches(); + + let metrics = self + .metrics + .e2e_metrics(process_sinks.keys().map(AsRef::as_ref)); + + let domain_to_network = self + .networks + .values() + .map(|domain| (domain.domain_number, domain.name().to_owned())) + .collect(); + + E2ELatency::new( + dispatch_faucets, + process_faucets, + domain_to_network, + metrics, + dispatch_sinks, + process_sinks, + ) + .spawn(); } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 847a6242..b2343824 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -61,6 +61,7 @@ async fn main() -> eyre::Result<()> { monitor.run_dispatch_to_update(&mut faucets); monitor.run_update_to_relay(&mut faucets); monitor.run_relay_to_process(&mut faucets); + monitor.run_e2e(&mut faucets); tracing::info!("tasks started"); diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 45e3b82a..9eae7cfa 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,7 +1,5 @@ use tracing::{info_span, Instrument}; -use tokio::sync::mpsc::{self}; - use crate::{ annotate::WithMeta, bail_task_if, pipe::Pipe, unwrap_pipe_item, ProcessStep, Restartable, }; diff --git a/configuration/CHANGELOG.md b/configuration/CHANGELOG.md index 205c35ad..f355a486 100644 --- a/configuration/CHANGELOG.md +++ b/configuration/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- feature: add `resolve_domain_to_number` convenience function to `NetworkInfo` - feature: add more explicit Processor Config TS declaration - refactor: make Processor config keys optional, and prevent trivial ser. - fix: update TS AgentConfig to match rust diff --git a/configuration/src/network.rs b/configuration/src/network.rs index 95182e72..76190d53 100644 --- a/configuration/src/network.rs +++ b/configuration/src/network.rs @@ -147,4 +147,9 @@ impl NetworkInfo { pub fn networks(&self) -> HashSet { self.networks.keys().map(ToOwned::to_owned).collect() } + + /// resolve a name to a number + pub fn resolve_name_to_domain(&self, name: &str) -> Option { + self.networks.get(name).map(|net| net.domain) + } } From 2d04988f7b409cbcd63d3467f7334702a44a45b9 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 10:57:52 -0700 Subject: [PATCH 27/80] chore: update dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 87a89129..2605f61b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ COPY ${TARGET_DIR}/release/updater \ ${TARGET_DIR}/release/watcher \ ${TARGET_DIR}/release/processor \ ${TARGET_DIR}/release/kathy \ + ${TARGET_DIR}/release/monitor \ ${TARGET_DIR}/release/kms-cli \ ${TARGET_DIR}/release/nomad-cli ./ From d5748db067f72aa668af5a704d1c313d7dcab6dc Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 12:04:47 -0700 Subject: [PATCH 28/80] chore: move magic numbers to named consts --- agents/monitor/src/steps/producer.rs | 35 +++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index a693eb69..38855c36 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -11,6 +11,9 @@ use crate::{ RelayFaucet, Restartable, StepHandle, UpdateFaucet, }; +pub const POLLING_INTERVAL_SECS: u64 = 5; +pub const BEHIND_TIP: u64 = 5; + #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct DispatchProducer { @@ -64,8 +67,8 @@ impl ProcessStep for DispatchProducer { async move { let provider = self.home.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; + let mut from = height - (2 * BEHIND_TIP); + let mut to = height - BEHIND_TIP; loop { if from < to { let res = self @@ -86,11 +89,11 @@ impl ProcessStep for DispatchProducer { let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - 5; + let tip = tip_res.unwrap() - BEHIND_TIP; from = to; to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; } } .instrument(span), @@ -155,8 +158,8 @@ impl ProcessStep for UpdateProducer { bail_task_if!(height.is_err(), self, "Err retrieving height"); let height = height.expect("checked"); - let mut from = height - 10; - let mut to = height - 5; + let mut from = height - (2 * BEHIND_TIP); + let mut to = height - BEHIND_TIP; loop { if from < to { let res = self @@ -177,11 +180,11 @@ impl ProcessStep for UpdateProducer { let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.expect("checked") - 5; + let tip = tip_res.expect("checked") - BEHIND_TIP; from = to; to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; } } .instrument(span), @@ -247,8 +250,8 @@ impl ProcessStep for RelayProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; + let mut from = height - (2 * BEHIND_TIP); + let mut to = height - BEHIND_TIP; loop { tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); if from < to { @@ -269,11 +272,11 @@ impl ProcessStep for RelayProducer { } let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - 5; + let tip = tip_res.unwrap() - BEHIND_TIP; from = to; to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; } } .instrument(span), @@ -339,8 +342,8 @@ impl ProcessStep for ProcessProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - 10; - let mut to = height - 5; + let mut from = height - (2 * BEHIND_TIP); + let mut to = height - BEHIND_TIP; loop { if from < to { let res = self @@ -361,11 +364,11 @@ impl ProcessStep for ProcessProducer { let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - 5; + let tip = tip_res.unwrap() - BEHIND_TIP; from = to; to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; + tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; } } .instrument(span), From 23b8fc6c6ae49f5f089a78bd1e738b1037b481fd Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Jul 2022 13:39:12 -0700 Subject: [PATCH 29/80] fix: remove debug todo!() --- agents/monitor/src/steps/e2e.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 202f8ec2..8044c6f5 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -101,8 +101,6 @@ impl E2ELatency { .insert(message_hash, now); } } - - todo!() } fn record_process(&mut self, network: String, replica_of: String, message_hash: H256) { From a27e4fb368f64e2d274945c84fdbdf8271c77801 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 Jul 2022 09:22:34 -0700 Subject: [PATCH 30/80] chore: update Cargo.toml --- agents/monitor/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index 4a136274..d0f35b64 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -2,6 +2,11 @@ name = "monitor" version = "0.1.0" edition = "2021" +authors = ["James Prestwich "] +description = "Network metrics for Nomad" +repository = "https://github.com/nomad-xyz/rust" +license = "MIT OR Apache-2.0" +keywords = ["Ethereum", "Nomad"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 85989ee8be07309c780560af51fccb7b183057bf Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 Jul 2022 11:35:27 -0700 Subject: [PATCH 31/80] feature: graceful resume for getlogs pollers --- agents/monitor/src/steps/producer.rs | 35 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 38855c36..df61ca24 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -1,4 +1,4 @@ -use ethers::prelude::Middleware; +use ethers::prelude::{Middleware, U64}; use nomad_ethereum::bindings::{ home::{DispatchFilter, Home, UpdateFilter}, replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, @@ -19,6 +19,7 @@ pub const BEHIND_TIP: u64 = 5; pub(crate) struct DispatchProducer { home: Home, network: String, + from: Option, tx: DispatchSink, } @@ -46,6 +47,7 @@ impl DispatchProducer { Self { home, network: network.as_ref().to_owned(), + from: None, tx, } } @@ -55,7 +57,7 @@ pub(crate) type DispatchProducerTask = Restartable; pub(crate) type DispatchProducerHandle = StepHandle; impl ProcessStep for DispatchProducer { - fn spawn(self) -> DispatchProducerTask { + fn spawn(mut self) -> DispatchProducerTask { let span = info_span!( "DispatchProducer", home = format!("{:?}", self.home.address()), @@ -67,7 +69,7 @@ impl ProcessStep for DispatchProducer { async move { let provider = self.home.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - (2 * BEHIND_TIP); + let from = self.from.unwrap_or(height - (2 * BEHIND_TIP)); let mut to = height - BEHIND_TIP; loop { if from < to { @@ -90,7 +92,7 @@ impl ProcessStep for DispatchProducer { bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); let tip = tip_res.unwrap() - BEHIND_TIP; - from = to; + self.from = Some(to); to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; @@ -106,6 +108,7 @@ impl ProcessStep for DispatchProducer { pub(crate) struct UpdateProducer { home: Home, network: String, + from: Option, tx: UnboundedSender>, } @@ -133,6 +136,7 @@ impl UpdateProducer { Self { home, network: network.as_ref().to_owned(), + from: None, tx, } } @@ -142,7 +146,7 @@ pub(crate) type UpdateProducerTask = Restartable; pub(crate) type UpdateProducerHandle = StepHandle; impl ProcessStep for UpdateProducer { - fn spawn(self) -> UpdateProducerTask { + fn spawn(mut self) -> UpdateProducerTask { let span = info_span!( "UpdateProducer", home = format!("{:?}", self.home.address()), @@ -157,8 +161,7 @@ impl ProcessStep for UpdateProducer { bail_task_if!(height.is_err(), self, "Err retrieving height"); let height = height.expect("checked"); - - let mut from = height - (2 * BEHIND_TIP); + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { if from < to { @@ -181,7 +184,7 @@ impl ProcessStep for UpdateProducer { bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); let tip = tip_res.expect("checked") - BEHIND_TIP; - from = to; + self.from = Some(to); to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; @@ -197,6 +200,7 @@ impl ProcessStep for UpdateProducer { pub(crate) struct RelayProducer { replica: Replica, network: String, + from: Option, replica_of: String, tx: UnboundedSender>, } @@ -227,6 +231,7 @@ impl RelayProducer { Self { replica, network: network.as_ref().to_owned(), + from: None, replica_of: replica_of.as_ref().to_owned(), tx, } @@ -237,7 +242,7 @@ pub(crate) type RelayProducerTask = Restartable; pub(crate) type RelayProducerHandle = StepHandle; impl ProcessStep for RelayProducer { - fn spawn(self) -> RelayProducerTask { + fn spawn(mut self) -> RelayProducerTask { let span = info_span!( "RelayProducer", replica = format!("{:?}", self.replica.address()), @@ -250,7 +255,7 @@ impl ProcessStep for RelayProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - (2 * BEHIND_TIP); + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); @@ -273,7 +278,7 @@ impl ProcessStep for RelayProducer { let tip_res = provider.get_block_number().await; bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); let tip = tip_res.unwrap() - BEHIND_TIP; - from = to; + self.from = Some(to); to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; @@ -289,6 +294,7 @@ impl ProcessStep for RelayProducer { pub(crate) struct ProcessProducer { replica: Replica, network: String, + from: Option, replica_of: String, tx: UnboundedSender>, } @@ -315,6 +321,7 @@ impl ProcessProducer { Self { replica, network: network.as_ref().to_owned(), + from: None, replica_of: replica_of.as_ref().to_owned(), tx, } @@ -329,7 +336,7 @@ pub(crate) type ProcessProducerTask = Restartable; pub(crate) type ProcessProducerHandle = StepHandle; impl ProcessStep for ProcessProducer { - fn spawn(self) -> ProcessProducerTask { + fn spawn(mut self) -> ProcessProducerTask { let span = info_span!( "ProcessProducer", replica = format!("{:?}", self.replica.address()), @@ -342,7 +349,7 @@ impl ProcessStep for ProcessProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let mut from = height - (2 * BEHIND_TIP); + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { if from < to { @@ -365,7 +372,7 @@ impl ProcessStep for ProcessProducer { bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); let tip = tip_res.unwrap() - BEHIND_TIP; - from = to; + self.from = Some(to); to = std::cmp::max(to, tip); tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; From 4f9d5feeefbaeed16f80c99deef68213a9bb8a47 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 23 Jul 2022 20:40:59 -0700 Subject: [PATCH 32/80] refactor: dispatch wait uses pipes --- agents/monitor/src/domain.rs | 13 ++---- agents/monitor/src/macros.rs | 6 +-- agents/monitor/src/steps/between.rs | 4 +- agents/monitor/src/steps/dispatch_wait.rs | 51 ++++++----------------- agents/monitor/src/steps/relay_wait.rs | 1 - 5 files changed, 20 insertions(+), 55 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index b63d9dc0..e0027de9 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -228,20 +228,15 @@ impl Domain { ) { let metrics = metrics.dispatch_wait_metrics(&self.network, &self.home_address()); - let (update_sink, update_faucet) = unbounded_channel(); - let (dispatch_sink, dispatch_faucet) = unbounded_channel(); - - let dispatch_faucet = faucets.swap_dispatch(self.name(), dispatch_faucet); - let update_faucet = faucets.swap_update(self.name(), update_faucet); + let dispatch_pipe = faucets.dispatch_pipe(self.name()); + let update_pipe = faucets.update_pipe(self.name()); DispatchWait::new( - dispatch_faucet, - update_faucet, + dispatch_pipe, + update_pipe, self.name().to_owned(), self.home_address(), metrics, - dispatch_sink, - update_sink, ) .spawn(); } diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index aa5c00b6..54f847e7 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -2,7 +2,7 @@ #[macro_export] macro_rules! bail_task_if { ($cond:expr, $self:ident, $err:expr,) => { - bail_task_if!($cond, $self, $err) + $crate::bail_task_if!($cond, $self, $err) }; ($cond:expr, $self:ident, $err:expr) => { if $cond { @@ -19,10 +19,10 @@ macro_rules! unwrap_pipe_item { unwrap_pipe_output!($pipe_output, $self) }}; ($pipe_output:ident, $self:ident) => {{ - bail_task_if!($pipe_output.is_err(), $self, $pipe_output.unwrap_err(),); + $crate::bail_task_if!($pipe_output.is_err(), $self, $pipe_output.unwrap_err(),); let item_opt = $pipe_output.unwrap(); - bail_task_if!(item_opt.is_none(), $self, "inbound pipe failed",); + $crate::bail_task_if!(item_opt.is_none(), $self, "inbound pipe failed",); item_opt.unwrap() }}; diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 9eae7cfa..24f26017 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,8 +1,6 @@ use tracing::{info_span, Instrument}; -use crate::{ - annotate::WithMeta, bail_task_if, pipe::Pipe, unwrap_pipe_item, ProcessStep, Restartable, -}; +use crate::{annotate::WithMeta, pipe::Pipe, unwrap_pipe_item, ProcessStep, Restartable}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 692edb6c..f34d4edd 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -4,7 +4,8 @@ use tokio::select; use tracing::{info_span, Instrument}; use crate::{ - bail_task_if, DispatchFaucet, DispatchSink, ProcessStep, Restartable, UpdateFaucet, UpdateSink, + pipe::{DispatchPipe, UpdatePipe}, + unwrap_pipe_item, ProcessStep, Restartable, }; #[derive(Debug)] @@ -16,8 +17,8 @@ pub(crate) struct DispatchWaitMetrics { #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] pub(crate) struct DispatchWait { - dispatch_faucet: DispatchFaucet, - update_faucet: UpdateFaucet, + dispatch_pipe: DispatchPipe, + update_pipe: UpdatePipe, network: String, emitter: String, @@ -26,9 +27,6 @@ pub(crate) struct DispatchWait { timers: Vec, blocks: Vec, - - dispatch_sink: DispatchSink, - update_sink: UpdateSink, } impl std::fmt::Display for DispatchWait { @@ -43,24 +41,20 @@ impl std::fmt::Display for DispatchWait { impl DispatchWait { pub(crate) fn new( - dispatch_faucet: DispatchFaucet, - update_faucet: UpdateFaucet, + dispatch_pipe: DispatchPipe, + update_pipe: UpdatePipe, network: String, emitter: String, metrics: DispatchWaitMetrics, - dispatch_sink: DispatchSink, - update_sink: UpdateSink, ) -> Self { Self { - dispatch_faucet, - update_faucet, + dispatch_pipe, + update_pipe, network, emitter, metrics, timers: vec![], blocks: vec![], - dispatch_sink, - update_sink, } } @@ -112,35 +106,14 @@ impl ProcessStep for DispatchWait { // first. i.e. ready dispatches will arrive first biased; - dispatch_next = self.dispatch_faucet.recv() => { - bail_task_if!( - dispatch_next.is_none(), - self, - "inbound dispatch broke" - ); - let dispatch = dispatch_next.expect("checked in block"); + dispatch_next = self.dispatch_pipe.next() => { + let dispatch = unwrap_pipe_item!(dispatch_next,self); let block_number = dispatch.meta.block_number; - bail_task_if!( - self.dispatch_sink.send(dispatch).is_err(), - self, - "outbound dispatch broke" - ); self.handle_dispatch(block_number); } - update_opt = self.update_faucet.recv() => { - bail_task_if!( - update_opt.is_none(), - self, - "inbound update broke" - ); - let update = update_opt.expect("checked in block"); + update_next = self.update_pipe.next() => { + let update = unwrap_pipe_item!(update_next, self); let block_number = update.meta.block_number; - - bail_task_if!( - self.update_sink.send(update).is_err(), - self, - "outbound update broke" - ); self.handle_update(block_number); } } diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 39eac126..637463e1 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -7,7 +7,6 @@ use tokio::time::Instant; use tracing::{info_span, Instrument}; use crate::{ - bail_task_if, pipe::{ProcessPipe, RelayPipe}, unwrap_pipe_item, ProcessStep, }; From 25874b5b92e419ae9ec5b5589b314d7bfe57e0e2 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 23 Jul 2022 20:49:59 -0700 Subject: [PATCH 33/80] refactor: remove handles --- agents/monitor/src/domain.rs | 39 ++++++++++++---------------- agents/monitor/src/init.rs | 30 ++++++++------------- agents/monitor/src/main.rs | 5 ---- agents/monitor/src/steps/producer.rs | 9 +------ agents/monitor/src/utils.rs | 39 ---------------------------- 5 files changed, 28 insertions(+), 94 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index e0027de9..7454002b 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -11,14 +11,11 @@ use crate::{ steps::{ between::{BetweenEvents, BetweenMetrics}, dispatch_wait::DispatchWait, - producer::{ - DispatchProducer, DispatchProducerHandle, ProcessProducer, ProcessProducerHandle, - RelayProducer, RelayProducerHandle, UpdateProducer, UpdateProducerHandle, - }, + producer::{DispatchProducer, ProcessProducer, RelayProducer, UpdateProducer}, relay_wait::RelayWait, update_wait::UpdateWait, }, - ProcessStep, Provider, StepHandle, + DispatchFaucet, ProcessFaucet, ProcessStep, Provider, RelayFaucet, UpdateFaucet, }; #[derive(Debug)] @@ -79,34 +76,30 @@ impl Domain { &self.replicas } - pub(crate) fn dispatch_producer(&self) -> DispatchProducerHandle { + pub(crate) fn dispatch_producer(&self) -> DispatchFaucet { let (tx, rx) = unbounded_channel(); - let handle = DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); + DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); - StepHandle { handle, rx } + rx } - pub(crate) fn update_producer(&self) -> UpdateProducerHandle { + pub(crate) fn update_producer(&self) -> UpdateFaucet { let (tx, rx) = unbounded_channel(); - let handle = UpdateProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); + UpdateProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); - StepHandle { handle, rx } + rx } - pub fn relay_producer_for( - &self, - replica: &Replica, - replica_of: &str, - ) -> RelayProducerHandle { + pub fn relay_producer_for(&self, replica: &Replica, replica_of: &str) -> RelayFaucet { let (tx, rx) = unbounded_channel(); - let handle = RelayProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); - StepHandle { handle, rx } + RelayProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + rx } - pub(crate) fn relay_producers(&self) -> HashMap<&str, RelayProducerHandle> { + pub(crate) fn relay_producers(&self) -> HashMap<&str, RelayFaucet> { self.replicas() .iter() .map(|(network, replica)| { @@ -120,14 +113,14 @@ impl Domain { &self, replica: &Replica, replica_of: &str, - ) -> ProcessProducerHandle { + ) -> ProcessFaucet { let (tx, rx) = unbounded_channel(); - let handle = ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); - StepHandle { handle, rx } + ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + rx } - pub(crate) fn process_producers(&self) -> HashMap<&str, ProcessProducerHandle> { + pub(crate) fn process_producers(&self) -> HashMap<&str, ProcessFaucet> { self.replicas() .iter() .map(|(replica_of, replica)| { diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index a41e5df3..6c239e23 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -10,17 +10,8 @@ use tokio::task::JoinHandle; use tracing_subscriber::EnvFilter; use crate::{ - domain::Domain, - faucets::Faucets, - metrics::Metrics, - steps::{ - e2e::E2ELatency, - producer::{ - DispatchProducerHandle, ProcessProducerHandle, RelayProducerHandle, - UpdateProducerHandle, - }, - }, - utils, ArcProvider, HomeReplicaMap, ProcessStep, + domain::Domain, faucets::Faucets, metrics::Metrics, steps::e2e::E2ELatency, ArcProvider, + DispatchFaucet, HomeReplicaMap, ProcessFaucet, ProcessStep, RelayFaucet, UpdateFaucet, }; pub(crate) fn config_from_file() -> Option { @@ -108,39 +99,40 @@ impl Monitor { self.metrics.clone().run_http_server() } - fn run_dispatch_producers(&self) -> HashMap<&str, DispatchProducerHandle> { + fn run_dispatch_producers(&self) -> HashMap<&str, DispatchFaucet> { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.dispatch_producer())) .collect() } - fn run_update_producers(&self) -> HashMap<&str, UpdateProducerHandle> { + fn run_update_producers(&self) -> HashMap<&str, UpdateFaucet> { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.update_producer())) .collect() } - fn run_relay_producers(&self) -> HomeReplicaMap { + fn run_relay_producers(&self) -> HomeReplicaMap { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.relay_producers())) .collect() } - fn run_process_producers(&self) -> HomeReplicaMap { + fn run_process_producers(&self) -> HomeReplicaMap { self.networks .iter() .map(|(network, domain)| (network.as_str(), domain.process_producers())) .collect() } + pub(crate) fn producers(&self) -> Faucets { Faucets { - dispatches: utils::split(self.run_dispatch_producers()).1, - updates: utils::split(self.run_update_producers()).1, - relays: utils::nested_split(self.run_relay_producers()).1, - processes: utils::nested_split(self.run_process_producers()).1, + dispatches: self.run_dispatch_producers(), + updates: self.run_update_producers(), + relays: self.run_relay_producers(), + processes: self.run_process_producers(), } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index b2343824..2ee79605 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -104,8 +104,3 @@ pub(crate) trait ProcessStep: std::fmt::Display { }) } } - -pub(crate) struct StepHandle { - handle: Restartable, - rx: Output, -} diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index df61ca24..24a21fb7 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -6,10 +6,7 @@ use nomad_ethereum::bindings::{ use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, Instrument}; -use crate::{ - annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessStep, - RelayFaucet, Restartable, StepHandle, UpdateFaucet, -}; +use crate::{annotate::WithMeta, bail_task_if, DispatchSink, ProcessStep, Restartable}; pub const POLLING_INTERVAL_SECS: u64 = 5; pub const BEHIND_TIP: u64 = 5; @@ -54,7 +51,6 @@ impl DispatchProducer { } pub(crate) type DispatchProducerTask = Restartable; -pub(crate) type DispatchProducerHandle = StepHandle; impl ProcessStep for DispatchProducer { fn spawn(mut self) -> DispatchProducerTask { @@ -143,7 +139,6 @@ impl UpdateProducer { } pub(crate) type UpdateProducerTask = Restartable; -pub(crate) type UpdateProducerHandle = StepHandle; impl ProcessStep for UpdateProducer { fn spawn(mut self) -> UpdateProducerTask { @@ -239,7 +234,6 @@ impl RelayProducer { } pub(crate) type RelayProducerTask = Restartable; -pub(crate) type RelayProducerHandle = StepHandle; impl ProcessStep for RelayProducer { fn spawn(mut self) -> RelayProducerTask { @@ -333,7 +327,6 @@ impl ProcessProducer { } pub(crate) type ProcessProducerTask = Restartable; -pub(crate) type ProcessProducerHandle = StepHandle; impl ProcessStep for ProcessProducer { fn spawn(mut self) -> ProcessProducerTask { diff --git a/agents/monitor/src/utils.rs b/agents/monitor/src/utils.rs index bdcc4cb6..75360e99 100644 --- a/agents/monitor/src/utils.rs +++ b/agents/monitor/src/utils.rs @@ -2,45 +2,6 @@ use std::{collections::HashMap, pin::Pin}; use tokio::sync::mpsc::UnboundedReceiver; -use crate::{HomeReplicaMap, ProcessStep, Restartable, StepHandle}; - -// split handles from outputs -pub(crate) fn split( - map: HashMap<&str, StepHandle>, -) -> (Vec>, HashMap<&str, U>) -where - T: ProcessStep, -{ - let mut handles = vec![]; - let map = map - .into_iter() - .map(|(name, out)| { - handles.push(out.handle); - (name, out.rx) - }) - .collect(); - (handles, map) -} - -// split handles from outputs in a nested map -pub(crate) fn nested_split( - map: HomeReplicaMap>, -) -> (Vec>, HomeReplicaMap) -where - T: ProcessStep, -{ - let mut handles = vec![]; - let map = map - .into_iter() - .map(|(name, map)| { - let (mut h, m) = split(map); - handles.append(&mut h); - (name, m) - }) - .collect(); - (handles, map) -} - // polls all channels in a hashmap pub(crate) fn nexts( map: &mut HashMap>, From 94a9a963fb43676d2f77e5369040f7f8113088cc Mon Sep 17 00:00:00 2001 From: James Date: Sat, 23 Jul 2022 21:34:38 -0700 Subject: [PATCH 34/80] refactor: change spawns to run_until_panic --- agents/monitor/src/domain.rs | 24 +++++++++++------------ agents/monitor/src/init.rs | 2 +- agents/monitor/src/main.rs | 4 ++-- agents/monitor/src/steps/between.rs | 2 +- agents/monitor/src/steps/combine.rs | 4 ++-- agents/monitor/src/steps/dispatch_wait.rs | 10 +++++----- agents/monitor/src/steps/e2e.rs | 6 +++--- agents/monitor/src/steps/producer.rs | 8 ++++---- agents/monitor/src/steps/relay_wait.rs | 2 +- agents/monitor/src/steps/terminal.rs | 2 +- agents/monitor/src/steps/update_wait.rs | 4 ++-- 11 files changed, 34 insertions(+), 34 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 7454002b..ab779acb 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -79,7 +79,7 @@ impl Domain { pub(crate) fn dispatch_producer(&self) -> DispatchFaucet { let (tx, rx) = unbounded_channel(); - DispatchProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); + DispatchProducer::new(self.home.clone(), &self.network, tx).run_until_panic(); rx } @@ -87,7 +87,7 @@ impl Domain { pub(crate) fn update_producer(&self) -> UpdateFaucet { let (tx, rx) = unbounded_channel(); - UpdateProducer::new(self.home.clone(), self.network.clone(), tx).spawn(); + UpdateProducer::new(self.home.clone(), &self.network, tx).run_until_panic(); rx } @@ -95,7 +95,7 @@ impl Domain { pub fn relay_producer_for(&self, replica: &Replica, replica_of: &str) -> RelayFaucet { let (tx, rx) = unbounded_channel(); - RelayProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + RelayProducer::new(replica.clone(), &self.network, replica_of, tx).run_until_panic(); rx } @@ -116,7 +116,7 @@ impl Domain { ) -> ProcessFaucet { let (tx, rx) = unbounded_channel(); - ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).spawn(); + ProcessProducer::new(replica.clone(), &self.network, replica_of, tx).run_until_panic(); rx } @@ -152,7 +152,7 @@ impl Domain { event, emitter, ) - .spawn(); + .run_until_panic(); } pub(crate) fn count_updates<'a>( @@ -171,7 +171,7 @@ impl Domain { event = event.as_ref(), "starting counter", ); - BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).run_until_panic(); } pub(crate) fn count_relays<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { @@ -190,7 +190,7 @@ impl Domain { let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).run_until_panic(); }); } @@ -210,7 +210,7 @@ impl Domain { let metrics = metrics.between_metrics(network, event, &emitter, Some(replica_of)); - BetweenEvents::new(pipe, metrics, network, event, emitter).spawn(); + BetweenEvents::new(pipe, metrics, network, event, emitter).run_until_panic(); }); } @@ -227,11 +227,11 @@ impl Domain { DispatchWait::new( dispatch_pipe, update_pipe, - self.name().to_owned(), + self.name(), self.home_address(), metrics, ) - .spawn(); + .run_until_panic(); } pub(crate) fn update_to_relay<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { @@ -273,7 +273,7 @@ impl Domain { update_sink, relay_sinks, ) - .spawn(); + .run_until_panic(); } pub(crate) fn relay_to_process<'a>(&'a self, faucets: &mut Faucets<'a>, metrics: Arc) { @@ -293,7 +293,7 @@ impl Domain { emitter, metrics, ) - .spawn(); + .run_until_panic(); }); } } diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 6c239e23..c001c30f 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -222,6 +222,6 @@ impl Monitor { dispatch_sinks, process_sinks, ) - .spawn(); + .run_until_panic(); } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 2ee79605..2fa7c942 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -85,7 +85,7 @@ pub(crate) trait ProcessStep: std::fmt::Display { Self: 'static + Send + Sync + Sized, { tokio::spawn(async move { - let mut handle = self.spawn(); + let mut handle = self.run_until_panic(); loop { let result = handle.await; @@ -99,7 +99,7 @@ pub(crate) trait ProcessStep: std::fmt::Display { panic!("JoinError in forever. Internal task panicked"); } }; - handle = again.spawn() + handle = again.run_until_panic() } }) } diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 24f26017..9ac40249 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -9,7 +9,7 @@ pub(crate) struct BetweenMetrics { } // Track time between events of the same kind -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct BetweenEvents { pub(crate) pipe: Pipe, pub(crate) metrics: BetweenMetrics, diff --git a/agents/monitor/src/steps/combine.rs b/agents/monitor/src/steps/combine.rs index 48a938a9..2854c103 100644 --- a/agents/monitor/src/steps/combine.rs +++ b/agents/monitor/src/steps/combine.rs @@ -6,7 +6,7 @@ use tracing::{info_span, Instrument}; use crate::{bail_task_if, utils::nexts, ProcessStep, Restartable}; #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct CombineChannels { pub(crate) faucets: HashMap>, pub(crate) sink: UnboundedSender<(String, T)>, @@ -31,7 +31,7 @@ impl CombineChannels { .into_iter() .map(|(k, v)| { let (sink, faucet) = unbounded_channel(); - CombineChannels::::new(v, sink).spawn(); + CombineChannels::::new(v, sink).run_until_panic(); (k, faucet) }) .collect(); diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index f34d4edd..c1e242ad 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -15,7 +15,7 @@ pub(crate) struct DispatchWaitMetrics { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct DispatchWait { dispatch_pipe: DispatchPipe, update_pipe: UpdatePipe, @@ -43,15 +43,15 @@ impl DispatchWait { pub(crate) fn new( dispatch_pipe: DispatchPipe, update_pipe: UpdatePipe, - network: String, - emitter: String, + network: impl AsRef, + emitter: impl AsRef, metrics: DispatchWaitMetrics, ) -> Self { Self { dispatch_pipe, update_pipe, - network, - emitter, + network: network.as_ref().to_owned(), + emitter: emitter.as_ref().to_owned(), metrics, timers: vec![], blocks: vec![], diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 8044c6f5..e32a4384 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -22,7 +22,7 @@ pub(crate) struct E2EMetrics { pub(crate) timers: HashMap, } -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub struct E2ELatency { dispatch_faucet: UnboundedReceiver<(String, WithMeta)>, process_faucet: UnboundedReceiver<(String, (String, WithMeta))>, @@ -58,8 +58,8 @@ impl E2ELatency { let (process_sink, process_faucet) = unbounded_channel(); let (dispatch_sink, dispatch_faucet) = unbounded_channel(); - CombineChannels::new(dispatch_faucets, dispatch_sink).spawn(); - CombineChannels::nested(process_faucets, process_sink).spawn(); + CombineChannels::new(dispatch_faucets, dispatch_sink).run_until_panic(); + CombineChannels::nested(process_faucets, process_sink).run_until_panic(); Self { dispatch_faucet, diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 24a21fb7..a916f4cd 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -12,7 +12,7 @@ pub const POLLING_INTERVAL_SECS: u64 = 5; pub const BEHIND_TIP: u64 = 5; #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct DispatchProducer { home: Home, network: String, @@ -100,7 +100,7 @@ impl ProcessStep for DispatchProducer { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct UpdateProducer { home: Home, network: String, @@ -191,7 +191,7 @@ impl ProcessStep for UpdateProducer { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct RelayProducer { replica: Replica, network: String, @@ -284,7 +284,7 @@ impl ProcessStep for RelayProducer { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct ProcessProducer { replica: Replica, network: String, diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 637463e1..deeb4198 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -18,7 +18,7 @@ pub(crate) struct RelayWaitMetrics { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct RelayWait { relay_pipe: RelayPipe, process_pipe: ProcessPipe, diff --git a/agents/monitor/src/steps/terminal.rs b/agents/monitor/src/steps/terminal.rs index 1a48359a..bd70b64d 100644 --- a/agents/monitor/src/steps/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -4,7 +4,7 @@ use tracing::{debug_span, Instrument}; use crate::{ProcessStep, Restartable}; #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] /// A process step that just drains its input and drops everything /// Its [`StepHandle`] will never produce values. pub(crate) struct Terminal diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 519d35d7..43e21dd8 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -17,7 +17,7 @@ pub(crate) struct UpdateWaitMetrics { } #[derive(Debug)] -#[must_use = "Tasks do nothing unless you call .spawn() or .forever()"] +#[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct UpdateWait { update_faucet: UpdateFaucet, relay_faucets: UnboundedReceiver<(String, WithMeta)>, @@ -43,7 +43,7 @@ impl UpdateWait { ) -> Self { let (tx, rx) = unbounded_channel(); - CombineChannels::new(relay_faucets, tx).spawn(); + CombineChannels::new(relay_faucets, tx).run_until_panic(); Self { update_faucet, From 509a066aba46f58a5359464a0b62466f27159962 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 23 Jul 2022 21:36:16 -0700 Subject: [PATCH 35/80] fix: run_until_panic no longer invokes itself --- agents/monitor/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 2fa7c942..74bc3cce 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -80,12 +80,12 @@ pub(crate) trait ProcessStep: std::fmt::Display { /// Run the task until it panics. Errors result in a task restart with the /// same channels. This means that an error causes the task to lose only /// the data that is in-scope when it faults. - fn run_until_panic(self) -> Restartable + fn run_until_panic(self) -> Restartable<()> where Self: 'static + Send + Sync + Sized, { tokio::spawn(async move { - let mut handle = self.run_until_panic(); + let mut handle = self.spawn(); loop { let result = handle.await; @@ -99,7 +99,7 @@ pub(crate) trait ProcessStep: std::fmt::Display { panic!("JoinError in forever. Internal task panicked"); } }; - handle = again.run_until_panic() + handle = again.spawn(); } }) } From fe6ead2e83d3b07da7f95e9d8cc9eb9b02dfc59a Mon Sep 17 00:00:00 2001 From: James Date: Sun, 24 Jul 2022 09:13:48 -0700 Subject: [PATCH 36/80] fix: improved logging in run_until_panic --- agents/monitor/src/main.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 74bc3cce..a95bb29f 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -84,21 +84,24 @@ pub(crate) trait ProcessStep: std::fmt::Display { where Self: 'static + Send + Sync + Sized, { + let task_description = format!("{}", self); tokio::spawn(async move { let mut handle = self.spawn(); loop { let result = handle.await; - let again = match result { - Ok((handle, report)) => { - tracing::warn!(error = %report, "Restarting task"); - handle - } + let (again, report) = match result { + Ok((handle, report)) => (handle, report), Err(e) => { - tracing::error!(err = %e, "JoinError in forever. Internal task panicked"); + tracing::error!(err = %e, task = task_description.as_str(), "Internal task panicked"); panic!("JoinError in forever. Internal task panicked"); } }; + tracing::warn!( + error = %report, + task = task_description.as_str(), + "Restarting task", + ); handle = again.spawn(); } }) From c14a4013dad08394620de6a933634f55002a27d5 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 24 Jul 2022 09:55:56 -0700 Subject: [PATCH 37/80] fix: terminate all channels to prevent memory leak --- agents/monitor/src/init.rs | 35 ++++++++++++++++++++++++++-- agents/monitor/src/main.rs | 14 +++++++---- agents/monitor/src/steps/terminal.rs | 11 ++++++++- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index c001c30f..b01e9ba0 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -10,8 +10,12 @@ use tokio::task::JoinHandle; use tracing_subscriber::EnvFilter; use crate::{ - domain::Domain, faucets::Faucets, metrics::Metrics, steps::e2e::E2ELatency, ArcProvider, - DispatchFaucet, HomeReplicaMap, ProcessFaucet, ProcessStep, RelayFaucet, UpdateFaucet, + domain::Domain, + faucets::Faucets, + metrics::Metrics, + steps::{e2e::E2ELatency, terminal::Terminal}, + ArcProvider, DispatchFaucet, HomeReplicaMap, ProcessFaucet, ProcessStep, RelayFaucet, + UpdateFaucet, }; pub(crate) fn config_from_file() -> Option { @@ -224,4 +228,31 @@ impl Monitor { ) .run_until_panic(); } + + /// take ownership of all faucets and terminate them + pub(crate) fn run_terminals<'a>(&'a self, faucets: Faucets<'a>) -> Vec> { + let mut tasks = vec![]; + + faucets.dispatches.into_iter().for_each(|(_, v)| { + tasks.push(Terminal::new(v).run_until_panic()); + }); + + faucets.updates.into_iter().for_each(|(_, v)| { + tasks.push(Terminal::new(v).run_until_panic()); + }); + + faucets.relays.into_iter().for_each(|(_, v)| { + v.into_iter().for_each(|(_, v)| { + tasks.push(Terminal::new(v).run_until_panic()); + }); + }); + + faucets.processes.into_iter().for_each(|(_, v)| { + v.into_iter().for_each(|(_, v)| { + tasks.push(Terminal::new(v).run_until_panic()); + }); + }); + + tasks + } } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index a95bb29f..2f676bab 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,4 +1,5 @@ use annotate::WithMeta; +use futures_util::future::select_all; use nomad_ethereum::bindings::{ home::{DispatchFilter, UpdateFilter}, replica::{ProcessFilter, UpdateFilter as RelayFilter}, @@ -63,13 +64,16 @@ async fn main() -> eyre::Result<()> { monitor.run_relay_to_process(&mut faucets); monitor.run_e2e(&mut faucets); + // sink em + let tasks = monitor.run_terminals(faucets); + tracing::info!("tasks started"); - // just run forever - loop { - tokio::time::sleep(std::time::Duration::from_secs(10000)).await - } + // run until there's a failure of a terminal + // this would imply there is a series of upstream channel failures + let (_, _, _) = select_all(tasks).await; } + Ok(()) } pub(crate) trait ProcessStep: std::fmt::Display { @@ -80,7 +84,7 @@ pub(crate) trait ProcessStep: std::fmt::Display { /// Run the task until it panics. Errors result in a task restart with the /// same channels. This means that an error causes the task to lose only /// the data that is in-scope when it faults. - fn run_until_panic(self) -> Restartable<()> + fn run_until_panic(self) -> JoinHandle<()> where Self: 'static + Send + Sync + Sized, { diff --git a/agents/monitor/src/steps/terminal.rs b/agents/monitor/src/steps/terminal.rs index bd70b64d..04da3147 100644 --- a/agents/monitor/src/steps/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -14,6 +14,15 @@ where pub(crate) rx: UnboundedReceiver, } +impl Terminal +where + T: std::fmt::Debug, +{ + pub(crate) fn new(rx: UnboundedReceiver) -> Self { + Self { rx } + } +} + impl std::fmt::Display for Terminal where T: std::fmt::Debug, @@ -35,7 +44,7 @@ where async move { loop { if self.rx.recv().await.is_none() { - tracing::info!("Upstream broke, shutting down"); + tracing::debug!(self = %self, "Upstream broke, shutting down"); return (self, eyre::eyre!("")); } } From 1279ca1fbcd0d250158199c50a115bdb092245f2 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 25 Jul 2022 09:52:44 -0700 Subject: [PATCH 38/80] feature: default to info logging --- agents/monitor/src/init.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index b01e9ba0..8a64fd02 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -7,6 +7,7 @@ use ethers::{ use nomad_xyz_configuration::{get_builtin, NomadConfig}; use tokio::task::JoinHandle; +use tracing::Level; use tracing_subscriber::EnvFilter; use crate::{ @@ -40,6 +41,7 @@ pub(crate) fn config() -> eyre::Result { pub(crate) fn init_tracing() { tracing_subscriber::FmtSubscriber::builder() .pretty() + .with_max_level(Level::INFO) .with_env_filter(EnvFilter::from_default_env()) .with_level(true) .init(); From b1b185eefcdd98cf139b4fd86e94e9a401d07eb9 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 00:34:43 -0700 Subject: [PATCH 39/80] refactor: make namespace a module constant in metrics --- agents/monitor/src/metrics.rs | 20 +++++++++++--------- nomad-base/src/metrics.rs | 22 ++++++++++++---------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index fb69b3ce..ee072aa7 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -4,6 +4,8 @@ use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, In use tokio::task::JoinHandle; use warp::Filter; +const NAMESPACE: &str = "nomad_monitor"; + use crate::steps::{ between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, e2e::E2EMetrics, relay_wait::RelayWaitMetrics, update_wait::UpdateWaitMetrics, @@ -45,7 +47,7 @@ impl Metrics { "e2e_ms", "Ms between dispatch and associated process, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain"], )?; @@ -55,7 +57,7 @@ impl Metrics { "update_to_relay_ms", "Ms between update and relay, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -65,7 +67,7 @@ impl Metrics { "dispatch_to_update_ms", "Ms between dispatch and update, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -75,7 +77,7 @@ impl Metrics { "dispatch_to_update_blocks", "Blocks between dispatch and update, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -85,7 +87,7 @@ impl Metrics { "relay_to_process_ms", "Ms between relay and process, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -95,7 +97,7 @@ impl Metrics { "relay_to_process_blocks", "Blocks between dispatch and update, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -105,7 +107,7 @@ impl Metrics { "inter_event_period_wallclock_ms", "Ms between events periods, as observed by this agent", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; @@ -115,7 +117,7 @@ impl Metrics { "inter_event_blocks", "Blocks between events, as marked by the chain (i.e. 0 means same block, 1 means next block, etc)", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; @@ -125,7 +127,7 @@ impl Metrics { "event_counts", "Counts of each event, labeled by name and chain", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; diff --git a/nomad-base/src/metrics.rs b/nomad-base/src/metrics.rs index d6c7c97b..773d699e 100644 --- a/nomad-base/src/metrics.rs +++ b/nomad-base/src/metrics.rs @@ -7,6 +7,8 @@ use prometheus::{ use std::sync::Arc; use tokio::task::JoinHandle; +const NAMESPACE: &str = "nomad"; + fn u16_from_env(s: impl AsRef) -> Option { std::env::var(s.as_ref()).ok().and_then(|i| i.parse().ok()) } @@ -44,7 +46,7 @@ impl CoreMetrics { "transactions_total", "Number of transactions sent by this agent since boot", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "wallet", "agent"], )?), @@ -53,7 +55,7 @@ impl CoreMetrics { "wallet_balance_total", "Balance of the smart contract wallet", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "wallet", "agent"], )?), @@ -62,7 +64,7 @@ impl CoreMetrics { "channel_faults", "Number of per home <> replica channel faults (errors)", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["home", "replica", "agent"], )?), @@ -71,7 +73,7 @@ impl CoreMetrics { "rpc_duration_ms", "Duration from dispatch to receipt-of-response for RPC calls", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "method", "agent"], )?), @@ -80,7 +82,7 @@ impl CoreMetrics { "span_duration_sec", "Duration from span creation to span destruction", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["span_name", "target"], )?), @@ -89,7 +91,7 @@ impl CoreMetrics { "home_failure_checks", "Number of times agent has checked home for failed state", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["home", "agent"] )?), @@ -98,7 +100,7 @@ impl CoreMetrics { "home_failure_observations", "Number of times agent has seen the home failed (anything > 0 is major red flag!)", ) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["home", "agent"] )?), @@ -132,7 +134,7 @@ impl CoreMetrics { ) -> Result { let gauge_vec = IntGaugeVec::new( Opts::new(metric_name, help) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), labels, )?; @@ -150,7 +152,7 @@ impl CoreMetrics { ) -> Result { let counter = IntCounterVec::new( Opts::new(metric_name, help) - .namespace("nomad") + .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), labels, )?; @@ -170,7 +172,7 @@ impl CoreMetrics { ) -> Result { let histogram = HistogramVec::new( HistogramOpts::new(metric_name, help) - .namespace("nomad") + .namespace(NAMESPACE) .buckets(buckets.to_owned()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), labels, From c91e5ce064e578e6b6282ece8a8e97a0f66fb384 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 00:41:27 -0700 Subject: [PATCH 40/80] fix: reasonable-ish buckets for histograms --- agents/monitor/src/metrics.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index ee072aa7..233bbdf0 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -5,6 +5,18 @@ use tokio::task::JoinHandle; use warp::Filter; const NAMESPACE: &str = "nomad_monitor"; +// 1 sec, 5 secs, 30 sec, 1 min, 5 min, 30 min, 1 hour +const TIME_BUCKETS: &[f64] = &[ + 1_000.0, + 5_000.0, + 30_000.0, + 60_000.0, + 120_000.0, + 600_000.0, + 3_600_000.0, + 7_200_000.0, +]; +const BLOCKS_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 200.0, 500.0, 1000.0]; use crate::steps::{ between::BetweenMetrics, dispatch_wait::DispatchWaitMetrics, e2e::E2EMetrics, @@ -48,6 +60,7 @@ impl Metrics { "Ms between dispatch and associated process, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain"], )?; @@ -58,6 +71,7 @@ impl Metrics { "Ms between update and relay, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -68,6 +82,7 @@ impl Metrics { "Ms between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -78,6 +93,7 @@ impl Metrics { "Blocks between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(BLOCKS_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -88,6 +104,7 @@ impl Metrics { "Ms between relay and process, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -98,6 +115,7 @@ impl Metrics { "Blocks between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(BLOCKS_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -108,6 +126,7 @@ impl Metrics { "Ms between events periods, as observed by this agent", ) .namespace(NAMESPACE) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; @@ -118,6 +137,7 @@ impl Metrics { "Blocks between events, as marked by the chain (i.e. 0 means same block, 1 means next block, etc)", ) .namespace(NAMESPACE) + .buckets(BLOCKS_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; From 2806282329609131b3686385ee96ba5b4acbdfe0 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 00:49:33 -0700 Subject: [PATCH 41/80] chore: differentiate e2e time buckets --- agents/monitor/src/metrics.rs | 43 +++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 233bbdf0..8c1e37e5 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -5,17 +5,30 @@ use tokio::task::JoinHandle; use warp::Filter; const NAMESPACE: &str = "nomad_monitor"; -// 1 sec, 5 secs, 30 sec, 1 min, 5 min, 30 min, 1 hour -const TIME_BUCKETS: &[f64] = &[ - 1_000.0, - 5_000.0, - 30_000.0, - 60_000.0, - 120_000.0, - 600_000.0, - 3_600_000.0, - 7_200_000.0, + +const LOCAL_TIME_BUCKETS: &[f64] = &[ + 1_000.0, // 1 sec + 5_000.0, // 5 secs + 30_000.0, // 30 secs + 60_000.0, // 1 minu + 120_000.0, // 2 min + 600_000.0, // 10 min + 1_800_000.0, // 30 min + 3_600_000.0, // 1 hour + 7_200_000.0, // 2 hour +]; +// time buckets for e2e metric +const E2E_TIME_BUCKETS: &[f64] = &[ + 2_100_000.0, // 35 minutes + 2_400_000.0, // 40 minutes + 2_700_000.0, // 45 minutes + 3_000_000.0, // 50 minutes + 3_300_000.0, // 55 minutes + 3_600_000.0, // 1 hour + 7_200_000.0, // 2 hours + 10_800_000.0, // 3 hours ]; + const BLOCKS_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 200.0, 500.0, 1000.0]; use crate::steps::{ @@ -60,7 +73,7 @@ impl Metrics { "Ms between dispatch and associated process, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(TIME_BUCKETS.to_vec()) + .buckets(E2E_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain"], )?; @@ -71,7 +84,7 @@ impl Metrics { "Ms between update and relay, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(TIME_BUCKETS.to_vec()) + .buckets(LOCAL_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -82,7 +95,7 @@ impl Metrics { "Ms between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(TIME_BUCKETS.to_vec()) + .buckets(LOCAL_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -104,7 +117,7 @@ impl Metrics { "Ms between relay and process, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(TIME_BUCKETS.to_vec()) + .buckets(LOCAL_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -126,7 +139,7 @@ impl Metrics { "Ms between events periods, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(TIME_BUCKETS.to_vec()) + .buckets(LOCAL_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; From eb4d9c23ed20b92d093d6afdeb91e316e951b636 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 00:50:58 -0700 Subject: [PATCH 42/80] chore: undo rename of const var --- agents/monitor/src/metrics.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 8c1e37e5..5e1bbf69 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -6,7 +6,7 @@ use warp::Filter; const NAMESPACE: &str = "nomad_monitor"; -const LOCAL_TIME_BUCKETS: &[f64] = &[ +const TIME_BUCKETS: &[f64] = &[ 1_000.0, // 1 sec 5_000.0, // 5 secs 30_000.0, // 30 secs @@ -84,7 +84,7 @@ impl Metrics { "Ms between update and relay, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(LOCAL_TIME_BUCKETS.to_vec()) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -95,7 +95,7 @@ impl Metrics { "Ms between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(LOCAL_TIME_BUCKETS.to_vec()) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "emitter"], )?; @@ -117,7 +117,7 @@ impl Metrics { "Ms between relay and process, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(LOCAL_TIME_BUCKETS.to_vec()) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "replica_of", "emitter"], )?; @@ -139,7 +139,7 @@ impl Metrics { "Ms between events periods, as observed by this agent", ) .namespace(NAMESPACE) - .buckets(LOCAL_TIME_BUCKETS.to_vec()) + .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), &["chain", "event", "emitter", "replica_of"], )?; From 10f8de43ee8294b78b30f1bbc07ba65f5c3e69a6 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 00:59:27 -0700 Subject: [PATCH 43/80] fix: agent http servers now respond to all reqs with 301 to metrics endpoints --- agents/monitor/src/metrics.rs | 20 ++++++++++---------- nomad-base/src/metrics.rs | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 5e1bbf69..a92932d2 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -10,7 +10,7 @@ const TIME_BUCKETS: &[f64] = &[ 1_000.0, // 1 sec 5_000.0, // 5 secs 30_000.0, // 30 secs - 60_000.0, // 1 minu + 60_000.0, // 1 min 120_000.0, // 2 min 600_000.0, // 10 min 1_800_000.0, // 30 min @@ -19,11 +19,11 @@ const TIME_BUCKETS: &[f64] = &[ ]; // time buckets for e2e metric const E2E_TIME_BUCKETS: &[f64] = &[ - 2_100_000.0, // 35 minutes - 2_400_000.0, // 40 minutes - 2_700_000.0, // 45 minutes - 3_000_000.0, // 50 minutes - 3_300_000.0, // 55 minutes + 2_100_000.0, // 35 min + 2_400_000.0, // 40 min + 2_700_000.0, // 45 min + 3_000_000.0, // 50 min + 3_300_000.0, // 55 min 3_600_000.0, // 1 hour 7_200_000.0, // 2 hours 10_800_000.0, // 3 hours @@ -227,10 +227,10 @@ impl Metrics { ) }) .or(warp::any().map(|| { - warp::reply::with_status( - "go look at /metrics", - warp::http::StatusCode::NOT_FOUND, - ) + warp::http::Response::builder() + .header("Location", "/metrics") + .status(301) + .body("".to_string()) })), ) .run(([0, 0, 0, 0], port)) diff --git a/nomad-base/src/metrics.rs b/nomad-base/src/metrics.rs index 773d699e..3d96c7f3 100644 --- a/nomad-base/src/metrics.rs +++ b/nomad-base/src/metrics.rs @@ -266,10 +266,10 @@ impl CoreMetrics { ) }) .or(warp::any().map(|| { - warp::reply::with_status( - "go look at /metrics", - warp::http::StatusCode::NOT_FOUND, - ) + warp::http::Response::builder() + .header("Location", "/metrics") + .status(301) + .body("".to_string()) })), ) .run(([0, 0, 0, 0], port)) From aed13a7b833e93389455eea892f67ced56e746ff Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 01:14:59 -0700 Subject: [PATCH 44/80] feature: replica label on update-to-relay timers --- agents/monitor/src/domain.rs | 15 ++++++++++--- agents/monitor/src/metrics.rs | 27 +++++++++++++++++------ agents/monitor/src/steps/update_wait.rs | 29 +++++++++++++++---------- 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index ab779acb..32d744b2 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -240,11 +240,18 @@ impl Domain { // we want to go through each network that is NOT this network // get the relay sink FOR THIS NETWORK'S REPLICA + let other_nets: Vec<&str> = faucets + .relays + .keys() + // does not match this network + .filter(|k| **k != self.network) + .copied() + .collect(); + faucets .relays .iter_mut() - // does not match this network - .filter(|(k, _)| **k != self.network) + .filter(|(k, _)| other_nets.contains(k)) .for_each(|(k, v)| { // create a new channel let (sink, faucet) = unbounded_channel(); @@ -265,11 +272,13 @@ impl Domain { let update_faucet = faucets.swap_update(self.name(), update_faucet); + let metrics = metrics.update_wait_metrics(self.name(), &other_nets, &self.home_address()); + UpdateWait::new( update_faucet, relay_faucets, self.name(), - metrics.update_wait_metrics(self.name(), &self.home_address()), + metrics, update_sink, relay_sinks, ) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index a92932d2..baa46e71 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -86,7 +86,7 @@ impl Metrics { .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), - &["chain", "emitter"], + &["chain", "emitter", "replica_chain"], )?; let dispatch_to_update_timers = HistogramVec::new( @@ -304,12 +304,25 @@ impl Metrics { } } - pub(crate) fn update_wait_metrics(&self, network: &str, emitter: &str) -> UpdateWaitMetrics { - UpdateWaitMetrics { - times: self - .update_to_relay_timers - .with_label_values(&[network, emitter]), - } + pub(crate) fn update_wait_metrics( + &self, + home_network: &str, + replica_networks: &[&str], + emitter: &str, + ) -> UpdateWaitMetrics { + let times = replica_networks + .iter() + .map(|replica_network| { + let timer = self.update_to_relay_timers.with_label_values(&[ + home_network, + emitter, + replica_network, + ]); + (replica_network.to_string(), timer) + }) + .collect(); + + UpdateWaitMetrics { times } } pub(crate) fn relay_wait_metrics( diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 43e21dd8..ec72d8d0 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -13,7 +13,7 @@ use crate::{ #[derive(Debug)] pub(crate) struct UpdateWaitMetrics { - pub(crate) times: Histogram, + pub(crate) times: HashMap, } #[derive(Debug)] @@ -26,6 +26,7 @@ pub(crate) struct UpdateWait { metrics: UpdateWaitMetrics, updates: HashMap, + // maps root -> replica -> timer relays: HashMap>, update_sink: UpdateSink, @@ -65,18 +66,18 @@ impl std::fmt::Display for UpdateWait { } impl UpdateWait { - fn handle_relay(&mut self, net: &str, root: H256) { + fn handle_relay(&mut self, replica_network: &str, root: H256) { let now = std::time::Instant::now(); // mem optimization: don't need to store the relay time // if we observe immediately if let Some(update_time) = self.updates.get(&root) { - self.record(now, *update_time) + self.record(replica_network, now, *update_time) } else { self.relays .entry(root) .or_default() - .insert(net.to_owned(), now); + .insert(replica_network.to_owned(), now); } } @@ -88,15 +89,19 @@ impl UpdateWait { // any future relay times for this root if let Some(mut relays) = self.relays.remove(&root) { // times - relays.drain().for_each(|(_, relay)| { - self.record(relay, now); + relays.drain().for_each(|(replica_network, relay)| { + self.record(&replica_network, relay, now); }) } } - fn record(&self, relay: Instant, update: Instant) { + fn record(&self, replica_network: &str, relay: Instant, update: Instant) { let v = relay.saturating_duration_since(update); - self.metrics.times.observe(v.as_millis() as f64) + self.metrics + .times + .get(replica_network) + .expect("missing metric") + .observe(v.as_millis() as f64) } } @@ -141,21 +146,21 @@ impl ProcessStep for UpdateWait { self, format!("Inbound relays broke"), ); - let (net, relay) = relay_opt.unwrap(); + let (replica_network, relay) = relay_opt.unwrap(); let root: H256 = relay.log.new_root.into(); bail_task_if!( // send onward self.relay_sinks - .get(&net) + .get(&replica_network) .expect("missing outgoing") .send(relay) .is_err(), self, - format!("outgoing relay for {} broke", &net) + format!("outgoing relay for {} broke", &replica_network) ); - self.handle_relay(&net, root); + self.handle_relay(&replica_network, root); } } } From dd5ff64d6c178d921cc438143cb47952bc111bf1 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 01:22:27 -0700 Subject: [PATCH 45/80] feature: finish function on pipe --- agents/monitor/src/pipe.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/agents/monitor/src/pipe.rs b/agents/monitor/src/pipe.rs index 662f325d..7c47fd9d 100644 --- a/agents/monitor/src/pipe.rs +++ b/agents/monitor/src/pipe.rs @@ -32,11 +32,15 @@ where self.contents.as_ref() } - /// Get the - pub(crate) async fn next(&mut self) -> eyre::Result> { + pub(crate) fn finish(&mut self) -> eyre::Result<()> { if let Some(contents) = self.contents.take() { self.tx.send(contents)?; } + Ok(()) + } + + pub(crate) async fn next(&mut self) -> eyre::Result> { + self.finish()?; self.contents = self.rx.recv().await; if self.contents.is_none() { From 7b930e0db0e80a7537fd99373f63dee90711aaf8 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 01:23:23 -0700 Subject: [PATCH 46/80] fix: standardize on f64 seconds, not millis --- agents/monitor/src/metrics.rs | 54 ++++++++++++------------- agents/monitor/src/steps/relay_wait.rs | 2 +- agents/monitor/src/steps/update_wait.rs | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index baa46e71..56c83758 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -7,26 +7,26 @@ use warp::Filter; const NAMESPACE: &str = "nomad_monitor"; const TIME_BUCKETS: &[f64] = &[ - 1_000.0, // 1 sec - 5_000.0, // 5 secs - 30_000.0, // 30 secs - 60_000.0, // 1 min - 120_000.0, // 2 min - 600_000.0, // 10 min - 1_800_000.0, // 30 min - 3_600_000.0, // 1 hour - 7_200_000.0, // 2 hour + 1.0, // 1 sec + 5.0, // 5 secs + 30.0, // 30 secs + 60.0, // 1 min + 120.0, // 2 min + 600.0, // 10 min + 1_800.0, // 30 min + 3_600.0, // 1 hour + 7_200.0, // 2 hour ]; // time buckets for e2e metric const E2E_TIME_BUCKETS: &[f64] = &[ - 2_100_000.0, // 35 min - 2_400_000.0, // 40 min - 2_700_000.0, // 45 min - 3_000_000.0, // 50 min - 3_300_000.0, // 55 min - 3_600_000.0, // 1 hour - 7_200_000.0, // 2 hours - 10_800_000.0, // 3 hours + 2_100.0, // 35 min + 2_400.0, // 40 min + 2_700.0, // 45 min + 3_000.0, // 50 min + 3_300.0, // 55 min + 3_600.0, // 1 hour + 7_200.0, // 2 hours + 10_800.0, // 3 hours ]; const BLOCKS_BUCKETS: &[f64] = &[0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 200.0, 500.0, 1000.0]; @@ -69,8 +69,8 @@ impl Metrics { pub(crate) fn new() -> eyre::Result { let e2e_timers = HistogramVec::new( HistogramOpts::new( - "e2e_ms", - "Ms between dispatch and associated process, as observed by this agent", + "e2e_sec", + "Seconds between dispatch and associated process, as observed by this agent", ) .namespace(NAMESPACE) .buckets(E2E_TIME_BUCKETS.to_vec()) @@ -80,8 +80,8 @@ impl Metrics { let update_to_relay_timers = HistogramVec::new( HistogramOpts::new( - "update_to_relay_ms", - "Ms between update and relay, as observed by this agent", + "update_to_relay_secs", + "Seconds between update and relay, as observed by this agent", ) .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) @@ -91,8 +91,8 @@ impl Metrics { let dispatch_to_update_timers = HistogramVec::new( HistogramOpts::new( - "dispatch_to_update_ms", - "Ms between dispatch and update, as observed by this agent", + "dispatch_to_update_secs", + "Seconds between dispatch and update, as observed by this agent", ) .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) @@ -113,8 +113,8 @@ impl Metrics { let relay_to_process_timers = HistogramVec::new( HistogramOpts::new( - "relay_to_process_ms", - "Ms between relay and process, as observed by this agent", + "relay_to_process_secs", + "Seconds between relay and process, as observed by this agent", ) .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) @@ -135,8 +135,8 @@ impl Metrics { let wallclock_times = HistogramVec::new( HistogramOpts::new( - "inter_event_period_wallclock_ms", - "Ms between events periods, as observed by this agent", + "inter_event_period_wallclock_secs", + "Seconds between events periods, as observed by this agent", ) .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index deeb4198..579bffb6 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -87,7 +87,7 @@ impl ProcessStep for RelayWait { let process_instant = tokio::time::Instant::now(); let process_block = process.meta.block_number; - let elapsed_ms = process_instant.saturating_duration_since(self.relay_instant).as_millis() as f64; + let elapsed_ms = process_instant.saturating_duration_since(self.relay_instant).as_secs_f64(); let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; self.metrics.timers.observe(elapsed_ms); diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index ec72d8d0..f31e1ff3 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -101,7 +101,7 @@ impl UpdateWait { .times .get(replica_network) .expect("missing metric") - .observe(v.as_millis() as f64) + .observe(v.as_secs_f64()) } } From 91380120a4525263b0e53c1cff51a7a7ec97c9b6 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 06:57:20 -0700 Subject: [PATCH 47/80] feature: more useful output in between and produce tracing --- agents/monitor/src/steps/between.rs | 2 + agents/monitor/src/steps/producer.rs | 55 +++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 9ac40249..d188a9f7 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -88,6 +88,8 @@ where target: "monitor::between", block_number = %incoming.meta.block_number, event = self.event.as_str(), + tx_hash = ?incoming.meta.transaction_hash, + emitter = ?incoming.meta.address, "received incoming event" ); diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index a916f4cd..f182f470 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -4,7 +4,7 @@ use nomad_ethereum::bindings::{ replica::{ProcessFilter, Replica, UpdateFilter as RelayFilter}, }; use tokio::sync::mpsc::UnboundedSender; -use tracing::{info_span, Instrument}; +use tracing::{info_span, trace, Instrument}; use crate::{annotate::WithMeta, bail_task_if, DispatchSink, ProcessStep, Restartable}; @@ -79,7 +79,18 @@ impl ProcessStep for DispatchProducer { bail_task_if!(res.is_err(), self, res.unwrap_err()); - for event in res.unwrap().into_iter() { + let events = res.unwrap(); + + if !events.is_empty() { + trace!( + count = events.len(), + from = %from, + to = %to, + "Received events" + ); + } + + for event in events.into_iter() { let res = self.tx.send(event.into()); bail_task_if!(res.is_err(), self, res.unwrap_err()); } @@ -170,7 +181,18 @@ impl ProcessStep for UpdateProducer { bail_task_if!(res.is_err(), self, res.unwrap_err()); - for event in res.unwrap().into_iter() { + let events = res.unwrap(); + + if !events.is_empty() { + trace!( + count = events.len(), + from = %from, + to = %to, + "Received events" + ); + } + + for event in events.into_iter() { let res = self.tx.send(event.into()); bail_task_if!(res.is_err(), self, res.unwrap_err()); } @@ -252,7 +274,6 @@ impl ProcessStep for RelayProducer { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { - tracing::trace!(from = from.as_u64(), to = to.as_u64(), "produce_loop"); if from < to { let res = self .replica @@ -264,7 +285,18 @@ impl ProcessStep for RelayProducer { bail_task_if!(res.is_err(), self, res.unwrap_err()); - for event in res.unwrap().into_iter() { + let events = res.unwrap(); + + if !events.is_empty() { + trace!( + count = events.len(), + from = %from, + to = %to, + "Received events" + ); + } + + for event in events.into_iter() { let res = self.tx.send(event.into()); bail_task_if!(res.is_err(), self, res.unwrap_err()); } @@ -356,7 +388,18 @@ impl ProcessStep for ProcessProducer { bail_task_if!(res.is_err(), self, res.unwrap_err()); - for event in res.unwrap().into_iter() { + let events = res.unwrap(); + + if !events.is_empty() { + trace!( + count = events.len(), + from = %from, + to = %to, + "Received events" + ); + } + + for event in events.into_iter() { let res = self.tx.send(event.into()); bail_task_if!(res.is_err(), self, res.unwrap_err()); } From ac2f11c6d23056219763a8c6977b6741633e550e Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 06:57:44 -0700 Subject: [PATCH 48/80] feature: set MONITOR_NETWORKS to filter monitor watching networks --- agents/monitor/src/domain.rs | 7 ++++++- agents/monitor/src/init.rs | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 32d744b2..54d0e72e 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -31,7 +31,11 @@ impl Domain { format!("{:?}", self.home.address()) } - pub(crate) fn from_config(config: &NomadConfig, network: &str) -> eyre::Result { + pub(crate) fn from_config( + config: &NomadConfig, + network: &str, + to_monitor: &[String], + ) -> eyre::Result { let network = network.to_owned(); let provider = provider_for(config, &network)?; @@ -47,6 +51,7 @@ impl Domain { let replicas = core .replicas .iter() + .filter(|(s, _)| to_monitor.contains(s)) .map(|(k, v)| { let replica = Replica::new( v.proxy.as_ethereum_address().expect("invalid addr"), diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 8a64fd02..e72f0462 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -47,6 +47,12 @@ pub(crate) fn init_tracing() { .init(); } +pub(crate) fn networks_from_env() -> Option> { + std::env::var("MONITOR_NETWORKS") + .ok() + .map(|s| s.split(',').map(ToOwned::to_owned).collect()) +} + pub(crate) fn rpc_from_env(network: &str) -> Option { std::env::var(format!("{}_CONNECTION_URL", network.to_uppercase())).ok() } @@ -94,8 +100,13 @@ pub(crate) struct Monitor { impl Monitor { pub(crate) fn from_config(config: &NomadConfig) -> eyre::Result { let mut networks = HashMap::new(); - for network in config.networks.iter() { - networks.insert(network.to_owned(), Domain::from_config(config, network)?); + let to_monitor = + networks_from_env().unwrap_or_else(|| config.networks.iter().cloned().collect()); + for network in config.networks.iter().filter(|s| to_monitor.contains(s)) { + networks.insert( + network.to_owned(), + Domain::from_config(config, network, &to_monitor)?, + ); } let metrics = Metrics::new()?.into(); Ok(Monitor { networks, metrics }) From 0d49c5c4144bccc17faf88b38dfbe9f12a65e042 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 10:57:16 -0700 Subject: [PATCH 49/80] fix: better distinguish between recoverable and non-recoverable errors --- agents/monitor/src/macros.rs | 20 ++++++------- agents/monitor/src/pipe.rs | 6 ++-- agents/monitor/src/steps/between.rs | 6 ++-- agents/monitor/src/steps/combine.rs | 18 ++++-------- agents/monitor/src/steps/dispatch_wait.rs | 6 ++-- agents/monitor/src/steps/e2e.rs | 29 ++++--------------- agents/monitor/src/steps/producer.rs | 12 +++----- agents/monitor/src/steps/relay_wait.rs | 6 ++-- agents/monitor/src/steps/update_wait.rs | 35 +++++------------------ 9 files changed, 43 insertions(+), 95 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index 54f847e7..cdabd1e0 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -14,16 +14,16 @@ macro_rules! bail_task_if { } #[macro_export] -macro_rules! unwrap_pipe_item { - ($pipe_output:ident, $self:ident,) => {{ - unwrap_pipe_output!($pipe_output, $self) +macro_rules! unwrap_channel_item { + ($channel_item:ident, $self:ident,) => {{ + unwrap_channel_item!($channel_item, $self) }}; - ($pipe_output:ident, $self:ident) => {{ - $crate::bail_task_if!($pipe_output.is_err(), $self, $pipe_output.unwrap_err(),); - - let item_opt = $pipe_output.unwrap(); - $crate::bail_task_if!(item_opt.is_none(), $self, "inbound pipe failed",); - - item_opt.unwrap() + ($channel_item:ident, $self:ident) => {{ + if $channel_item.is_none() { + tracing::debug!( + task = %$self, "inbound channel broke" + ); + } + $channel_item.expect("inbound channel broke") }}; } diff --git a/agents/monitor/src/pipe.rs b/agents/monitor/src/pipe.rs index 7c47fd9d..e8dc6f10 100644 --- a/agents/monitor/src/pipe.rs +++ b/agents/monitor/src/pipe.rs @@ -39,13 +39,13 @@ where Ok(()) } - pub(crate) async fn next(&mut self) -> eyre::Result> { + pub(crate) async fn next(&mut self) -> eyre::Result<&T> { self.finish()?; self.contents = self.rx.recv().await; if self.contents.is_none() { - bail!("Rx Broke") + bail!("rx broke") } - Ok(self.read()) + Ok(self.read().expect("checked")) } } diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index d188a9f7..b8e38422 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,6 +1,6 @@ use tracing::{info_span, Instrument}; -use crate::{annotate::WithMeta, pipe::Pipe, unwrap_pipe_item, ProcessStep, Restartable}; +use crate::{annotate::WithMeta, pipe::Pipe, ProcessStep, Restartable}; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, @@ -80,9 +80,7 @@ where loop { // get the next event from the channel - let incoming = self.pipe.next().await; - - let incoming = unwrap_pipe_item!(incoming, self); + let incoming = self.pipe.next().await.expect("inbound pipe failed"); tracing::debug!( target: "monitor::between", diff --git a/agents/monitor/src/steps/combine.rs b/agents/monitor/src/steps/combine.rs index 2854c103..c3a9ccb6 100644 --- a/agents/monitor/src/steps/combine.rs +++ b/agents/monitor/src/steps/combine.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, fmt::Display}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; -use crate::{bail_task_if, utils::nexts, ProcessStep, Restartable}; +use crate::{unwrap_channel_item, utils::nexts, ProcessStep, Restartable}; #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] @@ -59,17 +59,11 @@ where async move { loop { let ((net, next_opt), _, _) = select_all(nexts(&mut self.faucets)).await; - bail_task_if!( - next_opt.is_none(), - self, - format!("Inbound from {} broke", net), - ); - let next = next_opt.expect("checked"); - bail_task_if!( - self.sink.send((net, next)).is_err(), - self, - "sink channel broke" - ); + let next = unwrap_channel_item!(next_opt, self); + + self.sink + .send((net, next)) + .expect("outbound channel failed"); } } .instrument(span), diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index c1e242ad..336b9c8c 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -5,7 +5,7 @@ use tracing::{info_span, Instrument}; use crate::{ pipe::{DispatchPipe, UpdatePipe}, - unwrap_pipe_item, ProcessStep, Restartable, + ProcessStep, Restartable, }; #[derive(Debug)] @@ -107,12 +107,12 @@ impl ProcessStep for DispatchWait { biased; dispatch_next = self.dispatch_pipe.next() => { - let dispatch = unwrap_pipe_item!(dispatch_next,self); + let dispatch = dispatch_next.expect("inbound dispatch pipe failed"); let block_number = dispatch.meta.block_number; self.handle_dispatch(block_number); } update_next = self.update_pipe.next() => { - let update = unwrap_pipe_item!(update_next, self); + let update = update_next.expect("inbound update pipe failed"); let block_number = update.meta.block_number; self.handle_update(block_number); } diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index e32a4384..641c772b 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -11,8 +11,7 @@ use ethers::prelude::H256; use tracing::{info_span, Instrument}; use crate::{ - annotate::WithMeta, bail_task_if, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink, - ProcessStep, + annotate::WithMeta, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink, ProcessStep, }; use super::combine::CombineChannels; @@ -140,40 +139,22 @@ impl ProcessStep for E2ELatency { loop { tokio::select! { dispatch_opt = self.dispatch_faucet.recv() => { - bail_task_if!( - dispatch_opt.is_none(), - self, - "inbound dispatch broke", - ); - let (network, dispatch) = dispatch_opt.expect("checked"); + let (network, dispatch) = dispatch_opt.expect("inbound dispatch broke"); let message_hash: H256 = dispatch.log.message_hash.into(); let destination: u32 = (dispatch.log.destination_and_nonce >> 32) as u32; let outbound = self.dispatch_sinks.get(&network).expect("missing sink"); - - bail_task_if!( - outbound.send(dispatch).is_err(), self, "outbound dispatch broke", - ); + outbound.send(dispatch).expect("outbound dispatch broke"); self.record_dispatch(network,destination, message_hash); } process_opt = self.process_faucet.recv() => { - bail_task_if!( - process_opt.is_none(), - self, - "inbound process broke", - ); - - let (network, (replica_of, process)) = process_opt.expect("checked"); + let (network, (replica_of, process)) = process_opt.expect("inbound process broke"); let message_hash: H256 = process.log.message_hash.into(); let outbound = self.process_sinks.get(&network).expect("missing network").get(&replica_of).expect("missing sink"); + outbound.send(process).expect("outbound dispatch broke"); - bail_task_if!( - outbound.send(process).is_err(), - self, - "outbound dispatch broke", - ); self.record_process(network, replica_of, message_hash); } } diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index f182f470..9046a35d 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -91,8 +91,7 @@ impl ProcessStep for DispatchProducer { } for event in events.into_iter() { - let res = self.tx.send(event.into()); - bail_task_if!(res.is_err(), self, res.unwrap_err()); + self.tx.send(event.into()).expect("outbound channel broke"); } } let tip_res = provider.get_block_number().await; @@ -193,8 +192,7 @@ impl ProcessStep for UpdateProducer { } for event in events.into_iter() { - let res = self.tx.send(event.into()); - bail_task_if!(res.is_err(), self, res.unwrap_err()); + self.tx.send(event.into()).expect("outbound channel broke"); } } let tip_res = provider.get_block_number().await; @@ -297,8 +295,7 @@ impl ProcessStep for RelayProducer { } for event in events.into_iter() { - let res = self.tx.send(event.into()); - bail_task_if!(res.is_err(), self, res.unwrap_err()); + self.tx.send(event.into()).expect("outbound channel broke"); } } let tip_res = provider.get_block_number().await; @@ -400,8 +397,7 @@ impl ProcessStep for ProcessProducer { } for event in events.into_iter() { - let res = self.tx.send(event.into()); - bail_task_if!(res.is_err(), self, res.unwrap_err()); + self.tx.send(event.into()).expect("outbound channel broke"); } } let tip_res = provider.get_block_number().await; diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 579bffb6..1cf01622 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -8,7 +8,7 @@ use tracing::{info_span, Instrument}; use crate::{ pipe::{ProcessPipe, RelayPipe}, - unwrap_pipe_item, ProcessStep, + ProcessStep, }; #[derive(Debug)] @@ -83,7 +83,7 @@ impl ProcessStep for RelayWait { biased; process_next = self.process_pipe.next() => { - let process = unwrap_pipe_item!(process_next, self); + let process = process_next.expect("inbound process pipe failed"); let process_instant = tokio::time::Instant::now(); let process_block = process.meta.block_number; @@ -94,7 +94,7 @@ impl ProcessStep for RelayWait { self.metrics.blocks.observe(elapsed_blocks); } relay_next = self.relay_pipe.next() => { - let relay = unwrap_pipe_item!(relay_next, self); + let relay = relay_next.expect("inbound relay pipe failed"); self.relay_instant = tokio::time::Instant::now(); self.relay_block = relay.meta.block_number; } diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index f31e1ff3..95a87d3d 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -7,8 +7,8 @@ use tracing::{info_span, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ - annotate::WithMeta, bail_task_if, steps::combine::CombineChannels, ProcessStep, RelayFaucet, - RelaySink, UpdateFaucet, UpdateSink, + annotate::WithMeta, steps::combine::CombineChannels, ProcessStep, RelayFaucet, RelaySink, + UpdateFaucet, UpdateSink, }; #[derive(Debug)] @@ -124,41 +124,20 @@ impl ProcessStep for UpdateWait { biased; update_opt = self.update_faucet.recv() => { - bail_task_if!( - update_opt.is_none(), - self, - "Inbound updates broke", - ); - let update = update_opt.unwrap(); + let update = update_opt.expect("inbound update channel broke"); let root: H256 = update.log.new_root.into(); - bail_task_if!{ - // send onwards - self.update_sink.send(update).is_err(), - self, - "Outbound updates broke", - }; - + self.update_sink.send(update).expect("outbound update broke"); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { - bail_task_if!( - relay_opt.is_none(), - self, - format!("Inbound relays broke"), - ); - let (replica_network, relay) = relay_opt.unwrap(); + let (replica_network, relay) = relay_opt.expect("Inbound relays broke"); let root: H256 = relay.log.new_root.into(); - bail_task_if!( - // send onward - self.relay_sinks + self.relay_sinks .get(&replica_network) .expect("missing outgoing") .send(relay) - .is_err(), - self, - format!("outgoing relay for {} broke", &replica_network) - ); + .unwrap_or_else(|_| panic!("outgoing relay for {} broke", &replica_network)); self.handle_relay(&replica_network, root); } From fee18e2677aaad08899f556cb49b326a8360ab9f Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 11:08:45 -0700 Subject: [PATCH 50/80] fix: switch back to json logging --- agents/monitor/src/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index e72f0462..33af4f5c 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -40,7 +40,7 @@ pub(crate) fn config() -> eyre::Result { pub(crate) fn init_tracing() { tracing_subscriber::FmtSubscriber::builder() - .pretty() + .json() .with_max_level(Level::INFO) .with_env_filter(EnvFilter::from_default_env()) .with_level(true) From 5a9885678875c9e979facc8909e8be1321ab35c1 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 11:14:14 -0700 Subject: [PATCH 51/80] fix: producers properly update From --- agents/monitor/src/init.rs | 2 +- agents/monitor/src/steps/producer.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 33af4f5c..e72f0462 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -40,7 +40,7 @@ pub(crate) fn config() -> eyre::Result { pub(crate) fn init_tracing() { tracing_subscriber::FmtSubscriber::builder() - .json() + .pretty() .with_max_level(Level::INFO) .with_env_filter(EnvFilter::from_default_env()) .with_level(true) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 9046a35d..39a85021 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -65,9 +65,9 @@ impl ProcessStep for DispatchProducer { async move { let provider = self.home.client(); let height = provider.get_block_number().await.unwrap(); - let from = self.from.unwrap_or(height - (2 * BEHIND_TIP)); let mut to = height - BEHIND_TIP; loop { + let from = self.from.unwrap_or(height - (2 * BEHIND_TIP)); if from < to { let res = self .home @@ -166,9 +166,9 @@ impl ProcessStep for UpdateProducer { bail_task_if!(height.is_err(), self, "Err retrieving height"); let height = height.expect("checked"); - let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { let res = self .home @@ -269,9 +269,9 @@ impl ProcessStep for RelayProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { let res = self .replica @@ -371,9 +371,9 @@ impl ProcessStep for ProcessProducer { async move { let provider = self.replica.client(); let height = provider.get_block_number().await.unwrap(); - let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); let mut to = height - BEHIND_TIP; loop { + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { let res = self .replica From 7f5d6cd608c3d732306892e13bcb7d2cc11bf20d Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 11:15:51 -0700 Subject: [PATCH 52/80] fix: prevent james from accidentally committing pretty over and over --- agents/monitor/src/init.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index e72f0462..b251ede9 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -39,12 +39,15 @@ pub(crate) fn config() -> eyre::Result { } pub(crate) fn init_tracing() { - tracing_subscriber::FmtSubscriber::builder() - .pretty() + let builder = tracing_subscriber::FmtSubscriber::builder() .with_max_level(Level::INFO) .with_env_filter(EnvFilter::from_default_env()) - .with_level(true) - .init(); + .with_level(true); + if std::env::var("MONITOR_PRETTY").is_ok() { + builder.pretty().init() + } else { + builder.json().init() + } } pub(crate) fn networks_from_env() -> Option> { From be35c4e2658f0d32ae9102775777e09d27fcacbf Mon Sep 17 00:00:00 2001 From: James Date: Tue, 26 Jul 2022 11:30:37 -0700 Subject: [PATCH 53/80] fix: one more unintended crashloop --- agents/monitor/src/steps/terminal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/monitor/src/steps/terminal.rs b/agents/monitor/src/steps/terminal.rs index 04da3147..57cb7969 100644 --- a/agents/monitor/src/steps/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -45,7 +45,7 @@ where loop { if self.rx.recv().await.is_none() { tracing::debug!(self = %self, "Upstream broke, shutting down"); - return (self, eyre::eyre!("")); + panic!(); } } } From fce1067c56a7c74d2fa2e06407c1bbc78a5a6e0e Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 00:12:50 -0700 Subject: [PATCH 54/80] feature: TaskResult to differentiate panics from unrecoverable task errors --- agents/monitor/src/macros.rs | 16 +++++++++-- agents/monitor/src/main.rs | 41 +++++++++++++++++++++------- agents/monitor/src/steps/mod.rs | 6 ++++ agents/monitor/src/steps/producer.rs | 17 ++++++------ 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index cdabd1e0..a3117c88 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -8,7 +8,7 @@ macro_rules! bail_task_if { if $cond { let err = eyre::eyre!($err); tracing::error!(task = %$self, err = %err, "Task failed"); - return ($self, err); + return $crate::steps::TaskResult::Recoverable{task: $self, err}; } }; } @@ -23,7 +23,19 @@ macro_rules! unwrap_channel_item { tracing::debug!( task = %$self, "inbound channel broke" ); + return $crate::steps::TaskResult::Unrecoverable(eyre::eyre!("inbound channel broke")) } - $channel_item.expect("inbound channel broke") + $channel_item.unwrap() + }}; +} + +#[macro_export] +macro_rules! unwrap_or_bail { + ($result:ident, $self:ident,) => {{ + unwrap_err_or_bail!($result, $self) + }}; + ($result:ident, $self:ident) => {{ + bail_task_if!($result.is_err(), $self, $result.unwrap_err()); + $result.unwrap() }}; } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 2f676bab..0b1cf7a7 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -5,6 +5,7 @@ use nomad_ethereum::bindings::{ replica::{ProcessFilter, UpdateFilter as RelayFilter}, }; use std::{collections::HashMap, panic, sync::Arc}; +use steps::TaskResult; use tokio::{ sync::mpsc::{UnboundedReceiver, UnboundedSender}, task::JoinHandle, @@ -27,7 +28,7 @@ pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; // pub(crate) type ProviderError = ContractError; -pub(crate) type Restartable = JoinHandle<(Task, eyre::Report)>; +pub(crate) type Restartable = JoinHandle>; pub(crate) type Faucet = UnboundedReceiver; pub(crate) type Sink = UnboundedSender; @@ -73,6 +74,7 @@ async fn main() -> eyre::Result<()> { // this would imply there is a series of upstream channel failures let (_, _, _) = select_all(tasks).await; } + Ok(()) } @@ -94,18 +96,37 @@ pub(crate) trait ProcessStep: std::fmt::Display { loop { let result = handle.await; - let (again, report) = match result { - Ok((handle, report)) => (handle, report), + let again = match result { + Ok(TaskResult::Recoverable { task, err }) => { + tracing::warn!( + error = %err, + task = task_description.as_str(), + "Restarting task", + ); + task + } + + Ok(TaskResult::Unrecoverable(err)) => { + tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + break; + } + Err(e) => { - tracing::error!(err = %e, task = task_description.as_str(), "Internal task panicked"); - panic!("JoinError in forever. Internal task panicked"); + let panic_res = e.try_into_panic(); + + if panic_res.is_err() { + tracing::trace!( + task = task_description.as_str(), + "Internal task cancelled" + ); + break; + } + let p = panic_res.unwrap(); + tracing::error!("Internal task panicked"); + panic::resume_unwind(p); } }; - tracing::warn!( - error = %report, - task = task_description.as_str(), - "Restarting task", - ); + handle = again.spawn(); } }) diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index 44cbb238..e4067956 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -6,3 +6,9 @@ pub(crate) mod producer; pub(crate) mod relay_wait; pub(crate) mod terminal; pub(crate) mod update_wait; + +#[derive(Debug)] +pub(crate) enum TaskResult { + Recoverable { task: T, err: eyre::Report }, + Unrecoverable(eyre::Report), +} diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 39a85021..a3d357b9 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -6,7 +6,9 @@ use nomad_ethereum::bindings::{ use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, trace, Instrument}; -use crate::{annotate::WithMeta, bail_task_if, DispatchSink, ProcessStep, Restartable}; +use crate::{ + annotate::WithMeta, bail_task_if, unwrap_or_bail, DispatchSink, ProcessStep, Restartable, +}; pub const POLLING_INTERVAL_SECS: u64 = 5; pub const BEHIND_TIP: u64 = 5; @@ -95,9 +97,8 @@ impl ProcessStep for DispatchProducer { } } let tip_res = provider.get_block_number().await; - bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); + let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; - let tip = tip_res.unwrap() - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); @@ -196,9 +197,8 @@ impl ProcessStep for UpdateProducer { } } let tip_res = provider.get_block_number().await; - bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); + let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; - let tip = tip_res.expect("checked") - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); @@ -299,8 +299,8 @@ impl ProcessStep for RelayProducer { } } let tip_res = provider.get_block_number().await; - bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - BEHIND_TIP; + let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; + self.from = Some(to); to = std::cmp::max(to, tip); @@ -401,9 +401,8 @@ impl ProcessStep for ProcessProducer { } } let tip_res = provider.get_block_number().await; - bail_task_if!(tip_res.is_err(), self, tip_res.unwrap_err()); - let tip = tip_res.unwrap() - BEHIND_TIP; + let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); From fc2bfcfa32b9da3177597906ddbaa670f2b9ad71 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 00:19:24 -0700 Subject: [PATCH 55/80] feature: log count of producers --- agents/monitor/src/init.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index b251ede9..012e3cf7 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -120,31 +120,43 @@ impl Monitor { } fn run_dispatch_producers(&self) -> HashMap<&str, DispatchFaucet> { - self.networks + let faucets: HashMap<_, _> = self + .networks .iter() .map(|(network, domain)| (network.as_str(), domain.dispatch_producer())) - .collect() + .collect(); + tracing::debug!(count = faucets.len(), "running dispatch_producer"); + faucets } fn run_update_producers(&self) -> HashMap<&str, UpdateFaucet> { - self.networks + let faucets: HashMap<_, _> = self + .networks .iter() .map(|(network, domain)| (network.as_str(), domain.update_producer())) - .collect() + .collect(); + tracing::debug!(count = faucets.len(), "running update_producer"); + faucets } fn run_relay_producers(&self) -> HomeReplicaMap { - self.networks + let faucets: HashMap<_, _> = self + .networks .iter() .map(|(network, domain)| (network.as_str(), domain.relay_producers())) - .collect() + .collect(); + tracing::debug!(count = faucets.len(), "running relay_producers"); + faucets } fn run_process_producers(&self) -> HomeReplicaMap { - self.networks + let faucets: HashMap<_, _> = self + .networks .iter() .map(|(network, domain)| (network.as_str(), domain.process_producers())) - .collect() + .collect(); + tracing::debug!(count = faucets.len(), "running process_producers"); + faucets } pub(crate) fn producers(&self) -> Faucets { From 406256f9521e8f5414da60f7d99dcb6459747665 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 00:54:19 -0700 Subject: [PATCH 56/80] test: add recovery tests --- Cargo.lock | 39 +++++++++++++- agents/monitor/Cargo.toml | 1 + agents/monitor/src/main.rs | 102 +++++++++++++++++++++++++++++++++++-- 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49bea57e..2f767ca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2064,6 +2064,15 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.9" @@ -2290,6 +2299,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber 0.2.25", + "tracing-test", "warp", ] @@ -4405,7 +4415,7 @@ dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers", + "matchers 0.0.1", "regex", "serde 1.0.140", "serde_json", @@ -4425,16 +4435,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" dependencies = [ "ansi_term", + "matchers 0.1.0", + "once_cell", + "regex", "serde 1.0.140", "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", "tracing-serde", ] +[[package]] +name = "tracing-test" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3d272c44878d2bbc9f4a20ad463724f03e19dbc667c6e84ac433ab7ffcc70b" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber 0.3.15", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744324b12d69a9fc1edea4b38b7b1311295b662d161ad5deac17bb1358224a08" +dependencies = [ + "lazy_static", + "quote", + "syn", +] + [[package]] name = "try-lock" version = "0.2.3" diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index d0f35b64..72cc5df8 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -25,3 +25,4 @@ nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } warp = "0.3.2" async-trait = "0.1.56" futures-util = "0.3.21" +tracing-test = "0.2.3" diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 0b1cf7a7..73645eb2 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -4,7 +4,7 @@ use nomad_ethereum::bindings::{ home::{DispatchFilter, UpdateFilter}, replica::{ProcessFilter, UpdateFilter as RelayFilter}, }; -use std::{collections::HashMap, panic, sync::Arc}; +use std::{collections::HashMap, panic, sync::Arc, time::Duration}; use steps::TaskResult; use tokio::{ sync::mpsc::{UnboundedReceiver, UnboundedSender}, @@ -117,18 +117,114 @@ pub(crate) trait ProcessStep: std::fmt::Display { if panic_res.is_err() { tracing::trace!( task = task_description.as_str(), - "Internal task cancelled" + "Internal task cancelled", ); break; } let p = panic_res.unwrap(); - tracing::error!("Internal task panicked"); + tracing::error!(task = task_description.as_str(), "Internal task panicked"); panic::resume_unwind(p); } }; + tokio::time::sleep(Duration::from_secs(15)).await; handle = again.spawn(); } }) } } + +#[cfg(test)] +mod test { + use crate::{steps::TaskResult, ProcessStep}; + + struct RecoverableTask; + impl std::fmt::Display for RecoverableTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RecoverableTask") + } + } + + impl ProcessStep for RecoverableTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { + TaskResult::Recoverable { + task: self, + err: eyre::eyre!("This error was recoverable"), + } + }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_recovery() { + let handle = RecoverableTask.run_until_panic(); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + handle.abort(); + let result = handle.await; + + assert!(logs_contain("RecoverableTask")); + assert!(logs_contain("Restarting task")); + assert!(logs_contain("This error was recoverable")); + assert!(result.is_err() && result.unwrap_err().is_cancelled()); + } + + struct UnrecoverableTask; + impl std::fmt::Display for UnrecoverableTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "UnrecoverableTask") + } + } + + impl ProcessStep for UnrecoverableTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { + TaskResult::Unrecoverable(eyre::eyre!("This error was unrecoverable")) + }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_unrecoverable() { + let handle = UnrecoverableTask.run_until_panic(); + let result = handle.await; + assert!(logs_contain("UnrecoverableTask")); + assert!(logs_contain("Unrecoverable error encountered")); + assert!(logs_contain("This error was unrecoverable")); + assert!(result.is_ok()); + } + + struct PanicTask; + impl std::fmt::Display for PanicTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PanicTask") + } + } + + impl ProcessStep for PanicTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { panic!("intentional panic :)") }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_panic() { + let handle = PanicTask.run_until_panic(); + let result = handle.await; + assert!(logs_contain("PanicTask")); + assert!(logs_contain("Internal task panicked")); + assert!(result.is_err() && result.unwrap_err().is_panic()); + } +} From 8a64090c4321641df54cd5e746f00e41a853c3a3 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 01:03:35 -0700 Subject: [PATCH 57/80] feature: quieter logging for unremarkable errors --- agents/monitor/src/macros.rs | 2 +- agents/monitor/src/main.rs | 11 ++++++++--- agents/monitor/src/steps/mod.rs | 10 ++++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index a3117c88..7ab90917 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -23,7 +23,7 @@ macro_rules! unwrap_channel_item { tracing::debug!( task = %$self, "inbound channel broke" ); - return $crate::steps::TaskResult::Unrecoverable(eyre::eyre!("inbound channel broke")) + return $crate::steps::TaskResult::Unrecoverable{err: eyre::eyre!("inbound channel broke"), worth_logging: false} } $channel_item.unwrap() }}; diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 73645eb2..9429dbe2 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -106,8 +106,10 @@ pub(crate) trait ProcessStep: std::fmt::Display { task } - Ok(TaskResult::Unrecoverable(err)) => { - tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + Ok(TaskResult::Unrecoverable { err, worth_logging }) => { + if worth_logging { + tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + } break; } @@ -186,7 +188,10 @@ mod test { Self: 'static + Send + Sync + Sized, { tokio::spawn(async move { - TaskResult::Unrecoverable(eyre::eyre!("This error was unrecoverable")) + TaskResult::Unrecoverable { + err: eyre::eyre!("This error was unrecoverable"), + worth_logging: true, + } }) } } diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index e4067956..44a8dd81 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -9,6 +9,12 @@ pub(crate) mod update_wait; #[derive(Debug)] pub(crate) enum TaskResult { - Recoverable { task: T, err: eyre::Report }, - Unrecoverable(eyre::Report), + Recoverable { + task: T, + err: eyre::Report, + }, + Unrecoverable { + err: eyre::Report, + worth_logging: bool, + }, } From 3a1a9326a40fcddccfe66f39232b33385b9fa698 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 01:31:09 -0700 Subject: [PATCH 58/80] refactor: can't send is an unrecoverable, not a panic --- agents/monitor/src/macros.rs | 16 +++++++++++-- agents/monitor/src/main.rs | 2 ++ agents/monitor/src/steps/combine.rs | 4 ++-- agents/monitor/src/steps/producer.rs | 35 ++++++++++++++++------------ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index 7ab90917..aa2a3d86 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -14,7 +14,7 @@ macro_rules! bail_task_if { } #[macro_export] -macro_rules! unwrap_channel_item { +macro_rules! unwrap_channel_item_unrecoverable { ($channel_item:ident, $self:ident,) => {{ unwrap_channel_item!($channel_item, $self) }}; @@ -30,7 +30,7 @@ macro_rules! unwrap_channel_item { } #[macro_export] -macro_rules! unwrap_or_bail { +macro_rules! unwrap_result_recoverable { ($result:ident, $self:ident,) => {{ unwrap_err_or_bail!($result, $self) }}; @@ -39,3 +39,15 @@ macro_rules! unwrap_or_bail { $result.unwrap() }}; } + +#[macro_export] +macro_rules! send_unrecoverable { + ($tx:expr, $item:expr, $self:ident) => { + if $tx.send($item).is_err() { + return $crate::steps::TaskResult::Unrecoverable { + err: eyre::eyre!("outbound channel broke"), + worth_logging: false, + }; + } + }; +} diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 9429dbe2..9b161ea6 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -109,6 +109,8 @@ pub(crate) trait ProcessStep: std::fmt::Display { Ok(TaskResult::Unrecoverable { err, worth_logging }) => { if worth_logging { tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + } else { + tracing::trace!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); } break; } diff --git a/agents/monitor/src/steps/combine.rs b/agents/monitor/src/steps/combine.rs index c3a9ccb6..008b3e5b 100644 --- a/agents/monitor/src/steps/combine.rs +++ b/agents/monitor/src/steps/combine.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, fmt::Display}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; -use crate::{unwrap_channel_item, utils::nexts, ProcessStep, Restartable}; +use crate::{unwrap_channel_item_unrecoverable, utils::nexts, ProcessStep, Restartable}; #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] @@ -59,7 +59,7 @@ where async move { loop { let ((net, next_opt), _, _) = select_all(nexts(&mut self.faucets)).await; - let next = unwrap_channel_item!(next_opt, self); + let next = unwrap_channel_item_unrecoverable!(next_opt, self); self.sink .send((net, next)) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index a3d357b9..a330cf32 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -7,7 +7,8 @@ use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, trace, Instrument}; use crate::{ - annotate::WithMeta, bail_task_if, unwrap_or_bail, DispatchSink, ProcessStep, Restartable, + annotate::WithMeta, bail_task_if, send_unrecoverable, unwrap_result_recoverable, DispatchSink, + ProcessStep, Restartable, }; pub const POLLING_INTERVAL_SECS: u64 = 5; @@ -69,8 +70,9 @@ impl ProcessStep for DispatchProducer { let height = provider.get_block_number().await.unwrap(); let mut to = height - BEHIND_TIP; loop { - let from = self.from.unwrap_or(height - (2 * BEHIND_TIP)); + let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { + trace!(from = %from, to = %to, "querying dispatch events"); let res = self .home .dispatch_filter() @@ -88,16 +90,16 @@ impl ProcessStep for DispatchProducer { count = events.len(), from = %from, to = %to, - "Received events" + "Received dispatch events" ); } for event in events.into_iter() { - self.tx.send(event.into()).expect("outbound channel broke"); + send_unrecoverable!(self.tx, event.into(), self); } } let tip_res = provider.get_block_number().await; - let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; + let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); @@ -171,6 +173,7 @@ impl ProcessStep for UpdateProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { + trace!(from = %from, to = %to, "querying update events"); let res = self .home .update_filter() @@ -188,16 +191,16 @@ impl ProcessStep for UpdateProducer { count = events.len(), from = %from, to = %to, - "Received events" + "Received update events" ); } for event in events.into_iter() { - self.tx.send(event.into()).expect("outbound channel broke"); + send_unrecoverable!(self.tx, event.into(), self); } } let tip_res = provider.get_block_number().await; - let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; + let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); @@ -273,6 +276,7 @@ impl ProcessStep for RelayProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { + trace!(from = %from, to = %to, "querying relay events"); let res = self .replica .update_filter() @@ -288,18 +292,18 @@ impl ProcessStep for RelayProducer { if !events.is_empty() { trace!( count = events.len(), - from = %from, + from=%from, to = %to, - "Received events" + "Received relay events" ); } for event in events.into_iter() { - self.tx.send(event.into()).expect("outbound channel broke"); + send_unrecoverable!(self.tx, event.into(), self); } } let tip_res = provider.get_block_number().await; - let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; + let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); @@ -375,6 +379,7 @@ impl ProcessStep for ProcessProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { + trace!(from = %from, to = %to, "querying process events"); let res = self .replica .process_filter() @@ -392,17 +397,17 @@ impl ProcessStep for ProcessProducer { count = events.len(), from = %from, to = %to, - "Received events" + "Received process events" ); } for event in events.into_iter() { - self.tx.send(event.into()).expect("outbound channel broke"); + send_unrecoverable!(self.tx, event.into(), self); } } let tip_res = provider.get_block_number().await; - let tip = unwrap_or_bail!(tip_res, self) - BEHIND_TIP; + let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; self.from = Some(to); to = std::cmp::max(to, tip); From 8268a25fdde2e3dad585464e83d6a74eff46a9e5 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 02:06:03 -0700 Subject: [PATCH 59/80] feature: noisy_sleep to prevent producer sync --- agents/monitor/src/steps/mod.rs | 9 +++++++++ agents/monitor/src/steps/producer.rs | 25 ++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index 44a8dd81..998541a2 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -1,3 +1,5 @@ +use ethers::prelude::rand::{prelude::ThreadRng, Rng}; + pub(crate) mod between; pub(crate) mod combine; pub(crate) mod dispatch_wait; @@ -18,3 +20,10 @@ pub(crate) enum TaskResult { worth_logging: bool, }, } + +// adds up to a second of random delay to cause production tasks to not be synced +pub(crate) fn noisy_sleep(approx_millis: u64) -> tokio::time::Sleep { + let noise = ThreadRng::default().gen_range(0..1000u64); + let duration = std::time::Duration::from_millis(approx_millis + noise); + tokio::time::sleep(duration) +} diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index a330cf32..00413dd9 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -7,11 +7,11 @@ use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, trace, Instrument}; use crate::{ - annotate::WithMeta, bail_task_if, send_unrecoverable, unwrap_result_recoverable, DispatchSink, - ProcessStep, Restartable, + annotate::WithMeta, bail_task_if, send_unrecoverable, steps::noisy_sleep, + unwrap_result_recoverable, DispatchSink, ProcessStep, Restartable, }; -pub const POLLING_INTERVAL_SECS: u64 = 5; +pub const POLLING_INTERVAL_MILLIS: u64 = 5000; pub const BEHIND_TIP: u64 = 5; #[derive(Debug)] @@ -72,7 +72,7 @@ impl ProcessStep for DispatchProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { - trace!(from = %from, to = %to, "querying dispatch events"); + trace!(from = %from, to = %to, progress = %(to - from), "querying dispatch events"); let res = self .home .dispatch_filter() @@ -90,6 +90,7 @@ impl ProcessStep for DispatchProducer { count = events.len(), from = %from, to = %to, + progress = %(to - from), "Received dispatch events" ); } @@ -104,7 +105,7 @@ impl ProcessStep for DispatchProducer { self.from = Some(to); to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; + noisy_sleep(POLLING_INTERVAL_MILLIS).await; } } .instrument(span), @@ -173,7 +174,7 @@ impl ProcessStep for UpdateProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { - trace!(from = %from, to = %to, "querying update events"); + trace!(from = %from, to = %to, progress = %(to - from), "querying update events"); let res = self .home .update_filter() @@ -191,6 +192,7 @@ impl ProcessStep for UpdateProducer { count = events.len(), from = %from, to = %to, + progress = %(to - from), "Received update events" ); } @@ -205,7 +207,7 @@ impl ProcessStep for UpdateProducer { self.from = Some(to); to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; + noisy_sleep(POLLING_INTERVAL_MILLIS).await; } } .instrument(span), @@ -276,7 +278,7 @@ impl ProcessStep for RelayProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { - trace!(from = %from, to = %to, "querying relay events"); + trace!(from = %from, to = %to, progress = %(to - from), "querying relay events"); let res = self .replica .update_filter() @@ -308,7 +310,7 @@ impl ProcessStep for RelayProducer { self.from = Some(to); to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; + noisy_sleep(POLLING_INTERVAL_MILLIS).await; } } .instrument(span), @@ -379,7 +381,7 @@ impl ProcessStep for ProcessProducer { loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); if from < to { - trace!(from = %from, to = %to, "querying process events"); + trace!(from = %from, to = %to, progress = %(to - from), "querying process events"); let res = self .replica .process_filter() @@ -397,6 +399,7 @@ impl ProcessStep for ProcessProducer { count = events.len(), from = %from, to = %to, + progress = %(to - from), "Received process events" ); } @@ -411,7 +414,7 @@ impl ProcessStep for ProcessProducer { self.from = Some(to); to = std::cmp::max(to, tip); - tokio::time::sleep(std::time::Duration::from_secs(POLLING_INTERVAL_SECS)).await; + noisy_sleep(POLLING_INTERVAL_MILLIS).await; } } .instrument(span), From aa67e2a6e8de7c2a1f286b5e0f5e0fdb58883d6e Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 19:07:52 +0200 Subject: [PATCH 60/80] fix: do not double count events on block range edges --- agents/monitor/src/steps/producer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 00413dd9..0208e26b 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -102,7 +102,7 @@ impl ProcessStep for DispatchProducer { let tip_res = provider.get_block_number().await; let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; - self.from = Some(to); + self.from = Some(to + 1); to = std::cmp::max(to, tip); noisy_sleep(POLLING_INTERVAL_MILLIS).await; @@ -204,7 +204,7 @@ impl ProcessStep for UpdateProducer { let tip_res = provider.get_block_number().await; let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; - self.from = Some(to); + self.from = Some(to + 1); to = std::cmp::max(to, tip); noisy_sleep(POLLING_INTERVAL_MILLIS).await; @@ -307,7 +307,7 @@ impl ProcessStep for RelayProducer { let tip_res = provider.get_block_number().await; let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; - self.from = Some(to); + self.from = Some(to + 1); to = std::cmp::max(to, tip); noisy_sleep(POLLING_INTERVAL_MILLIS).await; @@ -411,7 +411,7 @@ impl ProcessStep for ProcessProducer { let tip_res = provider.get_block_number().await; let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; - self.from = Some(to); + self.from = Some(to + 1); to = std::cmp::max(to, tip); noisy_sleep(POLLING_INTERVAL_MILLIS).await; From ea2570206323c086e3d28514fae452dffb6a2faa Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 19:16:41 +0200 Subject: [PATCH 61/80] fix: prevent overly verbose logging of terminal --- agents/monitor/src/steps/producer.rs | 2 +- agents/monitor/src/steps/terminal.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index 0208e26b..cda967f3 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -409,8 +409,8 @@ impl ProcessStep for ProcessProducer { } } let tip_res = provider.get_block_number().await; - let tip = unwrap_result_recoverable!(tip_res, self) - BEHIND_TIP; + self.from = Some(to + 1); to = std::cmp::max(to, tip); diff --git a/agents/monitor/src/steps/terminal.rs b/agents/monitor/src/steps/terminal.rs index 57cb7969..f5fafe8a 100644 --- a/agents/monitor/src/steps/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -1,7 +1,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use tracing::{debug_span, Instrument}; -use crate::{ProcessStep, Restartable}; +use crate::{steps::TaskResult, ProcessStep, Restartable}; #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] @@ -45,7 +45,10 @@ where loop { if self.rx.recv().await.is_none() { tracing::debug!(self = %self, "Upstream broke, shutting down"); - panic!(); + return TaskResult::Unrecoverable { + err: eyre::eyre!("Upstream broke, shutting down"), + worth_logging: false, + }; } } } From f9083fe20b3e971fc379a0cd131e366208f73be2 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 19:21:34 +0200 Subject: [PATCH 62/80] fix: prevent double logging recoverable bails --- agents/monitor/src/macros.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index aa2a3d86..42549815 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -7,8 +7,7 @@ macro_rules! bail_task_if { ($cond:expr, $self:ident, $err:expr) => { if $cond { let err = eyre::eyre!($err); - tracing::error!(task = %$self, err = %err, "Task failed"); - return $crate::steps::TaskResult::Recoverable{task: $self, err}; + return $crate::steps::TaskResult::Recoverable { task: $self, err }; } }; } From a0db4fd05f1f485bc846e5a0e68fc7705843d058 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 19:30:01 +0200 Subject: [PATCH 63/80] fix: prevent producer panic on tip request --- agents/monitor/src/steps/producer.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index cda967f3..bef7e9d4 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -67,7 +67,8 @@ impl ProcessStep for DispatchProducer { tokio::spawn( async move { let provider = self.home.client(); - let height = provider.get_block_number().await.unwrap(); + let height_res = provider.get_block_number().await; + let height = unwrap_result_recoverable!(height_res, self); let mut to = height - BEHIND_TIP; loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); @@ -273,7 +274,8 @@ impl ProcessStep for RelayProducer { tokio::spawn( async move { let provider = self.replica.client(); - let height = provider.get_block_number().await.unwrap(); + let height_res = provider.get_block_number().await; + let height = unwrap_result_recoverable!(height_res, self); let mut to = height - BEHIND_TIP; loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); @@ -376,7 +378,8 @@ impl ProcessStep for ProcessProducer { tokio::spawn( async move { let provider = self.replica.client(); - let height = provider.get_block_number().await.unwrap(); + let height_res = provider.get_block_number().await; + let height = unwrap_result_recoverable!(height_res, self); let mut to = height - BEHIND_TIP; loop { let from = self.from.unwrap_or(height - (BEHIND_TIP * 2)); From 27911eb3c26b27e91d5151bc62036ee78f87e5f4 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 27 Jul 2022 23:57:48 +0200 Subject: [PATCH 64/80] feature: enhanced debug logging for all tasks --- agents/monitor/src/macros.rs | 18 +++++++++++- agents/monitor/src/steps/between.rs | 4 +-- agents/monitor/src/steps/dispatch_wait.rs | 15 +++++++--- agents/monitor/src/steps/e2e.rs | 22 +++++++++++++- agents/monitor/src/steps/relay_wait.rs | 21 ++++++++++---- agents/monitor/src/steps/update_wait.rs | 35 ++++++++++++++--------- 6 files changed, 88 insertions(+), 27 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index 42549815..da1c86ac 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -15,7 +15,7 @@ macro_rules! bail_task_if { #[macro_export] macro_rules! unwrap_channel_item_unrecoverable { ($channel_item:ident, $self:ident,) => {{ - unwrap_channel_item!($channel_item, $self) + unwrap_channel_item_unrecoverable!($channel_item, $self) }}; ($channel_item:ident, $self:ident) => {{ if $channel_item.is_none() { @@ -28,6 +28,22 @@ macro_rules! unwrap_channel_item_unrecoverable { }}; } +#[macro_export] +macro_rules! unwrap_pipe_item_unrecoverable { + ($pipe_item:ident, $self:ident,) => {{ + unwrap_pipe_item_unrecoverable!($pipe_item, $self) + }}; + ($pipe_item:ident, $self:ident) => {{ + if $pipe_item.is_ok() { + tracing::debug!( + task = %$self, "inbound channel broke" + ); + return $crate::steps::TaskResult::Unrecoverable{err: eyre::eyre!("inbound pipe broke"), worth_logging: false} + } + $pipe_item.unwrap() + }}; +} + #[macro_export] macro_rules! unwrap_result_recoverable { ($result:ident, $self:ident,) => {{ diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index b8e38422..1f0d1300 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,4 +1,4 @@ -use tracing::{info_span, Instrument}; +use tracing::{debug, info_span, Instrument}; use crate::{annotate::WithMeta, pipe::Pipe, ProcessStep, Restartable}; @@ -82,7 +82,7 @@ where // get the next event from the channel let incoming = self.pipe.next().await.expect("inbound pipe failed"); - tracing::debug!( + debug!( target: "monitor::between", block_number = %incoming.meta.block_number, event = self.event.as_str(), diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 336b9c8c..85db3166 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -1,7 +1,7 @@ use ethers::prelude::U64; use prometheus::{Histogram, HistogramTimer}; use tokio::select; -use tracing::{info_span, Instrument}; +use tracing::{debug, info_span, trace, Instrument}; use crate::{ pipe::{DispatchPipe, UpdatePipe}, @@ -61,16 +61,23 @@ impl DispatchWait { fn handle_dispatch(&mut self, block_number: U64) { self.timers.push(self.metrics.timer.start_timer()); self.blocks.push(block_number); + debug!(event = "dispatch", "Starting timer for dispatch event",); } fn handle_update(&mut self, block_number: U64) { + if !self.timers.is_empty() { + debug!(count = self.timers.len(), "Ending dispatch timers") + } + // drain the entire vec - self.timers - .drain(0..) - .for_each(|timer| timer.observe_duration()); + self.timers.drain(0..).for_each(|timer| { + let elapsed = timer.stop_and_record(); + trace!(elapsed = elapsed, "ending dispatch timer"); + }); self.blocks.drain(0..).for_each(|dispatch_height| { let diff = block_number.saturating_sub(dispatch_height); self.metrics.blocks.observe(diff.as_u64() as f64); + trace!(elapsed = %diff, "ending dispatch block count"); }); } } diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 641c772b..73c156b9 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -8,7 +8,7 @@ use tokio::{ }; use ethers::prelude::H256; -use tracing::{info_span, Instrument}; +use tracing::{debug, debug_span, info_span, trace, Instrument}; use crate::{ annotate::WithMeta, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink, ProcessStep, @@ -73,6 +73,15 @@ impl E2ELatency { } fn record_dispatch(&mut self, network: String, destination: u32, message_hash: H256) { + let _span = debug_span!( + "record_dispatch", + network = network.as_str(), + destination, + message_hash = ?message_hash, + destination_network = self.domain_to_network.get(&destination).unwrap().as_str(), + ) + .entered(); + debug!("Recording dispatch"); // ignore unknown destinations if let Some(destination) = self.domain_to_network.get(&destination) { let now = Instant::now(); @@ -86,12 +95,14 @@ impl E2ELatency { .and_then(|entry| entry.remove(&message_hash)) .is_some() { + trace!(elapsed = 0.0, "dispatch preceded by process"); self.metrics .timers .get_mut(&network) .unwrap() .observe(0 as f64); } else { + trace!("Starting dispatch e2e timer"); self.dispatches .entry(network.clone()) .or_default() @@ -103,6 +114,13 @@ impl E2ELatency { } fn record_process(&mut self, network: String, replica_of: String, message_hash: H256) { + debug_span!( + "record_process", + network = network.as_str(), + replica_of = replica_of.as_str(), + message_hash = ?message_hash + ); + tracing::debug!("Recording process"); let now = Instant::now(); // if we know of a matching dispatch, mark it and remove from map @@ -112,6 +130,7 @@ impl E2ELatency { .and_then(|inner| inner.get_mut(&network)) .and_then(|inner| inner.remove(&message_hash)) { + trace!("Matching dispatch found."); let time = now.saturating_duration_since(dispatch).as_secs_f64(); self.metrics .timers @@ -119,6 +138,7 @@ impl E2ELatency { .unwrap() .observe(time); } else { + trace!("No matching dispatch found"); // record it for later self.processes .entry(replica_of) diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 1cf01622..1e61e6f6 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -4,11 +4,11 @@ use ethers::prelude::U64; use prometheus::Histogram; use tokio::time::Instant; -use tracing::{info_span, Instrument}; +use tracing::{debug, info_span, Instrument}; use crate::{ pipe::{ProcessPipe, RelayPipe}, - ProcessStep, + unwrap_pipe_item_unrecoverable, ProcessStep, }; #[derive(Debug)] @@ -83,18 +83,27 @@ impl ProcessStep for RelayWait { biased; process_next = self.process_pipe.next() => { - let process = process_next.expect("inbound process pipe failed"); + let process = unwrap_pipe_item_unrecoverable!(process_next, self); let process_instant = tokio::time::Instant::now(); let process_block = process.meta.block_number; - let elapsed_ms = process_instant.saturating_duration_since(self.relay_instant).as_secs_f64(); + let elapsed = process_instant.saturating_duration_since(self.relay_instant).as_secs_f64(); let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; - self.metrics.timers.observe(elapsed_ms); + debug!( + elapsed_blocks = elapsed_blocks, + elapsed = elapsed, + "Recording time since relay" + ); + self.metrics.timers.observe(elapsed); self.metrics.blocks.observe(elapsed_blocks); } relay_next = self.relay_pipe.next() => { - let relay = relay_next.expect("inbound relay pipe failed"); + let relay = unwrap_pipe_item_unrecoverable!(relay_next, self); + debug!( + relay_block = %relay.meta.block_number, + "Starting relay timers" + ); self.relay_instant = tokio::time::Instant::now(); self.relay_block = relay.meta.block_number; } diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 95a87d3d..27a8c814 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -2,13 +2,14 @@ use ethers::prelude::H256; use prometheus::Histogram; use std::{collections::HashMap, time::Instant}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; -use tracing::{info_span, Instrument}; +use tracing::{debug, info_span, trace, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ - annotate::WithMeta, steps::combine::CombineChannels, ProcessStep, RelayFaucet, RelaySink, - UpdateFaucet, UpdateSink, + annotate::WithMeta, send_unrecoverable, steps::combine::CombineChannels, + unwrap_channel_item_unrecoverable, ProcessStep, RelayFaucet, RelaySink, UpdateFaucet, + UpdateSink, }; #[derive(Debug)] @@ -68,7 +69,11 @@ impl std::fmt::Display for UpdateWait { impl UpdateWait { fn handle_relay(&mut self, replica_network: &str, root: H256) { let now = std::time::Instant::now(); - + debug!( + replica_network = replica_network, + root = ?root, + "Handling relay" + ); // mem optimization: don't need to store the relay time // if we observe immediately if let Some(update_time) = self.updates.get(&root) { @@ -84,6 +89,7 @@ impl UpdateWait { fn handle_update(&mut self, root: H256) { let now = std::time::Instant::now(); self.updates.insert(root, now); + debug!(root = ?root, "Starting timers for update"); // mem optimization: remove the hashmap as we no longer need to store // any future relay times for this root @@ -96,12 +102,17 @@ impl UpdateWait { } fn record(&self, replica_network: &str, relay: Instant, update: Instant) { - let v = relay.saturating_duration_since(update); + let v = relay.saturating_duration_since(update).as_secs_f64(); + trace!( + elapsed = v, + replica_network = replica_network, + "Recording relay duration" + ); self.metrics .times .get(replica_network) .expect("missing metric") - .observe(v.as_secs_f64()) + .observe(v) } } @@ -124,20 +135,18 @@ impl ProcessStep for UpdateWait { biased; update_opt = self.update_faucet.recv() => { - let update = update_opt.expect("inbound update channel broke"); + let update = unwrap_channel_item_unrecoverable!(update_opt, self); let root: H256 = update.log.new_root.into(); - self.update_sink.send(update).expect("outbound update broke"); + send_unrecoverable!(self.update_sink, update, self); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { - let (replica_network, relay) = relay_opt.expect("Inbound relays broke"); + let (replica_network, relay) = unwrap_channel_item_unrecoverable!(relay_opt, self); let root: H256 = relay.log.new_root.into(); - self.relay_sinks + send_unrecoverable!(self.relay_sinks .get(&replica_network) - .expect("missing outgoing") - .send(relay) - .unwrap_or_else(|_| panic!("outgoing relay for {} broke", &replica_network)); + .expect("missing outgoing"), relay, self); self.handle_relay(&replica_network, root); } From 81579b90746dc74ac6ec91292a4a8d34e1c85121 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 00:03:40 +0200 Subject: [PATCH 65/80] fix: prevent panics in e2e --- agents/monitor/src/steps/e2e.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 73c156b9..545f25a6 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -11,7 +11,8 @@ use ethers::prelude::H256; use tracing::{debug, debug_span, info_span, trace, Instrument}; use crate::{ - annotate::WithMeta, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink, ProcessStep, + annotate::WithMeta, send_unrecoverable, unwrap_channel_item_unrecoverable, DispatchFaucet, + DispatchSink, ProcessFaucet, ProcessSink, ProcessStep, }; use super::combine::CombineChannels; @@ -159,21 +160,23 @@ impl ProcessStep for E2ELatency { loop { tokio::select! { dispatch_opt = self.dispatch_faucet.recv() => { - let (network, dispatch) = dispatch_opt.expect("inbound dispatch broke"); + let (network, dispatch) = unwrap_channel_item_unrecoverable!(dispatch_opt, self); let message_hash: H256 = dispatch.log.message_hash.into(); let destination: u32 = (dispatch.log.destination_and_nonce >> 32) as u32; let outbound = self.dispatch_sinks.get(&network).expect("missing sink"); - outbound.send(dispatch).expect("outbound dispatch broke"); + send_unrecoverable!(outbound, dispatch, self); + self.record_dispatch(network,destination, message_hash); } process_opt = self.process_faucet.recv() => { - let (network, (replica_of, process)) = process_opt.expect("inbound process broke"); + let (network, (replica_of, process)) = unwrap_channel_item_unrecoverable!(process_opt, self); let message_hash: H256 = process.log.message_hash.into(); let outbound = self.process_sinks.get(&network).expect("missing network").get(&replica_of).expect("missing sink"); - outbound.send(process).expect("outbound dispatch broke"); + + send_unrecoverable!(outbound, process, self); self.record_process(network, replica_of, message_hash); } From a6c01b127710f48f7068d1290018b25f6b63296a Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 09:17:04 +0200 Subject: [PATCH 66/80] fix: update_wait uses pipe now --- agents/monitor/src/domain.rs | 9 +++----- agents/monitor/src/main.rs | 2 +- agents/monitor/src/steps/update_wait.rs | 28 ++++++++++++------------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index 54d0e72e..a9fe70ef 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -273,19 +273,16 @@ impl Domain { relay_faucets.insert(k.to_string(), faucet); }); - let (update_sink, update_faucet) = unbounded_channel(); - - let update_faucet = faucets.swap_update(self.name(), update_faucet); + let update_pipe = faucets.update_pipe(self.name()); let metrics = metrics.update_wait_metrics(self.name(), &other_nets, &self.home_address()); UpdateWait::new( - update_faucet, - relay_faucets, + update_pipe, self.name(), metrics, - update_sink, relay_sinks, + relay_faucets, ) .run_until_panic(); } diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 9b161ea6..9232d048 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -38,7 +38,7 @@ pub(crate) type UpdateFaucet = Faucet>; pub(crate) type RelayFaucet = Faucet>; pub(crate) type ProcessFaucet = Faucet>; pub(crate) type DispatchSink = Sink>; -pub(crate) type UpdateSink = Sink>; +// pub(crate) type UpdateSink = Sink>; pub(crate) type RelaySink = Sink>; pub(crate) type ProcessSink = Sink>; diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 27a8c814..2ab1fb04 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -7,9 +7,9 @@ use tracing::{debug, info_span, trace, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; use crate::{ - annotate::WithMeta, send_unrecoverable, steps::combine::CombineChannels, - unwrap_channel_item_unrecoverable, ProcessStep, RelayFaucet, RelaySink, UpdateFaucet, - UpdateSink, + annotate::WithMeta, pipe::UpdatePipe, send_unrecoverable, steps::combine::CombineChannels, + unwrap_channel_item_unrecoverable, unwrap_pipe_item_unrecoverable, ProcessStep, RelayFaucet, + RelaySink, }; #[derive(Debug)] @@ -20,8 +20,7 @@ pub(crate) struct UpdateWaitMetrics { #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] pub(crate) struct UpdateWait { - update_faucet: UpdateFaucet, - relay_faucets: UnboundedReceiver<(String, WithMeta)>, + update_pipe: UpdatePipe, network: String, metrics: UpdateWaitMetrics, @@ -30,31 +29,30 @@ pub(crate) struct UpdateWait { // maps root -> replica -> timer relays: HashMap>, - update_sink: UpdateSink, + relay_faucets: UnboundedReceiver<(String, WithMeta)>, relay_sinks: HashMap, } impl UpdateWait { pub(crate) fn new( - update_faucet: UpdateFaucet, - relay_faucets: HashMap, + update_pipe: UpdatePipe, network: impl AsRef, metrics: UpdateWaitMetrics, - update_sink: UpdateSink, + relay_sinks: HashMap, + relay_faucets: HashMap, ) -> Self { let (tx, rx) = unbounded_channel(); CombineChannels::new(relay_faucets, tx).run_until_panic(); Self { - update_faucet, - relay_faucets: rx, + update_pipe, network: network.as_ref().to_owned(), metrics, updates: Default::default(), relays: Default::default(), - update_sink, + relay_faucets: rx, relay_sinks, } } @@ -79,6 +77,7 @@ impl UpdateWait { if let Some(update_time) = self.updates.get(&root) { self.record(replica_network, now, *update_time) } else { + trace!("Starting timer for relay"); self.relays .entry(root) .or_default() @@ -134,10 +133,9 @@ impl ProcessStep for UpdateWait { biased; - update_opt = self.update_faucet.recv() => { - let update = unwrap_channel_item_unrecoverable!(update_opt, self); + update_opt = self.update_pipe.next() => { + let update = unwrap_pipe_item_unrecoverable!(update_opt, self); let root: H256 = update.log.new_root.into(); - send_unrecoverable!(self.update_sink, update, self); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { From 93ff757cd8d3260fb3c332a2b9c2c4e4d660e255 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 10:49:34 +0200 Subject: [PATCH 67/80] fix: e2e no longer crashes on untracked network --- agents/monitor/src/steps/e2e.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 545f25a6..f15696b6 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -74,12 +74,17 @@ impl E2ELatency { } fn record_dispatch(&mut self, network: String, destination: u32, message_hash: H256) { + if !self.domain_to_network.contains_key(&destination) { + tracing::trace!("dispatch to un-monitored network"); + return; + } + let _span = debug_span!( "record_dispatch", network = network.as_str(), destination, message_hash = ?message_hash, - destination_network = self.domain_to_network.get(&destination).unwrap().as_str(), + destination_network = self.domain_to_network.get(&destination).expect("checked").as_str(), ) .entered(); debug!("Recording dispatch"); @@ -121,7 +126,7 @@ impl E2ELatency { replica_of = replica_of.as_str(), message_hash = ?message_hash ); - tracing::debug!("Recording process"); + debug!("Recording process"); let now = Instant::now(); // if we know of a matching dispatch, mark it and remove from map From da8466e01eedfb0e3d84c0fe9d0c8633b608f11a Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 12:04:03 +0200 Subject: [PATCH 68/80] fix: new macro had inverted if condition --- agents/monitor/src/macros.rs | 4 ++-- agents/monitor/src/steps/update_wait.rs | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/agents/monitor/src/macros.rs b/agents/monitor/src/macros.rs index da1c86ac..f55b2cc6 100644 --- a/agents/monitor/src/macros.rs +++ b/agents/monitor/src/macros.rs @@ -34,9 +34,9 @@ macro_rules! unwrap_pipe_item_unrecoverable { unwrap_pipe_item_unrecoverable!($pipe_item, $self) }}; ($pipe_item:ident, $self:ident) => {{ - if $pipe_item.is_ok() { + if $pipe_item.is_err() { tracing::debug!( - task = %$self, "inbound channel broke" + task = %$self, "inbound pipe broke" ); return $crate::steps::TaskResult::Unrecoverable{err: eyre::eyre!("inbound pipe broke"), worth_logging: false} } diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 2ab1fb04..a30fd8ce 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -14,6 +14,7 @@ use crate::{ #[derive(Debug)] pub(crate) struct UpdateWaitMetrics { + // maps replica network to timing histogram pub(crate) times: HashMap, } @@ -36,6 +37,7 @@ pub(crate) struct UpdateWait { impl UpdateWait { pub(crate) fn new( update_pipe: UpdatePipe, + network: impl AsRef, metrics: UpdateWaitMetrics, @@ -60,7 +62,7 @@ impl UpdateWait { impl std::fmt::Display for UpdateWait { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "UpdateToRelay - {}", self.network) + write!(f, "UpdateToRelay - updates from {}", self.network) } } @@ -75,7 +77,7 @@ impl UpdateWait { // mem optimization: don't need to store the relay time // if we observe immediately if let Some(update_time) = self.updates.get(&root) { - self.record(replica_network, now, *update_time) + self.record(replica_network, now, *update_time); } else { trace!("Starting timer for relay"); self.relays @@ -120,9 +122,10 @@ impl ProcessStep for UpdateWait { where Self: 'static + Send + Sync + Sized, { - let span = info_span!("UpdateWait"); + let span = info_span!("UpdateWait", home_network = self.network.as_str()); tokio::spawn( async move { + trace!("Top of update_wait spawn loop"); loop { tokio::select! { // how this works: @@ -134,11 +137,13 @@ impl ProcessStep for UpdateWait { biased; update_opt = self.update_pipe.next() => { + trace!("updatewait got update"); let update = unwrap_pipe_item_unrecoverable!(update_opt, self); let root: H256 = update.log.new_root.into(); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { + trace!("updatewait got relay"); let (replica_network, relay) = unwrap_channel_item_unrecoverable!(relay_opt, self); let root: H256 = relay.log.new_root.into(); From 4ed2b28d860f7902961e6eb008e0605754005014 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 12:36:15 +0200 Subject: [PATCH 69/80] fix: register e2e metric --- agents/monitor/src/metrics.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 56c83758..00fc6306 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -190,6 +190,9 @@ impl Metrics { registry .register(Box::new(relay_to_process_blocks.clone())) .expect("unable to register metric"); + registry + .register(Box::new(e2e_timers.clone())) + .expect("unable to register metric"); Ok(Self { wallclock_times, From 0e12527fcd67b9d6ba8e0627a062fa90d5497781 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 12:36:54 +0200 Subject: [PATCH 70/80] feature: lots of nice traces for update and relay wait loops --- agents/monitor/src/steps/dispatch_wait.rs | 22 +++++++++++++--- agents/monitor/src/steps/relay_wait.rs | 31 ++++++++++++++--------- agents/monitor/src/steps/update_wait.rs | 21 ++++++++++++--- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 85db3166..d4edb875 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -5,7 +5,7 @@ use tracing::{debug, info_span, trace, Instrument}; use crate::{ pipe::{DispatchPipe, UpdatePipe}, - ProcessStep, Restartable, + unwrap_pipe_item_unrecoverable, ProcessStep, Restartable, }; #[derive(Debug)] @@ -98,6 +98,10 @@ impl ProcessStep for DispatchWait { tokio::spawn( async move { loop { + trace!( + dispatches_tracked = self.timers.len(), + "top of DispatchWait::spawn() loop" + ); // how this works: // For each dispatch, we mark its block height and start a // timer. @@ -114,13 +118,25 @@ impl ProcessStep for DispatchWait { biased; dispatch_next = self.dispatch_pipe.next() => { - let dispatch = dispatch_next.expect("inbound dispatch pipe failed"); + trace!("got dispatch pipe item"); + let dispatch = unwrap_pipe_item_unrecoverable!(dispatch_next, self); let block_number = dispatch.meta.block_number; + trace!( + block_number = %block_number, + message_hash = ?dispatch.log.message_hash, + "dispatch channel item unwrapped" + ); self.handle_dispatch(block_number); } update_next = self.update_pipe.next() => { - let update = update_next.expect("inbound update pipe failed"); + trace!("got update pipe item"); + let update = unwrap_pipe_item_unrecoverable!(update_next, self); let block_number = update.meta.block_number; + trace!( + block_number = %block_number, + new_root = ?update.log.new_root, + "update channel item unwrapped" + ); self.handle_update(block_number); } } diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index 1e61e6f6..f787df1f 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -4,7 +4,7 @@ use ethers::prelude::U64; use prometheus::Histogram; use tokio::time::Instant; -use tracing::{debug, info_span, Instrument}; +use tracing::{debug, info_span, trace, Instrument}; use crate::{ pipe::{ProcessPipe, RelayPipe}, @@ -79,24 +79,31 @@ impl ProcessStep for RelayWait { tokio::spawn( async move { loop { + trace!( + relay_tracked = !self.relay_block.is_zero(), + "top of RelayWait::spawn() loop" + ); tokio::select! { biased; process_next = self.process_pipe.next() => { let process = unwrap_pipe_item_unrecoverable!(process_next, self); - let process_instant = tokio::time::Instant::now(); - let process_block = process.meta.block_number; - let elapsed = process_instant.saturating_duration_since(self.relay_instant).as_secs_f64(); - let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; + if !self.relay_block.is_zero() { + let process_instant = tokio::time::Instant::now(); + let process_block = process.meta.block_number; - debug!( - elapsed_blocks = elapsed_blocks, - elapsed = elapsed, - "Recording time since relay" - ); - self.metrics.timers.observe(elapsed); - self.metrics.blocks.observe(elapsed_blocks); + let elapsed = process_instant.saturating_duration_since(self.relay_instant).as_secs_f64(); + let elapsed_blocks = process_block.saturating_sub(self.relay_block).as_u64() as f64; + + debug!( + elapsed_blocks = elapsed_blocks, + elapsed = elapsed, + "Recording time since relay" + ); + self.metrics.timers.observe(elapsed); + self.metrics.blocks.observe(elapsed_blocks); + } } relay_next = self.relay_pipe.next() => { let relay = unwrap_pipe_item_unrecoverable!(relay_next, self); diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index a30fd8ce..cca83bd2 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -67,6 +67,10 @@ impl std::fmt::Display for UpdateWait { } impl UpdateWait { + fn relays_tracked(&self) -> usize { + self.relays.values().map(|v| v.len()).sum() + } + fn handle_relay(&mut self, replica_network: &str, root: H256) { let now = std::time::Instant::now(); debug!( @@ -125,7 +129,11 @@ impl ProcessStep for UpdateWait { let span = info_span!("UpdateWait", home_network = self.network.as_str()); tokio::spawn( async move { - trace!("Top of update_wait spawn loop"); + trace!( + updates_tracked = self.updates.len(), + relays_tracked = self.relays_tracked(), + "Top of UpdateWait::spawn() loop" + ); loop { tokio::select! { // how this works: @@ -137,16 +145,21 @@ impl ProcessStep for UpdateWait { biased; update_opt = self.update_pipe.next() => { - trace!("updatewait got update"); + trace!("got update pipe item"); let update = unwrap_pipe_item_unrecoverable!(update_opt, self); let root: H256 = update.log.new_root.into(); + trace!(root = ?root, "update chapipennel item unwrapped"); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { - trace!("updatewait got relay"); + trace!("got relay channel item"); let (replica_network, relay) = unwrap_channel_item_unrecoverable!(relay_opt, self); let root: H256 = relay.log.new_root.into(); - + trace!( + root = ?root, + replica_network = replica_network.as_str(), + "relay channel item unwrapped" + ); send_unrecoverable!(self.relay_sinks .get(&replica_network) .expect("missing outgoing"), relay, self); From 5799565b24b553303de4d2d2666e085a0721ec3d Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 15:28:00 +0200 Subject: [PATCH 71/80] nit: minor tracing event updates --- agents/monitor/src/steps/dispatch_wait.rs | 6 +++--- agents/monitor/src/steps/e2e.rs | 9 ++++----- agents/monitor/src/steps/update_wait.rs | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index d4edb875..68e2a950 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -1,4 +1,4 @@ -use ethers::prelude::U64; +use ethers::prelude::{H256, U64}; use prometheus::{Histogram, HistogramTimer}; use tokio::select; use tracing::{debug, info_span, trace, Instrument}; @@ -123,7 +123,7 @@ impl ProcessStep for DispatchWait { let block_number = dispatch.meta.block_number; trace!( block_number = %block_number, - message_hash = ?dispatch.log.message_hash, + message_hash = ?H256::from(dispatch.log.message_hash), "dispatch channel item unwrapped" ); self.handle_dispatch(block_number); @@ -134,7 +134,7 @@ impl ProcessStep for DispatchWait { let block_number = update.meta.block_number; trace!( block_number = %block_number, - new_root = ?update.log.new_root, + new_root = ?H256::from(update.log.new_root), "update channel item unwrapped" ); self.handle_update(block_number); diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index f15696b6..0c3f3cbd 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -87,9 +87,9 @@ impl E2ELatency { destination_network = self.domain_to_network.get(&destination).expect("checked").as_str(), ) .entered(); - debug!("Recording dispatch"); - // ignore unknown destinations + // this will always pass because we check earlier if let Some(destination) = self.domain_to_network.get(&destination) { + debug!("Recording dispatch"); let now = Instant::now(); // if we know of a matching process on the appropriate destination @@ -126,7 +126,6 @@ impl E2ELatency { replica_of = replica_of.as_str(), message_hash = ?message_hash ); - debug!("Recording process"); let now = Instant::now(); // if we know of a matching dispatch, mark it and remove from map @@ -136,7 +135,7 @@ impl E2ELatency { .and_then(|inner| inner.get_mut(&network)) .and_then(|inner| inner.remove(&message_hash)) { - trace!("Matching dispatch found."); + debug!("Recording process w/ matching dispatch"); let time = now.saturating_duration_since(dispatch).as_secs_f64(); self.metrics .timers @@ -144,7 +143,7 @@ impl E2ELatency { .unwrap() .observe(time); } else { - trace!("No matching dispatch found"); + debug!("Recording process w/o matching dispatch"); // record it for later self.processes .entry(replica_of) diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index cca83bd2..ef5f970f 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -148,7 +148,7 @@ impl ProcessStep for UpdateWait { trace!("got update pipe item"); let update = unwrap_pipe_item_unrecoverable!(update_opt, self); let root: H256 = update.log.new_root.into(); - trace!(root = ?root, "update chapipennel item unwrapped"); + trace!(root = ?root, "update pipe item unwrapped"); self.handle_update(root); } relay_opt = self.relay_faucets.recv() => { From 87d6631c9712879d098368fd5342476df281daa5 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 15:46:24 +0200 Subject: [PATCH 72/80] feature: gauge for unprocessed dispatches --- agents/monitor/src/metrics.rs | 44 ++++++++++++++++++++++++--------- agents/monitor/src/steps/e2e.rs | 22 ++++++++++++++--- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 00fc6306..3066cb36 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -1,6 +1,8 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; -use prometheus::{Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec}; +use prometheus::{ + Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGaugeVec, +}; use tokio::task::JoinHandle; use warp::Filter; @@ -51,6 +53,7 @@ pub(crate) struct Metrics { relay_to_process_blocks: prometheus::HistogramVec, e2e_timers: prometheus::HistogramVec, + unprocessed_dispatches: prometheus::IntGaugeVec, } fn u16_from_env(s: impl AsRef) -> Option { @@ -67,6 +70,16 @@ fn gather() -> prometheus::Result> { impl Metrics { pub(crate) fn new() -> eyre::Result { + let unprocessed_dispatches = IntGaugeVec::new( + prometheus::core::Opts::new( + "unprocessed_messages", + "Dispatch events for which no corresponding process has been observed", + ) + .namespace(NAMESPACE) + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["home_network"], + )?; + let e2e_timers = HistogramVec::new( HistogramOpts::new( "e2e_sec", @@ -193,6 +206,9 @@ impl Metrics { registry .register(Box::new(e2e_timers.clone())) .expect("unable to register metric"); + registry + .register(Box::new(unprocessed_dispatches.clone())) + .expect("unable to register metric"); Ok(Self { wallclock_times, @@ -204,6 +220,7 @@ impl Metrics { relay_to_process_timers, relay_to_process_blocks, e2e_timers, + unprocessed_dispatches, }) } @@ -345,15 +362,20 @@ impl Metrics { } pub(crate) fn e2e_metrics<'a>(&self, networks: impl Iterator) -> E2EMetrics { - let timers = networks - .map(|network| { - ( - network.to_owned(), - self.e2e_timers.with_label_values(&[network]), - ) - }) - .collect(); + let mut gauges = HashMap::new(); + let mut timers = HashMap::new(); + + for network in networks { + timers.insert( + network.to_owned(), + self.e2e_timers.with_label_values(&[network]), + ); + gauges.insert( + network.to_owned(), + self.unprocessed_dispatches.with_label_values(&[network]), + ); + } - E2EMetrics { timers } + E2EMetrics { timers, gauges } } } diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 0c3f3cbd..d65fac75 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use nomad_ethereum::bindings::{home::DispatchFilter, replica::ProcessFilter}; -use prometheus::Histogram; +use prometheus::{Histogram, IntGauge}; use tokio::{ sync::mpsc::{unbounded_channel, UnboundedReceiver}, time::Instant, @@ -20,6 +20,8 @@ use super::combine::CombineChannels; pub(crate) struct E2EMetrics { // home to times pub(crate) timers: HashMap, + // home to gauges + pub(crate) gauges: HashMap, } #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] @@ -108,13 +110,22 @@ impl E2ELatency { .unwrap() .observe(0 as f64); } else { - trace!("Starting dispatch e2e timer"); self.dispatches .entry(network.clone()) .or_default() .entry(destination.to_owned()) .or_default() .insert(message_hash, now); + let gauge = self + .metrics + .gauges + .get_mut(&network) + .expect("missing gauge"); + gauge.inc(); + trace!( + unprocessed_dispatches = gauge.get(), + "Started dispatch e2e timer" + ); } } } @@ -135,13 +146,18 @@ impl E2ELatency { .and_then(|inner| inner.get_mut(&network)) .and_then(|inner| inner.remove(&message_hash)) { - debug!("Recording process w/ matching dispatch"); let time = now.saturating_duration_since(dispatch).as_secs_f64(); self.metrics .timers .get_mut(&replica_of) .unwrap() .observe(time); + let gauge = self.metrics.gauges.get(&replica_of).expect("missing guage"); + gauge.dec(); + debug!( + unprocessed_dispatches = gauge.get(), + "Recorded process w/ matching dispatch" + ); } else { debug!("Recording process w/o matching dispatch"); // record it for later From c5b260de9377d129ee975fed828ad6fd49bb0fba Mon Sep 17 00:00:00 2001 From: James Date: Thu, 28 Jul 2022 21:36:14 +0200 Subject: [PATCH 73/80] chore: additional comment --- agents/monitor/src/steps/e2e.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index d65fac75..c9659c73 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -152,6 +152,7 @@ impl E2ELatency { .get_mut(&replica_of) .unwrap() .observe(time); + // gauges is keyed by the home network, so we use replica_of here let gauge = self.metrics.gauges.get(&replica_of).expect("missing guage"); gauge.dec(); debug!( From e6175214cd9aaa319ffd4d57605bfdf3fbcc983d Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 10:39:07 +0200 Subject: [PATCH 74/80] refactor: associate e2e metrics with both home and remote --- agents/monitor/src/metrics.rs | 31 ++++--- agents/monitor/src/steps/e2e.rs | 141 +++++++++++++++++++------------- 2 files changed, 104 insertions(+), 68 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 3066cb36..46343114 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -77,7 +77,7 @@ impl Metrics { ) .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), - &["home_network"], + &["home_network", "replica_network"], )?; let e2e_timers = HistogramVec::new( @@ -88,7 +88,7 @@ impl Metrics { .namespace(NAMESPACE) .buckets(E2E_TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), - &["chain"], + &["home_network", "replica_network"], )?; let update_to_relay_timers = HistogramVec::new( @@ -365,15 +365,24 @@ impl Metrics { let mut gauges = HashMap::new(); let mut timers = HashMap::new(); - for network in networks { - timers.insert( - network.to_owned(), - self.e2e_timers.with_label_values(&[network]), - ); - gauges.insert( - network.to_owned(), - self.unprocessed_dispatches.with_label_values(&[network]), - ); + let networks: Vec<_> = networks.collect(); + + for home in networks.iter() { + let mut gauge = HashMap::new(); + let mut timer = HashMap::new(); + for remote in networks.iter().filter(|n| n != &home) { + gauge.insert( + remote.to_string(), + self.unprocessed_dispatches + .with_label_values(&[home, remote]), + ); + timer.insert( + remote.to_string(), + self.e2e_timers.with_label_values(&[home, remote]), + ); + } + gauges.insert(home.to_string(), gauge); + timers.insert(home.to_string(), timer); } E2EMetrics { timers, gauges } diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index c9659c73..28b1cd75 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -19,9 +19,9 @@ use super::combine::CombineChannels; pub(crate) struct E2EMetrics { // home to times - pub(crate) timers: HashMap, - // home to gauges - pub(crate) gauges: HashMap, + pub(crate) timers: HashMap>, + // home network to remote network to gauges + pub(crate) gauges: HashMap>, } #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] @@ -49,6 +49,58 @@ impl std::fmt::Display for E2ELatency { } impl E2ELatency { + fn timer(&mut self, home: &String, remote: &String) -> &mut Histogram { + self.metrics + .timers + .get_mut(home) + .expect("missing network") + .get_mut(remote) + .expect("missing histogram") + } + + fn gauge(&mut self, home: &String, remote: &String) -> &mut IntGauge { + self.metrics + .gauges + .get_mut(home) + .expect("missing network") + .get_mut(remote) + .expect("missing gauge") + } + + fn inc(&mut self, home: &String, remote: &String) -> i64 { + let gauge = &mut self.gauge(home, remote); + gauge.inc(); + gauge.get() + } + + fn dec(&mut self, home: &String, remote: &String) -> i64 { + let gauge = &mut self.gauge(home, remote); + gauge.dec(); + gauge.get() + } + + fn get_dispatch_timer( + &mut self, + home: &String, + remote: &String, + message_hash: H256, + ) -> Option { + self.dispatches + .get_mut(home) + .and_then(|inner| inner.get_mut(remote)) + .and_then(|inner| inner.remove(&message_hash)) + } + + fn start_dispatch_timer(&mut self, home: &str, remote: &str, message_hash: H256) { + let now = Instant::now(); + self.dispatches + .entry(home.to_string()) + .or_default() + .entry(remote.to_string()) + .or_default() + .insert(message_hash, now); + } + pub(crate) fn new( dispatch_faucets: HashMap, process_faucets: HashMap>, @@ -80,53 +132,37 @@ impl E2ELatency { tracing::trace!("dispatch to un-monitored network"); return; } + let destination = self + .domain_to_network + .get(&destination) + .expect("checked") + .clone(); let _span = debug_span!( "record_dispatch", network = network.as_str(), destination, message_hash = ?message_hash, - destination_network = self.domain_to_network.get(&destination).expect("checked").as_str(), ) .entered(); - // this will always pass because we check earlier - if let Some(destination) = self.domain_to_network.get(&destination) { - debug!("Recording dispatch"); - let now = Instant::now(); - - // if we know of a matching process on the appropriate destination - // mark it - // otherwise store in dispatch map - if self - .processes - .get_mut(&network) - .and_then(|entry| entry.remove(&message_hash)) - .is_some() - { - trace!(elapsed = 0.0, "dispatch preceded by process"); - self.metrics - .timers - .get_mut(&network) - .unwrap() - .observe(0 as f64); - } else { - self.dispatches - .entry(network.clone()) - .or_default() - .entry(destination.to_owned()) - .or_default() - .insert(message_hash, now); - let gauge = self - .metrics - .gauges - .get_mut(&network) - .expect("missing gauge"); - gauge.inc(); - trace!( - unprocessed_dispatches = gauge.get(), - "Started dispatch e2e timer" - ); - } + + debug!("Recording dispatch"); + // if we know of a matching process on the appropriate destination + // mark it + // otherwise store in dispatch map + if self + .processes + .get_mut(&network) + .and_then(|entry| entry.remove(&message_hash)) + .is_some() + { + trace!(elapsed = 0.0, "dispatch preceded by process"); + self.timer(&network, &destination).observe(0 as f64); + } else { + self.start_dispatch_timer(&network, &destination, message_hash); + + let unprocessed_dispatches = self.inc(&network, &destination); + trace!(unprocessed_dispatches, "Started dispatch e2e timer"); } } @@ -140,24 +176,15 @@ impl E2ELatency { let now = Instant::now(); // if we know of a matching dispatch, mark it and remove from map - if let Some(dispatch) = self - .dispatches - .get_mut(&replica_of) - .and_then(|inner| inner.get_mut(&network)) - .and_then(|inner| inner.remove(&message_hash)) - { + if let Some(dispatch) = self.get_dispatch_timer(&replica_of, &network, message_hash) { let time = now.saturating_duration_since(dispatch).as_secs_f64(); - self.metrics - .timers - .get_mut(&replica_of) - .unwrap() - .observe(time); + // gauges is keyed by the home network, so we use replica_of here - let gauge = self.metrics.gauges.get(&replica_of).expect("missing guage"); - gauge.dec(); + let unprocessed_dispatches = self.dec(&replica_of, &network); + self.timer(&replica_of, &network).observe(time); debug!( - unprocessed_dispatches = gauge.get(), - "Recorded process w/ matching dispatch" + unprocessed_dispatches, + time, "Recorded process w/ matching dispatch", ); } else { debug!("Recording process w/o matching dispatch"); From 3e59236501e07894d157cf02158904044b90da9c Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 10:41:32 +0200 Subject: [PATCH 75/80] fix: use a float literal instead of a cast --- agents/monitor/src/steps/e2e.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 28b1cd75..5b596797 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -157,7 +157,7 @@ impl E2ELatency { .is_some() { trace!(elapsed = 0.0, "dispatch preceded by process"); - self.timer(&network, &destination).observe(0 as f64); + self.timer(&network, &destination).observe(0.0); } else { self.start_dispatch_timer(&network, &destination, message_hash); From db5bfbd8b6aaa6c057df413bfce9877a6f2b724d Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 10:43:44 +0200 Subject: [PATCH 76/80] lint: missing space --- agents/monitor/src/steps/e2e.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 5b596797..33580609 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -215,7 +215,7 @@ impl ProcessStep for E2ELatency { let outbound = self.dispatch_sinks.get(&network).expect("missing sink"); send_unrecoverable!(outbound, dispatch, self); - self.record_dispatch(network,destination, message_hash); + self.record_dispatch(network, destination, message_hash); } process_opt = self.process_faucet.recv() => { let (network, (replica_of, process)) = unwrap_channel_item_unrecoverable!(process_opt, self); From d644a78833de62d4e88b18443835820adb0b953e Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 11:20:30 +0200 Subject: [PATCH 77/80] feature: dispatch queue length gauge --- agents/monitor/src/metrics.rs | 17 +++++++++++++++++ agents/monitor/src/steps/dispatch_wait.rs | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 46343114..2a3caa13 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -44,14 +44,19 @@ pub(crate) struct Metrics { event_blocks: prometheus::HistogramVec, event_counts: prometheus::IntCounterVec, + // dispatch to update dispatch_to_update_timers: prometheus::HistogramVec, dispatch_to_update_blocks: prometheus::HistogramVec, + dispatch_queue: prometheus::IntGaugeVec, + // update to relay update_to_relay_timers: prometheus::HistogramVec, + // relay to process relay_to_process_timers: prometheus::HistogramVec, relay_to_process_blocks: prometheus::HistogramVec, + // e2e e2e_timers: prometheus::HistogramVec, unprocessed_dispatches: prometheus::IntGaugeVec, } @@ -124,6 +129,13 @@ impl Metrics { &["chain", "emitter"], )?; + let dispatch_queue = IntGaugeVec::new( + prometheus::core::Opts::new("dispatches_waiting", "Dispatch events awaiting an update") + .namespace(NAMESPACE) + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "emitter"], + )?; + let relay_to_process_timers = HistogramVec::new( HistogramOpts::new( "relay_to_process_secs", @@ -194,6 +206,9 @@ impl Metrics { registry .register(Box::new(dispatch_to_update_blocks.clone())) .expect("unable to register metric"); + registry + .register(Box::new(dispatch_queue.clone())) + .expect("unable to register metric"); registry .register(Box::new(update_to_relay_timers.clone())) .expect("unable to register metric"); @@ -216,6 +231,7 @@ impl Metrics { event_counts: counts, dispatch_to_update_blocks, dispatch_to_update_timers, + dispatch_queue, update_to_relay_timers, relay_to_process_timers, relay_to_process_blocks, @@ -321,6 +337,7 @@ impl Metrics { blocks: self .dispatch_to_update_blocks .with_label_values(&[network, emitter]), + in_queue: self.dispatch_queue.with_label_values(&[network, emitter]), } } diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 68e2a950..8ce49c5b 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -1,5 +1,5 @@ use ethers::prelude::{H256, U64}; -use prometheus::{Histogram, HistogramTimer}; +use prometheus::{Histogram, HistogramTimer, IntGauge}; use tokio::select; use tracing::{debug, info_span, trace, Instrument}; @@ -12,6 +12,7 @@ use crate::{ pub(crate) struct DispatchWaitMetrics { pub(crate) timer: Histogram, pub(crate) blocks: Histogram, + pub(crate) in_queue: IntGauge, } #[derive(Debug)] @@ -61,6 +62,7 @@ impl DispatchWait { fn handle_dispatch(&mut self, block_number: U64) { self.timers.push(self.metrics.timer.start_timer()); self.blocks.push(block_number); + self.metrics.in_queue.set(self.timers.len() as i64); debug!(event = "dispatch", "Starting timer for dispatch event",); } @@ -79,6 +81,7 @@ impl DispatchWait { self.metrics.blocks.observe(diff.as_u64() as f64); trace!(elapsed = %diff, "ending dispatch block count"); }); + self.metrics.in_queue.set(0); } } From 7ea43dc883d400e2a8c66a945c5c662502c1a51f Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 12:17:56 +0200 Subject: [PATCH 78/80] feature: gauges for updates not relayed --- agents/monitor/src/metrics.rs | 31 ++++++++- agents/monitor/src/steps/update_wait.rs | 85 ++++++++++++++++++------- 2 files changed, 92 insertions(+), 24 deletions(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 2a3caa13..93cf5bf1 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -51,6 +51,7 @@ pub(crate) struct Metrics { // update to relay update_to_relay_timers: prometheus::HistogramVec, + unrelayed_updates: prometheus::IntGaugeVec, // relay to process relay_to_process_timers: prometheus::HistogramVec, @@ -104,7 +105,17 @@ impl Metrics { .namespace(NAMESPACE) .buckets(TIME_BUCKETS.to_vec()) .const_label("VERSION", env!("CARGO_PKG_VERSION")), - &["chain", "emitter", "replica_chain"], + &["chain", "emitter", "destination"], + )?; + + let unrelayed_updates = IntGaugeVec::new( + prometheus::core::Opts::new( + "unrelayed_updates", + "Update events that have not been relayed to their destination", + ) + .namespace(NAMESPACE) + .const_label("VERSION", env!("CARGO_PKG_VERSION")), + &["chain", "emitter", "destination"], )?; let dispatch_to_update_timers = HistogramVec::new( @@ -212,6 +223,9 @@ impl Metrics { registry .register(Box::new(update_to_relay_timers.clone())) .expect("unable to register metric"); + registry + .register(Box::new(unrelayed_updates.clone())) + .expect("unable to register metric"); registry .register(Box::new(relay_to_process_timers.clone())) .expect("unable to register metric"); @@ -233,6 +247,7 @@ impl Metrics { dispatch_to_update_timers, dispatch_queue, update_to_relay_timers, + unrelayed_updates, relay_to_process_timers, relay_to_process_blocks, e2e_timers, @@ -359,7 +374,19 @@ impl Metrics { }) .collect(); - UpdateWaitMetrics { times } + let unrelayed = replica_networks + .iter() + .map(|replica_network| { + let gauge = self.unrelayed_updates.with_label_values(&[ + home_network, + emitter, + replica_network, + ]); + (replica_network.to_string(), gauge) + }) + .collect(); + + UpdateWaitMetrics { times, unrelayed } } pub(crate) fn relay_wait_metrics( diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index ef5f970f..8ca829c4 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -1,5 +1,5 @@ use ethers::prelude::H256; -use prometheus::Histogram; +use prometheus::{Histogram, IntGauge}; use std::{collections::HashMap, time::Instant}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tracing::{debug, info_span, trace, Instrument}; @@ -16,6 +16,7 @@ use crate::{ pub(crate) struct UpdateWaitMetrics { // maps replica network to timing histogram pub(crate) times: HashMap, + pub(crate) unrelayed: HashMap, } #[derive(Debug)] @@ -67,20 +68,62 @@ impl std::fmt::Display for UpdateWait { } impl UpdateWait { - fn relays_tracked(&self) -> usize { - self.relays.values().map(|v| v.len()).sum() + fn incr_update_gauges(&mut self) { + self.metrics + .unrelayed + .values_mut() + .for_each(|gauge| gauge.inc()); } - fn handle_relay(&mut self, replica_network: &str, root: H256) { + fn decr_update_gauge(&self, destination: &str) { + self.metrics + .unrelayed + .get(destination) + .expect("missing gauge") + .dec(); + } + + fn start_update_timer(&mut self, root: H256) { let now = std::time::Instant::now(); + self.updates.insert(root, now); + self.incr_update_gauges(); debug!( - replica_network = replica_network, root = ?root, - "Handling relay" + pending_relays = self.pending_relays(), + "Received new update" ); + } + + fn finish_relay_timers(&mut self, root: H256) { + // mem optimization: remove the hashmap as we no longer need to store + // any future relay times for this root + if let Some(mut relays) = self.relays.remove(&root) { + // times + relays.drain().for_each(|(destination, relay)| { + self.record(&destination, relay, *self.updates.get(&root).unwrap()); + }) + } + } + + fn pending_relays(&self) -> i64 { + self.metrics + .unrelayed + .values() + .map(|gauge| gauge.get()) + .sum() + } + + fn relays_tracked(&self) -> usize { + self.relays.values().map(|v| v.len()).sum() + } + + fn handle_relay(&mut self, replica_network: &str, root: H256) { + let now = std::time::Instant::now(); + // mem optimization: don't need to store the relay time // if we observe immediately if let Some(update_time) = self.updates.get(&root) { + trace!("Relay for already-seen update"); self.record(replica_network, now, *update_time); } else { trace!("Starting timer for relay"); @@ -89,35 +132,32 @@ impl UpdateWait { .or_default() .insert(replica_network.to_owned(), now); } + debug!( + replica_network = replica_network, + root = ?root, + pending_relays = self.pending_relays(), + "Handled relay" + ); } fn handle_update(&mut self, root: H256) { - let now = std::time::Instant::now(); - self.updates.insert(root, now); - debug!(root = ?root, "Starting timers for update"); - - // mem optimization: remove the hashmap as we no longer need to store - // any future relay times for this root - if let Some(mut relays) = self.relays.remove(&root) { - // times - relays.drain().for_each(|(replica_network, relay)| { - self.record(&replica_network, relay, now); - }) - } + self.start_update_timer(root); + self.finish_relay_timers(root); } - fn record(&self, replica_network: &str, relay: Instant, update: Instant) { + fn record(&self, destination: &str, relay: Instant, update: Instant) { let v = relay.saturating_duration_since(update).as_secs_f64(); trace!( elapsed = v, - replica_network = replica_network, + destination = destination, "Recording relay duration" ); self.metrics .times - .get(replica_network) + .get(destination) .expect("missing metric") - .observe(v) + .observe(v); + self.decr_update_gauge(destination); } } @@ -130,6 +170,7 @@ impl ProcessStep for UpdateWait { tokio::spawn( async move { trace!( + pending_relays = self.pending_relays(), updates_tracked = self.updates.len(), relays_tracked = self.relays_tracked(), "Top of UpdateWait::spawn() loop" From 24347f75ae5462683febd9cfbf65bec78f0607b4 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 29 Jul 2022 12:20:35 +0200 Subject: [PATCH 79/80] chore: update help text for unrelayed updates --- agents/monitor/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/monitor/src/metrics.rs b/agents/monitor/src/metrics.rs index 93cf5bf1..1367744f 100644 --- a/agents/monitor/src/metrics.rs +++ b/agents/monitor/src/metrics.rs @@ -111,7 +111,7 @@ impl Metrics { let unrelayed_updates = IntGaugeVec::new( prometheus::core::Opts::new( "unrelayed_updates", - "Update events that have not been relayed to their destination", + "Update events that have not been relayed to their destination. Note: in rare circumstances, this gauge may have some noise, as relays for an update may have occurred before the agent booted.", ) .namespace(NAMESPACE) .const_label("VERSION", env!("CARGO_PKG_VERSION")), From ce27242460f0727c1d2c3174c4993924a3a505d2 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 10 Aug 2022 12:12:09 -0700 Subject: [PATCH 80/80] refactor: break tools out from monitor into agent-utils --- Cargo.lock | 18 ++ Cargo.toml | 1 + agent-utils/Cargo.toml | 26 +++ agent-utils/src/aliases.rs | 11 + agent-utils/src/init.rs | 44 ++++ agent-utils/src/lib.rs | 181 +++++++++++++++++ {agents/monitor => agent-utils}/src/macros.rs | 8 +- agent-utils/src/pipe.rs | 42 ++++ {agents/monitor => agent-utils}/src/utils.rs | 10 +- agents/monitor/Cargo.toml | 4 +- agents/monitor/src/domain.rs | 3 +- agents/monitor/src/faucets.rs | 6 +- agents/monitor/src/init.rs | 53 +---- agents/monitor/src/main.rs | 190 ++---------------- agents/monitor/src/pipe.rs | 51 ----- agents/monitor/src/steps/between.rs | 3 +- agents/monitor/src/steps/combine.rs | 2 +- agents/monitor/src/steps/dispatch_wait.rs | 7 +- agents/monitor/src/steps/e2e.rs | 9 +- agents/monitor/src/steps/mod.rs | 21 -- agents/monitor/src/steps/producer.rs | 8 +- agents/monitor/src/steps/relay_wait.rs | 10 +- agents/monitor/src/steps/terminal.rs | 2 +- agents/monitor/src/steps/update_wait.rs | 11 +- 24 files changed, 392 insertions(+), 329 deletions(-) create mode 100644 agent-utils/Cargo.toml create mode 100644 agent-utils/src/aliases.rs create mode 100644 agent-utils/src/init.rs create mode 100644 agent-utils/src/lib.rs rename {agents/monitor => agent-utils}/src/macros.rs (81%) create mode 100644 agent-utils/src/pipe.rs rename {agents/monitor => agent-utils}/src/utils.rs (54%) delete mode 100644 agents/monitor/src/pipe.rs diff --git a/Cargo.lock b/Cargo.lock index 2f767ca9..a664758d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,6 +63,23 @@ dependencies = [ "convert_case", ] +[[package]] +name = "agent-utils" +version = "0.1.0" +dependencies = [ + "async-trait", + "ethers", + "eyre", + "futures-util", + "nomad-core", + "nomad-xyz-configuration", + "tokio", + "tracing", + "tracing-subscriber 0.2.25", + "tracing-test", + "warp", +] + [[package]] name = "ahash" version = "0.7.6" @@ -2288,6 +2305,7 @@ dependencies = [ name = "monitor" version = "0.1.0" dependencies = [ + "agent-utils", "async-trait", "ethers", "eyre", diff --git a/Cargo.toml b/Cargo.toml index 60ed217c..21d01495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ resolver = "2" members = [ "accumulator", + "agent-utils", "nomad-types", "nomad-core", "nomad-base", diff --git a/agent-utils/Cargo.toml b/agent-utils/Cargo.toml new file mode 100644 index 00000000..510402fe --- /dev/null +++ b/agent-utils/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "agent-utils" +version = "0.1.0" +edition = "2021" +authors = ["James Prestwich "] +description = "Utils for building better agents" +repository = "https://github.com/nomad-xyz/rust" +license = "MIT OR Apache-2.0" +keywords = ["Ethereum", "Nomad"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "1.0.1", features = ["rt", "macros"] } +tracing = "0.1.35" +ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false } + +tracing-subscriber = "0.2.15" +eyre = "0.6.8" +warp = "0.3.2" +async-trait = "0.1.56" +futures-util = "0.3.21" +tracing-test = "0.2.3" + +nomad-core = { path = "../nomad-core", default-features = false } +nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../configuration" } \ No newline at end of file diff --git a/agent-utils/src/aliases.rs b/agent-utils/src/aliases.rs new file mode 100644 index 00000000..3cab398c --- /dev/null +++ b/agent-utils/src/aliases.rs @@ -0,0 +1,11 @@ +use std::collections::HashMap; + +use tokio::{sync::mpsc, task::JoinHandle}; + +pub type Restartable = JoinHandle>; + +pub type Faucet = mpsc::UnboundedReceiver; +pub type Sink = mpsc::UnboundedSender; + +pub type NetworkMap<'a, T> = HashMap<&'a str, T>; +pub type HomeReplicaMap<'a, T> = HashMap<&'a str, std::collections::HashMap<&'a str, T>>; diff --git a/agent-utils/src/init.rs b/agent-utils/src/init.rs new file mode 100644 index 00000000..8e2f020f --- /dev/null +++ b/agent-utils/src/init.rs @@ -0,0 +1,44 @@ +use nomad_xyz_configuration::{get_builtin, NomadConfig}; +use tracing::Level; +use tracing_subscriber::EnvFilter; + +pub fn config_from_file() -> Option { + std::env::var("CONFIG_PATH") + .ok() + .and_then(|path| NomadConfig::from_file(path).ok()) +} + +pub fn config_from_env() -> Option { + std::env::var("RUN_ENV") + .ok() + .and_then(|env| get_builtin(&env)) + .map(ToOwned::to_owned) +} + +pub fn config() -> eyre::Result { + config_from_file() + .or_else(config_from_env) + .ok_or_else(|| eyre::eyre!("Unable to load config from file or env")) +} + +pub fn init_tracing() { + let builder = tracing_subscriber::FmtSubscriber::builder() + .with_max_level(Level::INFO) + .with_env_filter(EnvFilter::from_default_env()) + .with_level(true); + if std::env::var("MONITOR_PRETTY").is_ok() { + builder.pretty().init() + } else { + builder.json().init() + } +} + +pub fn networks_from_env() -> Option> { + std::env::var("MONITOR_NETWORKS") + .ok() + .map(|s| s.split(',').map(ToOwned::to_owned).collect()) +} + +pub fn rpc_from_env(network: &str) -> Option { + std::env::var(format!("{}_CONNECTION_URL", network.to_uppercase())).ok() +} diff --git a/agent-utils/src/lib.rs b/agent-utils/src/lib.rs new file mode 100644 index 00000000..ad40e8d0 --- /dev/null +++ b/agent-utils/src/lib.rs @@ -0,0 +1,181 @@ +pub mod aliases; +pub mod init; +pub mod macros; +pub mod pipe; +pub mod utils; + +use std::panic; + +pub use aliases::*; + +use tokio::task::JoinHandle; + +#[derive(Debug)] +pub enum TaskResult { + Recoverable { + task: T, + err: eyre::Report, + }, + Unrecoverable { + err: eyre::Report, + worth_logging: bool, + }, +} + +pub trait ProcessStep: std::fmt::Display { + fn spawn(self) -> Restartable + where + Self: 'static + Send + Sync + Sized; + + /// Run the task until it panics. Errors result in a task restart with the + /// same channels. This means that an error causes the task to lose only + /// the data that is in-scope when it faults. + fn run_until_panic(self) -> JoinHandle<()> + where + Self: 'static + Send + Sync + Sized, + { + let task_description = format!("{}", self); + tokio::spawn(async move { + let mut handle = self.spawn(); + loop { + let result = handle.await; + + let again = match result { + Ok(TaskResult::Recoverable { task, err }) => { + tracing::warn!( + error = %err, + task = task_description.as_str(), + "Restarting task", + ); + task + } + + Ok(TaskResult::Unrecoverable { err, worth_logging }) => { + if worth_logging { + tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + } else { + tracing::trace!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); + } + break; + } + + Err(e) => { + let panic_res = e.try_into_panic(); + + if panic_res.is_err() { + tracing::trace!( + task = task_description.as_str(), + "Internal task cancelled", + ); + break; + } + let p = panic_res.unwrap(); + tracing::error!(task = task_description.as_str(), "Internal task panicked"); + panic::resume_unwind(p); + } + }; + + utils::noisy_sleep(15_000).await; + handle = again.spawn(); + } + }) + } +} + +#[cfg(test)] +mod test { + use crate::{ProcessStep, TaskResult}; + + struct RecoverableTask; + impl std::fmt::Display for RecoverableTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RecoverableTask") + } + } + + impl ProcessStep for RecoverableTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { + TaskResult::Recoverable { + task: self, + err: eyre::eyre!("This error was recoverable"), + } + }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_recovery() { + let handle = RecoverableTask.run_until_panic(); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + handle.abort(); + let result = handle.await; + + assert!(logs_contain("RecoverableTask")); + assert!(logs_contain("Restarting task")); + assert!(logs_contain("This error was recoverable")); + assert!(result.is_err() && result.unwrap_err().is_cancelled()); + } + + struct UnrecoverableTask; + impl std::fmt::Display for UnrecoverableTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "UnrecoverableTask") + } + } + + impl ProcessStep for UnrecoverableTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { + TaskResult::Unrecoverable { + err: eyre::eyre!("This error was unrecoverable"), + worth_logging: true, + } + }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_unrecoverable() { + let handle = UnrecoverableTask.run_until_panic(); + let result = handle.await; + assert!(logs_contain("UnrecoverableTask")); + assert!(logs_contain("Unrecoverable error encountered")); + assert!(logs_contain("This error was unrecoverable")); + assert!(result.is_ok()); + } + + struct PanicTask; + impl std::fmt::Display for PanicTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PanicTask") + } + } + + impl ProcessStep for PanicTask { + fn spawn(self) -> crate::Restartable + where + Self: 'static + Send + Sync + Sized, + { + tokio::spawn(async move { panic!("intentional panic :)") }) + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_panic() { + let handle = PanicTask.run_until_panic(); + let result = handle.await; + assert!(logs_contain("PanicTask")); + assert!(logs_contain("Internal task panicked")); + assert!(result.is_err() && result.unwrap_err().is_panic()); + } +} diff --git a/agents/monitor/src/macros.rs b/agent-utils/src/macros.rs similarity index 81% rename from agents/monitor/src/macros.rs rename to agent-utils/src/macros.rs index f55b2cc6..6c13ba97 100644 --- a/agents/monitor/src/macros.rs +++ b/agent-utils/src/macros.rs @@ -7,7 +7,7 @@ macro_rules! bail_task_if { ($cond:expr, $self:ident, $err:expr) => { if $cond { let err = eyre::eyre!($err); - return $crate::steps::TaskResult::Recoverable { task: $self, err }; + return $crate::TaskResult::Recoverable { task: $self, err }; } }; } @@ -22,7 +22,7 @@ macro_rules! unwrap_channel_item_unrecoverable { tracing::debug!( task = %$self, "inbound channel broke" ); - return $crate::steps::TaskResult::Unrecoverable{err: eyre::eyre!("inbound channel broke"), worth_logging: false} + return $crate::TaskResult::Unrecoverable{err: eyre::eyre!("inbound channel broke"), worth_logging: false} } $channel_item.unwrap() }}; @@ -38,7 +38,7 @@ macro_rules! unwrap_pipe_item_unrecoverable { tracing::debug!( task = %$self, "inbound pipe broke" ); - return $crate::steps::TaskResult::Unrecoverable{err: eyre::eyre!("inbound pipe broke"), worth_logging: false} + return $crate::TaskResult::Unrecoverable{err: eyre::eyre!("inbound pipe broke"), worth_logging: false} } $pipe_item.unwrap() }}; @@ -59,7 +59,7 @@ macro_rules! unwrap_result_recoverable { macro_rules! send_unrecoverable { ($tx:expr, $item:expr, $self:ident) => { if $tx.send($item).is_err() { - return $crate::steps::TaskResult::Unrecoverable { + return $crate::TaskResult::Unrecoverable { err: eyre::eyre!("outbound channel broke"), worth_logging: false, }; diff --git a/agent-utils/src/pipe.rs b/agent-utils/src/pipe.rs new file mode 100644 index 00000000..8ff42d98 --- /dev/null +++ b/agent-utils/src/pipe.rs @@ -0,0 +1,42 @@ +use std::fmt::Debug; + +use eyre::bail; + +use crate::aliases::*; + +#[derive(Debug)] +pub struct Pipe { + rx: Faucet, + tx: Sink, + contents: Option, +} + +impl Pipe +where + T: Debug + Send + Sync + 'static, +{ + pub fn new(rx: Faucet, tx: Sink, contents: Option) -> Self { + Self { rx, tx, contents } + } + + pub fn read(&self) -> Option<&T> { + self.contents.as_ref() + } + + pub fn finish(&mut self) -> eyre::Result<()> { + if let Some(contents) = self.contents.take() { + self.tx.send(contents)?; + } + Ok(()) + } + + pub async fn next(&mut self) -> eyre::Result<&T> { + self.finish()?; + + self.contents = self.rx.recv().await; + if self.contents.is_none() { + bail!("rx broke") + } + Ok(self.read().expect("checked")) + } +} diff --git a/agents/monitor/src/utils.rs b/agent-utils/src/utils.rs similarity index 54% rename from agents/monitor/src/utils.rs rename to agent-utils/src/utils.rs index 75360e99..29d30301 100644 --- a/agents/monitor/src/utils.rs +++ b/agent-utils/src/utils.rs @@ -1,9 +1,10 @@ use std::{collections::HashMap, pin::Pin}; +use ethers::prelude::rand::{rngs::ThreadRng, Rng}; use tokio::sync::mpsc::UnboundedReceiver; // polls all channels in a hashmap -pub(crate) fn nexts( +pub fn nexts( map: &mut HashMap>, ) -> Vec::Owned, Option)> + '_>>> { map.iter_mut() @@ -15,3 +16,10 @@ pub(crate) fn nexts( .map(Box::pin) .collect() } + +// adds up to a second of random delay to cause production tasks to not be synced +pub fn noisy_sleep(approx_millis: u64) -> tokio::time::Sleep { + let noise = ThreadRng::default().gen_range(0..1000u64); + let duration = std::time::Duration::from_millis(approx_millis + noise); + tokio::time::sleep(duration) +} diff --git a/agents/monitor/Cargo.toml b/agents/monitor/Cargo.toml index 72cc5df8..ed9cbd18 100644 --- a/agents/monitor/Cargo.toml +++ b/agents/monitor/Cargo.toml @@ -18,10 +18,12 @@ tracing = "0.1.35" ethers = { git = "https://github.com/gakonst/ethers-rs", branch = "master", default-features = false } nomad-core = { path = "../../nomad-core", default-features = false } +agent-utils = { path = "../../agent-utils" } nomad-xyz-configuration = { version = "0.1.0-rc.25", path = "../../configuration" } +nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } + tracing-subscriber = "0.2.15" eyre = "0.6.8" -nomad-ethereum = { version = "0.1.0", path = "../../chains/nomad-ethereum" } warp = "0.3.2" async-trait = "0.1.56" futures-util = "0.3.21" diff --git a/agents/monitor/src/domain.rs b/agents/monitor/src/domain.rs index a9fe70ef..3af4567c 100644 --- a/agents/monitor/src/domain.rs +++ b/agents/monitor/src/domain.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, sync::Arc}; +use agent_utils::ProcessStep; use nomad_ethereum::bindings::{home::Home, replica::Replica}; use nomad_xyz_configuration::{contracts::CoreContracts, NomadConfig}; use tokio::sync::mpsc::unbounded_channel; @@ -15,7 +16,7 @@ use crate::{ relay_wait::RelayWait, update_wait::UpdateWait, }, - DispatchFaucet, ProcessFaucet, ProcessStep, Provider, RelayFaucet, UpdateFaucet, + DispatchFaucet, ProcessFaucet, Provider, RelayFaucet, UpdateFaucet, }; #[derive(Debug)] diff --git a/agents/monitor/src/faucets.rs b/agents/monitor/src/faucets.rs index 97ca3e35..a5faafa5 100644 --- a/agents/monitor/src/faucets.rs +++ b/agents/monitor/src/faucets.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; +use agent_utils::{pipe::Pipe, HomeReplicaMap, NetworkMap}; use tokio::sync::mpsc::unbounded_channel; use crate::{ - pipe::{DispatchPipe, Pipe, ProcessPipe, RelayPipe, UpdatePipe}, - DispatchFaucet, DispatchSink, HomeReplicaMap, NetworkMap, ProcessFaucet, ProcessSink, - RelayFaucet, UpdateFaucet, + DispatchFaucet, DispatchPipe, DispatchSink, ProcessFaucet, ProcessPipe, ProcessSink, + RelayFaucet, RelayPipe, UpdateFaucet, UpdatePipe, }; pub(crate) struct Faucets<'a> { diff --git a/agents/monitor/src/init.rs b/agents/monitor/src/init.rs index 012e3cf7..33738261 100644 --- a/agents/monitor/src/init.rs +++ b/agents/monitor/src/init.rs @@ -5,61 +5,22 @@ use ethers::{ prelude::{Http, Provider as EthersProvider}, }; -use nomad_xyz_configuration::{get_builtin, NomadConfig}; +use nomad_xyz_configuration::NomadConfig; use tokio::task::JoinHandle; -use tracing::Level; -use tracing_subscriber::EnvFilter; + +use agent_utils::{ + init::{config, networks_from_env, rpc_from_env}, + HomeReplicaMap, ProcessStep, +}; use crate::{ domain::Domain, faucets::Faucets, metrics::Metrics, steps::{e2e::E2ELatency, terminal::Terminal}, - ArcProvider, DispatchFaucet, HomeReplicaMap, ProcessFaucet, ProcessStep, RelayFaucet, - UpdateFaucet, + ArcProvider, DispatchFaucet, ProcessFaucet, RelayFaucet, UpdateFaucet, }; -pub(crate) fn config_from_file() -> Option { - std::env::var("CONFIG_PATH") - .ok() - .and_then(|path| NomadConfig::from_file(path).ok()) -} - -pub(crate) fn config_from_env() -> Option { - std::env::var("RUN_ENV") - .ok() - .and_then(|env| get_builtin(&env)) - .map(ToOwned::to_owned) -} - -pub(crate) fn config() -> eyre::Result { - config_from_file() - .or_else(config_from_env) - .ok_or_else(|| eyre::eyre!("Unable to load config from file or env")) -} - -pub(crate) fn init_tracing() { - let builder = tracing_subscriber::FmtSubscriber::builder() - .with_max_level(Level::INFO) - .with_env_filter(EnvFilter::from_default_env()) - .with_level(true); - if std::env::var("MONITOR_PRETTY").is_ok() { - builder.pretty().init() - } else { - builder.json().init() - } -} - -pub(crate) fn networks_from_env() -> Option> { - std::env::var("MONITOR_NETWORKS") - .ok() - .map(|s| s.split(',').map(ToOwned::to_owned).collect()) -} - -pub(crate) fn rpc_from_env(network: &str) -> Option { - std::env::var(format!("{}_CONNECTION_URL", network.to_uppercase())).ok() -} - pub(crate) fn provider_for(config: &NomadConfig, network: &str) -> eyre::Result { tracing::info!(network, "Instantiating provider"); diff --git a/agents/monitor/src/main.rs b/agents/monitor/src/main.rs index 9232d048..ef4ebf27 100644 --- a/agents/monitor/src/main.rs +++ b/agents/monitor/src/main.rs @@ -1,38 +1,28 @@ -use annotate::WithMeta; +use ethers::prelude::{Http, Provider as EthersProvider}; use futures_util::future::select_all; +use std::sync::Arc; +use tracing::info_span; + use nomad_ethereum::bindings::{ home::{DispatchFilter, UpdateFilter}, replica::{ProcessFilter, UpdateFilter as RelayFilter}, }; -use std::{collections::HashMap, panic, sync::Arc, time::Duration}; -use steps::TaskResult; -use tokio::{ - sync::mpsc::{UnboundedReceiver, UnboundedSender}, - task::JoinHandle, -}; -use tracing::info_span; -use ethers::prelude::{Http, Provider as EthersProvider}; +use agent_utils::{init::init_tracing, pipe::Pipe, Faucet, Sink}; + +use annotate::WithMeta; pub(crate) mod annotate; pub(crate) mod domain; pub(crate) mod faucets; pub(crate) mod init; -pub(crate) mod macros; pub(crate) mod metrics; -pub(crate) mod pipe; pub(crate) mod steps; -pub(crate) mod utils; pub(crate) type Provider = ethers::prelude::TimeLag>; pub(crate) type ArcProvider = Arc; // pub(crate) type ProviderError = ContractError; -pub(crate) type Restartable = JoinHandle>; - -pub(crate) type Faucet = UnboundedReceiver; -pub(crate) type Sink = UnboundedSender; - pub(crate) type DispatchFaucet = Faucet>; pub(crate) type UpdateFaucet = Faucet>; pub(crate) type RelayFaucet = Faucet>; @@ -42,12 +32,14 @@ pub(crate) type DispatchSink = Sink>; pub(crate) type RelaySink = Sink>; pub(crate) type ProcessSink = Sink>; -pub(crate) type NetworkMap<'a, T> = HashMap<&'a str, T>; -pub(crate) type HomeReplicaMap<'a, T> = HashMap<&'a str, HashMap<&'a str, T>>; +pub(crate) type DispatchPipe = Pipe>; +pub(crate) type UpdatePipe = Pipe>; +pub(crate) type RelayPipe = Pipe>; +pub(crate) type ProcessPipe = Pipe>; #[tokio::main] async fn main() -> eyre::Result<()> { - init::init_tracing(); + init_tracing(); { let monitor = info_span!("MonitorBootup").in_scope(|| { let monitor = init::monitor()?; @@ -77,161 +69,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -pub(crate) trait ProcessStep: std::fmt::Display { - fn spawn(self) -> Restartable - where - Self: 'static + Send + Sync + Sized; - - /// Run the task until it panics. Errors result in a task restart with the - /// same channels. This means that an error causes the task to lose only - /// the data that is in-scope when it faults. - fn run_until_panic(self) -> JoinHandle<()> - where - Self: 'static + Send + Sync + Sized, - { - let task_description = format!("{}", self); - tokio::spawn(async move { - let mut handle = self.spawn(); - loop { - let result = handle.await; - - let again = match result { - Ok(TaskResult::Recoverable { task, err }) => { - tracing::warn!( - error = %err, - task = task_description.as_str(), - "Restarting task", - ); - task - } - - Ok(TaskResult::Unrecoverable { err, worth_logging }) => { - if worth_logging { - tracing::error!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); - } else { - tracing::trace!(err = %err, task = task_description.as_str(), "Unrecoverable error encountered"); - } - break; - } - - Err(e) => { - let panic_res = e.try_into_panic(); - - if panic_res.is_err() { - tracing::trace!( - task = task_description.as_str(), - "Internal task cancelled", - ); - break; - } - let p = panic_res.unwrap(); - tracing::error!(task = task_description.as_str(), "Internal task panicked"); - panic::resume_unwind(p); - } - }; - - tokio::time::sleep(Duration::from_secs(15)).await; - handle = again.spawn(); - } - }) - } -} - -#[cfg(test)] -mod test { - use crate::{steps::TaskResult, ProcessStep}; - - struct RecoverableTask; - impl std::fmt::Display for RecoverableTask { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "RecoverableTask") - } - } - - impl ProcessStep for RecoverableTask { - fn spawn(self) -> crate::Restartable - where - Self: 'static + Send + Sync + Sized, - { - tokio::spawn(async move { - TaskResult::Recoverable { - task: self, - err: eyre::eyre!("This error was recoverable"), - } - }) - } - } - - #[tokio::test] - #[tracing_test::traced_test] - async fn test_recovery() { - let handle = RecoverableTask.run_until_panic(); - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - handle.abort(); - let result = handle.await; - - assert!(logs_contain("RecoverableTask")); - assert!(logs_contain("Restarting task")); - assert!(logs_contain("This error was recoverable")); - assert!(result.is_err() && result.unwrap_err().is_cancelled()); - } - - struct UnrecoverableTask; - impl std::fmt::Display for UnrecoverableTask { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "UnrecoverableTask") - } - } - - impl ProcessStep for UnrecoverableTask { - fn spawn(self) -> crate::Restartable - where - Self: 'static + Send + Sync + Sized, - { - tokio::spawn(async move { - TaskResult::Unrecoverable { - err: eyre::eyre!("This error was unrecoverable"), - worth_logging: true, - } - }) - } - } - - #[tokio::test] - #[tracing_test::traced_test] - async fn test_unrecoverable() { - let handle = UnrecoverableTask.run_until_panic(); - let result = handle.await; - assert!(logs_contain("UnrecoverableTask")); - assert!(logs_contain("Unrecoverable error encountered")); - assert!(logs_contain("This error was unrecoverable")); - assert!(result.is_ok()); - } - - struct PanicTask; - impl std::fmt::Display for PanicTask { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "PanicTask") - } - } - - impl ProcessStep for PanicTask { - fn spawn(self) -> crate::Restartable - where - Self: 'static + Send + Sync + Sized, - { - tokio::spawn(async move { panic!("intentional panic :)") }) - } - } - - #[tokio::test] - #[tracing_test::traced_test] - async fn test_panic() { - let handle = PanicTask.run_until_panic(); - let result = handle.await; - assert!(logs_contain("PanicTask")); - assert!(logs_contain("Internal task panicked")); - assert!(result.is_err() && result.unwrap_err().is_panic()); - } -} diff --git a/agents/monitor/src/pipe.rs b/agents/monitor/src/pipe.rs deleted file mode 100644 index e8dc6f10..00000000 --- a/agents/monitor/src/pipe.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::fmt::Debug; - -use eyre::bail; -use nomad_ethereum::bindings::{ - home::{DispatchFilter, UpdateFilter}, - replica::{ProcessFilter, UpdateFilter as RelayFilter}, -}; - -use crate::{annotate::WithMeta, Faucet, Sink}; - -#[derive(Debug)] -pub(crate) struct Pipe { - rx: Faucet, - tx: Sink, - contents: Option, -} - -pub(crate) type DispatchPipe = Pipe>; -pub(crate) type UpdatePipe = Pipe>; -pub(crate) type RelayPipe = Pipe>; -pub(crate) type ProcessPipe = Pipe>; - -impl Pipe -where - T: Debug + Send + Sync + 'static, -{ - pub(crate) fn new(rx: Faucet, tx: Sink, contents: Option) -> Self { - Self { rx, tx, contents } - } - - pub(crate) fn read(&self) -> Option<&T> { - self.contents.as_ref() - } - - pub(crate) fn finish(&mut self) -> eyre::Result<()> { - if let Some(contents) = self.contents.take() { - self.tx.send(contents)?; - } - Ok(()) - } - - pub(crate) async fn next(&mut self) -> eyre::Result<&T> { - self.finish()?; - - self.contents = self.rx.recv().await; - if self.contents.is_none() { - bail!("rx broke") - } - Ok(self.read().expect("checked")) - } -} diff --git a/agents/monitor/src/steps/between.rs b/agents/monitor/src/steps/between.rs index 1f0d1300..21c21f96 100644 --- a/agents/monitor/src/steps/between.rs +++ b/agents/monitor/src/steps/between.rs @@ -1,6 +1,7 @@ +use agent_utils::{pipe::Pipe, ProcessStep, Restartable}; use tracing::{debug, info_span, Instrument}; -use crate::{annotate::WithMeta, pipe::Pipe, ProcessStep, Restartable}; +use crate::annotate::WithMeta; pub(crate) struct BetweenMetrics { pub(crate) count: prometheus::IntCounter, diff --git a/agents/monitor/src/steps/combine.rs b/agents/monitor/src/steps/combine.rs index 008b3e5b..140de4ab 100644 --- a/agents/monitor/src/steps/combine.rs +++ b/agents/monitor/src/steps/combine.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, fmt::Display}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tracing::{info_span, Instrument}; -use crate::{unwrap_channel_item_unrecoverable, utils::nexts, ProcessStep, Restartable}; +use agent_utils::{unwrap_channel_item_unrecoverable, utils::nexts, ProcessStep, Restartable}; #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] diff --git a/agents/monitor/src/steps/dispatch_wait.rs b/agents/monitor/src/steps/dispatch_wait.rs index 8ce49c5b..dd9b5021 100644 --- a/agents/monitor/src/steps/dispatch_wait.rs +++ b/agents/monitor/src/steps/dispatch_wait.rs @@ -3,10 +3,9 @@ use prometheus::{Histogram, HistogramTimer, IntGauge}; use tokio::select; use tracing::{debug, info_span, trace, Instrument}; -use crate::{ - pipe::{DispatchPipe, UpdatePipe}, - unwrap_pipe_item_unrecoverable, ProcessStep, Restartable, -}; +use agent_utils::{unwrap_pipe_item_unrecoverable, ProcessStep, Restartable}; + +use crate::{DispatchPipe, UpdatePipe}; #[derive(Debug)] pub(crate) struct DispatchWaitMetrics { diff --git a/agents/monitor/src/steps/e2e.rs b/agents/monitor/src/steps/e2e.rs index 33580609..55387687 100644 --- a/agents/monitor/src/steps/e2e.rs +++ b/agents/monitor/src/steps/e2e.rs @@ -10,9 +10,10 @@ use tokio::{ use ethers::prelude::H256; use tracing::{debug, debug_span, info_span, trace, Instrument}; -use crate::{ - annotate::WithMeta, send_unrecoverable, unwrap_channel_item_unrecoverable, DispatchFaucet, - DispatchSink, ProcessFaucet, ProcessSink, ProcessStep, +use crate::{annotate::WithMeta, DispatchFaucet, DispatchSink, ProcessFaucet, ProcessSink}; + +use agent_utils::{ + send_unrecoverable, unwrap_channel_item_unrecoverable, ProcessStep, Restartable, }; use super::combine::CombineChannels; @@ -198,7 +199,7 @@ impl E2ELatency { } impl ProcessStep for E2ELatency { - fn spawn(mut self) -> crate::Restartable + fn spawn(mut self) -> Restartable where Self: 'static + Send + Sync + Sized, { diff --git a/agents/monitor/src/steps/mod.rs b/agents/monitor/src/steps/mod.rs index 998541a2..44cbb238 100644 --- a/agents/monitor/src/steps/mod.rs +++ b/agents/monitor/src/steps/mod.rs @@ -1,5 +1,3 @@ -use ethers::prelude::rand::{prelude::ThreadRng, Rng}; - pub(crate) mod between; pub(crate) mod combine; pub(crate) mod dispatch_wait; @@ -8,22 +6,3 @@ pub(crate) mod producer; pub(crate) mod relay_wait; pub(crate) mod terminal; pub(crate) mod update_wait; - -#[derive(Debug)] -pub(crate) enum TaskResult { - Recoverable { - task: T, - err: eyre::Report, - }, - Unrecoverable { - err: eyre::Report, - worth_logging: bool, - }, -} - -// adds up to a second of random delay to cause production tasks to not be synced -pub(crate) fn noisy_sleep(approx_millis: u64) -> tokio::time::Sleep { - let noise = ThreadRng::default().gen_range(0..1000u64); - let duration = std::time::Duration::from_millis(approx_millis + noise); - tokio::time::sleep(duration) -} diff --git a/agents/monitor/src/steps/producer.rs b/agents/monitor/src/steps/producer.rs index bef7e9d4..6a2f73b0 100644 --- a/agents/monitor/src/steps/producer.rs +++ b/agents/monitor/src/steps/producer.rs @@ -6,11 +6,13 @@ use nomad_ethereum::bindings::{ use tokio::sync::mpsc::UnboundedSender; use tracing::{info_span, trace, Instrument}; -use crate::{ - annotate::WithMeta, bail_task_if, send_unrecoverable, steps::noisy_sleep, - unwrap_result_recoverable, DispatchSink, ProcessStep, Restartable, +use agent_utils::{ + bail_task_if, send_unrecoverable, unwrap_result_recoverable, utils::noisy_sleep, ProcessStep, + Restartable, }; +use crate::{annotate::WithMeta, DispatchSink}; + pub const POLLING_INTERVAL_MILLIS: u64 = 5000; pub const BEHIND_TIP: u64 = 5; diff --git a/agents/monitor/src/steps/relay_wait.rs b/agents/monitor/src/steps/relay_wait.rs index f787df1f..847a12ce 100644 --- a/agents/monitor/src/steps/relay_wait.rs +++ b/agents/monitor/src/steps/relay_wait.rs @@ -1,15 +1,15 @@ use std::time::Duration; +use agent_utils::Restartable; use ethers::prelude::U64; use prometheus::Histogram; use tokio::time::Instant; use tracing::{debug, info_span, trace, Instrument}; -use crate::{ - pipe::{ProcessPipe, RelayPipe}, - unwrap_pipe_item_unrecoverable, ProcessStep, -}; +use agent_utils::{unwrap_pipe_item_unrecoverable, ProcessStep}; + +use crate::{ProcessPipe, RelayPipe}; #[derive(Debug)] pub(crate) struct RelayWaitMetrics { @@ -67,7 +67,7 @@ impl std::fmt::Display for RelayWait { } impl ProcessStep for RelayWait { - fn spawn(mut self) -> crate::Restartable + fn spawn(mut self) -> Restartable where Self: 'static + Send + Sync + Sized, { diff --git a/agents/monitor/src/steps/terminal.rs b/agents/monitor/src/steps/terminal.rs index f5fafe8a..a13ad26a 100644 --- a/agents/monitor/src/steps/terminal.rs +++ b/agents/monitor/src/steps/terminal.rs @@ -1,7 +1,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use tracing::{debug_span, Instrument}; -use crate::{steps::TaskResult, ProcessStep, Restartable}; +use agent_utils::{ProcessStep, Restartable, TaskResult}; #[derive(Debug)] #[must_use = "Tasks do nothing unless you call .spawn() or .run_until_panic()"] diff --git a/agents/monitor/src/steps/update_wait.rs b/agents/monitor/src/steps/update_wait.rs index 8ca829c4..844b02bf 100644 --- a/agents/monitor/src/steps/update_wait.rs +++ b/agents/monitor/src/steps/update_wait.rs @@ -6,10 +6,13 @@ use tracing::{debug, info_span, trace, Instrument}; use nomad_ethereum::bindings::replica::UpdateFilter as RelayFilter; +use agent_utils::{ + send_unrecoverable, unwrap_channel_item_unrecoverable, unwrap_pipe_item_unrecoverable, + ProcessStep, Restartable, +}; + use crate::{ - annotate::WithMeta, pipe::UpdatePipe, send_unrecoverable, steps::combine::CombineChannels, - unwrap_channel_item_unrecoverable, unwrap_pipe_item_unrecoverable, ProcessStep, RelayFaucet, - RelaySink, + annotate::WithMeta, steps::combine::CombineChannels, RelayFaucet, RelaySink, UpdatePipe, }; #[derive(Debug)] @@ -162,7 +165,7 @@ impl UpdateWait { } impl ProcessStep for UpdateWait { - fn spawn(mut self) -> crate::Restartable + fn spawn(mut self) -> Restartable where Self: 'static + Send + Sync + Sized, {