From 1756a259a0f1296981e5aa24a80222a66b99341c Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 12 Apr 2024 09:44:31 +0300 Subject: [PATCH 1/4] fix docker demo --- .env | 2 +- builder/src/bin/permissioned-builder.rs | 28 ++++++++++++++++++------- docker/cdn-broker.Dockerfile | 2 +- docker/cdn-marshal.Dockerfile | 2 +- process-compose.yaml | 8 +++---- sequencer/src/main.rs | 19 +++++++++++++++-- sequencer/src/options.rs | 11 +++++----- 7 files changed, 49 insertions(+), 23 deletions(-) diff --git a/.env b/.env index 02ef3eb2d..0d019beb8 100644 --- a/.env +++ b/.env @@ -23,7 +23,7 @@ ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT=30s ESPRESSO_ORCHESTRATOR_MIN_TRANSACTIONS=50 ESPRESSO_ORCHESTRATOR_MIN_PROPOSE_TIME=1s ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=2s -ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal:${ESPRESSO_CDN_SERVER_PORT} +ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal-0:${ESPRESSO_CDN_SERVER_PORT} ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:${ESPRESSO_ORCHESTRATOR_PORT} ESPRESSO_SEQUENCER_API_PORT=50000 ESPRESSO_SEQUENCER1_API_PORT=50001 diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index 39e4fdd96..fafd646bd 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -14,7 +14,7 @@ use hotshot_types::traits::node_implementation::ConsensusTime; use sequencer::persistence::no_storage::NoStorage; use sequencer::{BuilderParams, L1Params, NetworkParams}; use snafu::Snafu; -use std::net::SocketAddr; +use std::net::ToSocketAddrs; use std::num::NonZeroUsize; use std::{collections::HashMap, path::PathBuf, str::FromStr, time::Duration}; use url::Url; @@ -44,24 +44,24 @@ pub struct PermissionedBuilderOptions { )] pub cdn_endpoint: String, - /// The address to bind to for Libp2p (in `IP:port` form) + /// The address to bind to for Libp2p (in `host:port` form) #[clap( short, long, env = "ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS", default_value = "0.0.0.0:1769" )] - pub libp2p_bind_address: SocketAddr, + pub libp2p_bind_address: String, /// The address we advertise to other nodes as being a Libp2p endpoint. - /// Should be supplied in `IP:port` form. + /// Should be supplied in `host:port` form. #[clap( short, long, env = "ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS", - default_value = "127.0.0.1:1769" + default_value = "localhost:1769" )] - pub libp2p_advertise_address: SocketAddr, + pub libp2p_advertise_address: String, /// URL of the Light Client State Relay Server #[clap( @@ -210,10 +210,22 @@ async fn main() -> anyhow::Result<()> { // get from the private key let builder_pub_key = BLSPubKey::from_private(&private_staking_key); + // Parse supplied Libp2p addresses to their socket form + let libp2p_advertise_address = opt + .libp2p_advertise_address + .to_socket_addrs()? + .next() + .ok_or(anyhow::anyhow!("Failed to resolve Libp2p advertise address"))?; + let libp2p_bind_address = opt + .libp2p_bind_address + .to_socket_addrs()? + .next() + .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; + let network_params = NetworkParams { cdn_endpoint: opt.cdn_endpoint, - libp2p_advertise_address: opt.libp2p_advertise_address, - libp2p_bind_address: opt.libp2p_bind_address, + libp2p_advertise_address, + libp2p_bind_address, orchestrator_url: opt.orchestrator_url, state_relay_server_url: opt.state_relay_server_url, private_staking_key: private_staking_key.clone(), diff --git a/docker/cdn-broker.Dockerfile b/docker/cdn-broker.Dockerfile index ca21d44f4..49c8b2e61 100644 --- a/docker/cdn-broker.Dockerfile +++ b/docker/cdn-broker.Dockerfile @@ -11,5 +11,5 @@ RUN chmod +x /bin/cdn-broker ENV RUST_LOG="info" -HEALTHCHECK --interval=1s --timeout=1s --retries=100 CMD curl --fail http://localhost:${ESPRESSO_CDN_SERVER_METRICS_PORT} || exit 1 +HEALTHCHECK --interval=1s --timeout=1s --retries=100 CMD curl --fail http://localhost:${ESPRESSO_CDN_SERVER_METRICS_PORT}/metrics || exit 1 ENTRYPOINT ["cdn-broker"] \ No newline at end of file diff --git a/docker/cdn-marshal.Dockerfile b/docker/cdn-marshal.Dockerfile index 6da77a418..7ba370fe9 100644 --- a/docker/cdn-marshal.Dockerfile +++ b/docker/cdn-marshal.Dockerfile @@ -11,5 +11,5 @@ RUN chmod +x /bin/cdn-marshal ENV RUST_LOG="info" -HEALTHCHECK --interval=1s --timeout=1s --retries=100 CMD curl --fail http://localhost:${ESPRESSO_CDN_SERVER_METRICS_PORT} || exit 1 +HEALTHCHECK --interval=1s --timeout=1s --retries=100 CMD curl --fail http://localhost:${ESPRESSO_CDN_SERVER_METRICS_PORT}/metrics || exit 1 ENTRYPOINT ["cdn-marshal"] \ No newline at end of file diff --git a/process-compose.yaml b/process-compose.yaml index a00c755b2..9d2d7ce8d 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -75,7 +75,7 @@ processes: environment: - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 - - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=127.0.0.1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=localhost:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 - ESPRESSO_SEQUENCER_API_PEERS=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT - ESPRESSO_SEQUENCER_STORAGE_PATH=$ESPRESSO_BASE_STORAGE_PATH/seq0 @@ -109,7 +109,7 @@ processes: environment: - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER1_API_PORT - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 - - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=127.0.0.1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=localhost:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 - ESPRESSO_SEQUENCER_API_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER2_API_PORT - ESPRESSO_SEQUENCER_POSTGRES_HOST=localhost @@ -147,7 +147,7 @@ processes: environment: - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER2_API_PORT - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2 - - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=127.0.0.1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=localhost:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2 - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER3_API_PORT - ESPRESSO_SEQUENCER_STORAGE_PATH=$ESPRESSO_BASE_STORAGE_PATH/seq2 - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_2 @@ -179,7 +179,7 @@ processes: environment: - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER3_API_PORT - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3 - - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=127.0.0.1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3 + - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=localhost:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3 - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER4_API_PORT - ESPRESSO_SEQUENCER_STORAGE_PATH=$ESPRESSO_BASE_STORAGE_PATH/seq3 - ESPRESSO_SEQUENCER_PRIVATE_STAKING_KEY=$ESPRESSO_DEMO_SEQUENCER_STAKING_PRIVATE_KEY_3 diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index 84f46faca..07da8de10 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -1,3 +1,5 @@ +use std::net::ToSocketAddrs; + use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use clap::Parser; use es_version::SEQUENCER_VERSION; @@ -55,10 +57,23 @@ where prefunded_accounts: opt.prefunded_builder_accounts, eth_account_index: opt.eth_account_index, }; + + // Parse supplied Libp2p addresses to their socket form + let libp2p_advertise_address = opt + .libp2p_advertise_address + .to_socket_addrs()? + .next() + .ok_or(anyhow::anyhow!("Failed to resolve Libp2p advertise address"))?; + let libp2p_bind_address = opt + .libp2p_bind_address + .to_socket_addrs()? + .next() + .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; + let network_params = NetworkParams { cdn_endpoint: opt.cdn_endpoint, - libp2p_advertise_address: opt.libp2p_advertise_address, - libp2p_bind_address: opt.libp2p_bind_address, + libp2p_advertise_address, + libp2p_bind_address, orchestrator_url: opt.orchestrator_url, state_relay_server_url: opt.state_relay_server_url, private_staking_key, diff --git a/sequencer/src/options.rs b/sequencer/src/options.rs index f3c5723f5..fc4abd312 100644 --- a/sequencer/src/options.rs +++ b/sequencer/src/options.rs @@ -9,7 +9,6 @@ use snafu::Snafu; use std::{ collections::{HashMap, HashSet}, iter::once, - net::SocketAddr, path::PathBuf, str::FromStr, time::Duration, @@ -60,24 +59,24 @@ pub struct Options { )] pub cdn_endpoint: String, - /// The address to bind to for Libp2p (in `IP:port` form) + /// The address to bind to for Libp2p (in `host:port` form) #[clap( short, long, env = "ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS", default_value = "0.0.0.0:1769" )] - pub libp2p_bind_address: SocketAddr, + pub libp2p_bind_address: String, /// The address we advertise to other nodes as being a Libp2p endpoint. - /// Should be supplied in `IP:port` form. + /// Should be supplied in `host:port` form. #[clap( short, long, env = "ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS", - default_value = "127.0.0.1:1769" + default_value = "localhost:1769" )] - pub libp2p_advertise_address: SocketAddr, + pub libp2p_advertise_address: String, /// URL of the Light Client State Relay Server #[clap( From d8e30ba3a414b7db8561a36c7ae804f2b738b43a Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 12 Apr 2024 10:04:53 +0300 Subject: [PATCH 2/4] format --- builder/src/bin/permissioned-builder.rs | 4 +++- sequencer/src/main.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index fafd646bd..76fb2b7a9 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -215,7 +215,9 @@ async fn main() -> anyhow::Result<()> { .libp2p_advertise_address .to_socket_addrs()? .next() - .ok_or(anyhow::anyhow!("Failed to resolve Libp2p advertise address"))?; + .ok_or(anyhow::anyhow!( + "Failed to resolve Libp2p advertise address" + ))?; let libp2p_bind_address = opt .libp2p_bind_address .to_socket_addrs()? diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index 07da8de10..01b222436 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -63,7 +63,9 @@ where .libp2p_advertise_address .to_socket_addrs()? .next() - .ok_or(anyhow::anyhow!("Failed to resolve Libp2p advertise address"))?; + .ok_or(anyhow::anyhow!( + "Failed to resolve Libp2p advertise address" + ))?; let libp2p_bind_address = opt .libp2p_bind_address .to_socket_addrs()? From 048d5124fc6bfd764dd8f66d6a4246b3ee7f60cb Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 12 Apr 2024 10:34:14 +0300 Subject: [PATCH 3/4] enforce ipv4 --- builder/src/bin/permissioned-builder.rs | 4 ++-- sequencer/src/main.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index 76fb2b7a9..acbb44add 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -214,14 +214,14 @@ async fn main() -> anyhow::Result<()> { let libp2p_advertise_address = opt .libp2p_advertise_address .to_socket_addrs()? - .next() + .find(|x| x.is_ipv4()) .ok_or(anyhow::anyhow!( "Failed to resolve Libp2p advertise address" ))?; let libp2p_bind_address = opt .libp2p_bind_address .to_socket_addrs()? - .next() + .find(|x| x.is_ipv4()) .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; let network_params = NetworkParams { diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index 01b222436..11c9a562f 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -62,14 +62,14 @@ where let libp2p_advertise_address = opt .libp2p_advertise_address .to_socket_addrs()? - .next() + .find(|x| x.is_ipv4()) .ok_or(anyhow::anyhow!( "Failed to resolve Libp2p advertise address" ))?; let libp2p_bind_address = opt .libp2p_bind_address .to_socket_addrs()? - .next() + .find(|x| x.is_ipv4()) .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; let network_params = NetworkParams { From f094eb213a33ab7f16be5632bfa1d56f4bf4d76b Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 12 Apr 2024 10:56:23 +0300 Subject: [PATCH 4/4] ipv4 comment --- builder/src/bin/permissioned-builder.rs | 2 ++ sequencer/src/main.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index acbb44add..e2fdbca90 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -211,6 +211,8 @@ async fn main() -> anyhow::Result<()> { let builder_pub_key = BLSPubKey::from_private(&private_staking_key); // Parse supplied Libp2p addresses to their socket form + // We expect all nodes to be reachable via IPv4, so we filter out any IPv6 addresses. + // Downstream in HotShot we pin the IP address to v4, but this can be fixed in the future. let libp2p_advertise_address = opt .libp2p_advertise_address .to_socket_addrs()? diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index 11c9a562f..7a2ca4210 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -59,6 +59,8 @@ where }; // Parse supplied Libp2p addresses to their socket form + // We expect all nodes to be reachable via IPv4, so we filter out any IPv6 addresses. + // Downstream in HotShot we pin the IP address to v4, but this can be fixed in the future. let libp2p_advertise_address = opt .libp2p_advertise_address .to_socket_addrs()?