Merge branch 'devnet5' into fix/initial-earliest-exit-epoch

.github/workflows/ci.yml

@@ -4,10 +4,16 @@ on: [push]
 
 jobs:
   build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
     steps:
+    - name: Disable autocrlf (Windows)
+      if: runner.os == 'Windows'
+      run: git config --global core.autocrlf false
     - uses: actions/checkout@v4
       with:
         submodules: true
@@ -19,14 +25,18 @@ jobs:
     - uses: Swatinem/rust-cache@v2
     - name: Run cargo build
       run: cargo build --release --features default-networks
-    - name: Check if code is formatted
+    - name: Check if code is formatted (Linux)
+      if: runner.os == 'Linux'
       run: cargo fmt --check
-    - name: Run Clippy
+    - name: Run Clippy (Linux)
+      if: runner.os == 'Linux'
       run: scripts/ci/clippy.bash --deny warnings
     - name: Run tests
       run: cargo test --release --no-fail-fast
-    - name: Check consensus-spec-tests coverage
+    - name: Check consensus-spec-tests coverage (Linux)
+      if: runner.os == 'Linux'
       run: scripts/ci/consensus-spec-tests-coverage.rb
     # Ignore RUSTSEC-2023-0071 because we don't use RSA in `jwt-simple` 
-    - name: Run cargo audit
+    - name: Run cargo audit (Linux)
+      if: runner.os == 'Linux'
       run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2023-0071

Cargo.toml

@@ -423,7 +423,7 @@ snap = '1'
 static_assertions = '1'
 strum = { version = '0.26', features = ['derive'] }
 syn = { version = '2', features = ['full'] }
-sysinfo = '0.31'
+sysinfo = '0.33'
 tap = '1'
 tempfile = '3'
 test-case = '3'
@@ -509,6 +509,7 @@ try_from_iterator = { path = 'try_from_iterator' }
 types = { path = 'types' }
 validator = { path = 'validator' }
 validator_key_cache = { path = 'validator_key_cache' }
+winsafe = { git = 'https://github.com/rodrigocfd/winsafe', features = ['kernel', 'psapi'] }
 
 # Banned crates
 #

ad_hoc_bench/Cargo.toml

@@ -16,7 +16,9 @@ eth2_cache_utils = { workspace = true }
 fork_choice_control = { workspace = true }
 fork_choice_store = { workspace = true }
 futures = { workspace = true }
-jemalloc-ctl = { workspace = true }
 log = { workspace = true }
 rand = { workspace = true }
 types = { workspace = true }
+
+[target.'cfg(not(windows))'.dependencies]
+jemalloc-ctl = { workspace = true }

ad_hoc_bench/src/main.rs

@@ -2,13 +2,11 @@ use core::ops::RangeInclusive;
 use std::{collections::BTreeMap, sync::Arc, time::Instant};
 
 use allocator as _;
-use anyhow::{Error, Result};
-use bytesize::ByteSize;
+use anyhow::Result;
 use clap::{Parser, ValueEnum};
 use eth2_cache_utils::{goerli, holesky, holesky_devnet, mainnet, medalla, withdrawal_devnet_4};
 use fork_choice_control::AdHocBenchController;
 use fork_choice_store::StoreConfig;
-use jemalloc_ctl::Result as JemallocResult;
 use log::info;
 use rand::seq::SliceRandom as _;
 use types::{
@@ -247,7 +245,7 @@ impl From<Blocks> for BlockParameters {
 fn main() -> Result<()> {
     binary_utils::initialize_logger(module_path!(), false)?;
     binary_utils::initialize_rayon()?;
-
+    #[cfg(not(target_os = "windows"))]
     print_jemalloc_stats()?;
 
     let options = Options::parse();
@@ -296,7 +294,7 @@ fn main() -> Result<()> {
             |_, _| BTreeMap::new(),
         ),
     }?;
-
+    #[cfg(not(target_os = "windows"))]
     print_jemalloc_stats()?;
 
     Ok(())
@@ -311,6 +309,7 @@ fn run<P: Preset>(
     beacon_blocks: impl FnOnce(RangeInclusive<Slot>, usize) -> Vec<Arc<SignedBeaconBlock<P>>>,
     blob_sidecars: impl FnOnce(RangeInclusive<Slot>, usize) -> BTreeMap<Slot, Vec<Arc<BlobSidecar<P>>>>,
 ) -> Result<()> {
+    #[cfg(not(target_os = "windows"))]
     print_jemalloc_stats()?;
 
     let Options {
@@ -421,11 +420,15 @@ fn run<P: Preset>(
     info!("average block throughput: {block_throughput:.3} blocks/s");
     info!("average slot throughput:  {slot_throughput:.3} slots/s");
 
-    print_jemalloc_stats()
+    #[cfg(not(target_os = "windows"))]
+    print_jemalloc_stats()?;
+
+    Ok(())
 }
 
+#[cfg(not(target_os = "windows"))]
 fn print_jemalloc_stats() -> Result<()> {
-    jemalloc_ctl::epoch::advance().map_err(Error::msg)?;
+    jemalloc_ctl::epoch::advance().map_err(anyhow::Error::msg)?;
 
     info!(
         "allocated: {}, \
@@ -444,9 +447,9 @@ fn print_jemalloc_stats() -> Result<()> {
 
     Ok(())
 }
-
-fn human_readable_size(result: JemallocResult<usize>) -> Result<String> {
-    let size = result.map_err(Error::msg)?;
+#[cfg(not(target_os = "windows"))]
+fn human_readable_size(result: jemalloc_ctl::Result<usize>) -> Result<String> {
+    let size = result.map_err(anyhow::Error::msg)?;
     let size = size.try_into()?;
-    Ok(ByteSize(size).to_string_as(true))
+    Ok(bytesize::ByteSize(size).to_string_as(true))
 }

grandine/src/grandine_args.rs

@@ -44,8 +44,8 @@ use reqwest::header::HeaderValue;
 use runtime::{
     MetricsConfig, StorageConfig, DEFAULT_ETH1_DB_SIZE, DEFAULT_ETH2_DB_SIZE,
     DEFAULT_LIBP2P_IPV4_PORT, DEFAULT_LIBP2P_IPV6_PORT, DEFAULT_LIBP2P_QUIC_IPV4_PORT,
-    DEFAULT_LIBP2P_QUIC_IPV6_PORT, DEFAULT_METRICS_PORT, DEFAULT_REQUEST_TIMEOUT,
-    DEFAULT_TARGET_PEERS, DEFAULT_TARGET_SUBNET_PEERS, DEFAULT_TIMEOUT,
+    DEFAULT_LIBP2P_QUIC_IPV6_PORT, DEFAULT_METRICS_PORT, DEFAULT_METRICS_UPDATE_INTERVAL_SECONDS,
+    DEFAULT_REQUEST_TIMEOUT, DEFAULT_TARGET_PEERS, DEFAULT_TARGET_SUBNET_PEERS, DEFAULT_TIMEOUT,
 };
 use serde::{de::DeserializeOwned, Serialize};
 use serde_json::Value;
@@ -343,8 +343,8 @@ struct BeaconNodeOptions {
     back_sync_enabled: bool,
 
     /// Collect Prometheus metrics
-    #[clap(long)]
-    metrics: bool,
+    #[clap(long = "metrics")]
+    metrics_enabled: bool,
 
     /// Metrics address for metrics endpoint
     #[clap(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))]
@@ -354,6 +354,10 @@ struct BeaconNodeOptions {
     #[clap(long, default_value_t = DEFAULT_METRICS_PORT)]
     metrics_port: u16,
 
+    /// Update system metrics every n seconds
+    #[clap(long, default_value_t = DEFAULT_METRICS_UPDATE_INTERVAL_SECONDS)]
+    metrics_update_interval: u64,
+
     /// Optional remote metrics URL that Grandine will periodically send metrics to
     #[clap(long)]
     remote_metrics_url: Option<RedactingUrl>,
@@ -499,7 +503,7 @@ impl NetworkConfigOptions {
         self,
         network: Network,
         network_dir: PathBuf,
-        metrics: bool,
+        metrics_enabled: bool,
         in_memory: bool,
     ) -> NetworkConfig {
         let Self {
@@ -542,7 +546,7 @@ impl NetworkConfigOptions {
         network_config.discv5_config.enr_update = !disable_enr_auto_update;
         network_config.upnp_enabled = !disable_upnp;
         network_config.network_dir = in_memory.not().then_some(network_dir);
-        network_config.metrics_enabled = metrics;
+        network_config.metrics_enabled = metrics_enabled;
         network_config.target_peers = target_peers;
         network_config.target_subnet_peers = target_subnet_peers;
         network_config.trusted_peers = trusted_peers;
@@ -900,9 +904,10 @@ impl GrandineArgs {
             jwt_secret,
             jwt_version,
             back_sync_enabled,
-            metrics,
+            metrics_enabled,
             metrics_address,
             metrics_port,
+            metrics_update_interval,
             remote_metrics_url,
             track_liveness,
             detect_doppelgangers,
@@ -949,9 +954,10 @@ impl GrandineArgs {
             warn!("both --configuration-file and --verify-configuration-file specified");
         }
 
-        if remote_metrics_url.is_some() && !metrics {
+        if remote_metrics_url.is_some() && !metrics_enabled {
             warn!(
-                "Remote metrics enabled without ---metrics. Network metrics will not be available"
+                "remote metrics enabled without ---metrics. \
+                 Network, system and process metrics will not be available"
             );
         }
 
@@ -1100,7 +1106,7 @@ impl GrandineArgs {
         );
 
         // enable global feature for easier checking
-        if metrics {
+        if metrics_enabled {
             features.push(Feature::PrometheusMetrics);
         }
 
@@ -1112,15 +1118,15 @@ impl GrandineArgs {
             || features.contains(&Feature::PrometheusMetrics)
             || features.contains(&Feature::ServeLeakyEndpoints))
         .then(|| MetricsServiceConfig {
-            remote_metrics_url,
             directories: directories.clone_arc(),
+            metrics_update_interval: Duration::from_secs(metrics_update_interval),
+            remote_metrics_url,
         });
 
-        let metrics_server_config = metrics.then_some(MetricsServerConfig {
+        let metrics_server_config = metrics_enabled.then_some(MetricsServerConfig {
             metrics_address,
             metrics_port,
             timeout: request_timeout,
-            directories: directories.clone_arc(),
         });
 
         let http_api_config = HttpApiConfig::from(http_api_options);
@@ -1147,8 +1153,7 @@ impl GrandineArgs {
             );
         }
 
-        let metrics_enabled = metrics;
-        let metrics = if metrics {
+        let metrics = if metrics_enabled {
             let metrics = Metrics::new()?;
             metrics.register_with_default_metrics()?;
             let metrics = Arc::new(metrics);

grandine/src/grandine_config.rs

@@ -117,11 +117,18 @@ impl GrandineConfig {
 
         if let Some(metrics_server_config) = &metrics_config.metrics_server_config {
             info!(
-                "Metrics server address: {}",
+                "metrics server address: {}",
                 SocketAddr::from(metrics_server_config),
             );
         }
 
+        if let Some(metrics_service_config) = &metrics_config.metrics_service_config {
+            info!(
+                "metrics service configured with {:?} update interval",
+                metrics_service_config.metrics_update_interval,
+            );
+        }
+
         if let Some(validator_api_config) = validator_api_config.as_ref() {
             info!("validator API address: {}", validator_api_config.address);
         } else {

helper_functions/src/fork.rs

@@ -549,7 +549,7 @@ pub fn upgrade_to_electra<P: Preset>(
     // initial value of `earliest_exit_epoch`
     let earliest_activation_epoch = misc::compute_activation_exit_epoch::<P>(epoch);
 
-    let earliest_exit_epoch = validators
+    for exit_epoch in validators
         .into_iter()
         .map(|validator| validator.exit_epoch)
         .filter(|exit_epoch| *exit_epoch != FAR_FUTURE_EPOCH)

http_api/Cargo.toml

@@ -28,7 +28,6 @@ itertools = { workspace = true }
 liveness_tracker = { workspace = true }
 log = { workspace = true }
 mediatype = { workspace = true }
-metrics = { workspace = true }
 mime = { workspace = true }
 operation_pools = { workspace = true }
 p2p = { workspace = true }

http_api/src/context.rs

@@ -335,7 +335,6 @@ impl<P: Preset> Context<P> {
 
         let channels = Channels {
             api_to_liveness_tx: Some(api_to_liveness_tx),
-            api_to_metrics_tx: None,
             api_to_p2p_tx,
             api_to_validator_tx,
             subnet_service_tx,

http_api/src/lib.rs

@@ -7,7 +7,6 @@ mod block_id;
 mod error;
 mod extractors;
 mod full_config;
-mod global;
 mod gui;
 mod http_api_config;
 mod middleware;