From 42ea4b17c1e2676ae1e1bbf4136b790ad62bed86 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Wed, 27 Nov 2024 16:12:52 +0100 Subject: [PATCH 01/12] feat(engine): wire StateRootTask in EngineApiTreeHandler --- crates/engine/tree/src/tree/mod.rs | 77 +++++++++++++++++++---------- crates/engine/tree/src/tree/root.rs | 1 - 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 234a96a47d07..ae45e3e70b08 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -50,6 +50,7 @@ use reth_stages_api::ControlFlow; use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInput}; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; use revm_primitives::EvmState; +use root::{StateRootConfig, StateRootMessage, StateRootTask}; use std::{ cmp::Ordering, collections::{btree_map, hash_map, BTreeMap, VecDeque}, @@ -2224,13 +2225,25 @@ where let exec_time = Instant::now(); - // TODO: create StateRootTask with the receiving end of a channel and - // pass the sending end of the channel to the state hook. - let noop_state_hook = |_state: &EvmState| {}; + let (state_root_tx, state_root_rx) = std::sync::mpsc::channel(); + + let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + + let input = self + .compute_trie_input(consistent_view.clone(), block.parent_hash) + .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; + let state_root_config = StateRootConfig { consistent_view, input: Arc::new(input) }; + let state_root_task = + StateRootTask::new(state_root_config, state_root_tx.clone(), state_root_rx); + let state_root_handle = state_root_task.spawn(); + let state_hook = move |state: &EvmState| { + let _ = state_root_tx.send(StateRootMessage::StateUpdate(state.clone())); + }; + let output = self.metrics.executor.execute_metered( executor, (&block, U256::MAX).into(), - Box::new(noop_state_hook), + Box::new(state_hook), )?; trace!(target: "engine::tree", elapsed=?exec_time.elapsed(), ?block_number, "Executed block"); @@ -2255,8 +2268,6 @@ where let root_time = Instant::now(); let mut state_root_result = None; - // TODO: switch to calculate state root using `StateRootTask`. - // We attempt to compute state root in parallel if we are currently not persisting anything // to database. This is safe, because the database state cannot change until we // finish parallel computation. It is important that nothing is being persisted as @@ -2264,9 +2275,14 @@ where // per thread and it might end up with a different view of the database. let persistence_in_progress = self.persistence_state.in_progress(); if !persistence_in_progress { - state_root_result = match self - .compute_state_root_parallel(block.header().parent_hash(), &hashed_state) - { + let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + let mut input = self + .compute_trie_input(consistent_view.clone(), block.parent_hash) + .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; + // Extend with block we are validating root for. + input.append_ref(&hashed_state); + + state_root_result = match self.compute_state_root_parallel(consistent_view, input) { Ok((state_root, trie_output)) => Some((state_root, trie_output)), Err(ParallelStateRootError::Provider(ProviderError::ConsistentView(error))) => { debug!(target: "engine", %error, "Parallel state root computation failed consistency check, falling back"); @@ -2277,6 +2293,14 @@ where } let (state_root, trie_output) = if let Some(result) = state_root_result { + match state_root_handle.wait_for_result() { + Ok(state_root_task_result) => { + info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_result=?state_root_task_result.0, regular_state_root_result = ?result.0); + } + Err(e) => { + info!(target: "engine::tree", error=?e, "on state root task wait_for_result") + } + } result } else { debug!(target: "engine::tree", block=?sealed_block.num_hash(), persistence_in_progress, "Failed to compute state root in parallel"); @@ -2331,23 +2355,11 @@ where Ok(InsertPayloadOk2::Inserted(BlockStatus2::Valid)) } - /// Compute state root for the given hashed post state in parallel. - /// - /// # Returns - /// - /// Returns `Ok(_)` if computed successfully. - /// Returns `Err(_)` if error was encountered during computation. - /// `Err(ProviderError::ConsistentView(_))` can be safely ignored and fallback computation - /// should be used instead. - fn compute_state_root_parallel( + fn compute_trie_input( &self, + consistent_view: ConsistentDbView

, parent_hash: B256, - hashed_state: &HashedPostState, - ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { - // TODO: when we switch to calculate state root using `StateRootTask` this - // method can be still useful to calculate the required `TrieInput` to - // create the task. - let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + ) -> Result { let mut input = TrieInput::default(); if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(parent_hash) { @@ -2367,9 +2379,22 @@ where input.append(revert_state); } - // Extend with block we are validating root for. - input.append_ref(hashed_state); + Ok(input) + } + /// Compute state root for the given hashed post state in parallel. + /// + /// # Returns + /// + /// Returns `Ok(_)` if computed successfully. + /// Returns `Err(_)` if error was encountered during computation. + /// `Err(ProviderError::ConsistentView(_))` can be safely ignored and fallback computation + /// should be used instead. + fn compute_state_root_parallel( + &self, + consistent_view: ConsistentDbView

, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { ParallelStateRoot::new(consistent_view, input).incremental_root_with_updates() } diff --git a/crates/engine/tree/src/tree/root.rs b/crates/engine/tree/src/tree/root.rs index f9d16e0fe400..230c3dccbdbc 100644 --- a/crates/engine/tree/src/tree/root.rs +++ b/crates/engine/tree/src/tree/root.rs @@ -46,7 +46,6 @@ pub struct StateRootHandle { rx: mpsc::Receiver, } -#[allow(dead_code)] impl StateRootHandle { /// Creates a new handle from a receiver. pub(crate) const fn new(rx: mpsc::Receiver) -> Self { From c10577b7a410c02f293ed46fd580342ce4322814 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 29 Nov 2024 10:00:38 +0100 Subject: [PATCH 02/12] show TrieUpdates for task and regular case --- crates/engine/tree/src/tree/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index ae45e3e70b08..0787bf3eb4e1 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2296,6 +2296,7 @@ where match state_root_handle.wait_for_result() { Ok(state_root_task_result) => { info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_result=?state_root_task_result.0, regular_state_root_result = ?result.0); + info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_trie_updates=?state_root_task_result.1, regular_state_root_trie_updates = ?result.1); } Err(e) => { info!(target: "engine::tree", error=?e, "on state root task wait_for_result") From 9921542b2f472fe47092b524392329fe1d6f81e0 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 29 Nov 2024 13:29:58 +0100 Subject: [PATCH 03/12] show storage trie updates diff --- crates/engine/tree/src/tree/mod.rs | 153 ++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 0787bf3eb4e1..dec083f851cd 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -47,7 +47,10 @@ use reth_provider::{ }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; -use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInput}; +use reth_trie::{ + updates::{StorageTrieUpdates, TrieUpdates}, + HashedPostState, Nibbles, TrieInput, +}; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; use revm_primitives::EvmState; use root::{StateRootConfig, StateRootMessage, StateRootTask}; @@ -2296,7 +2299,17 @@ where match state_root_handle.wait_for_result() { Ok(state_root_task_result) => { info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_result=?state_root_task_result.0, regular_state_root_result = ?result.0); - info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_trie_updates=?state_root_task_result.1, regular_state_root_trie_updates = ?result.1); + let diff = compare_trie_updates(&state_root_task_result.1, &result.1); + if diff.has_differences() { + info!(target: "engine::tree", + block=?sealed_block.num_hash(), + storage_nodes_only_in_first= ?diff.storage_nodes_only_in_first, + storage_nodes_only_in_second= ?diff.storage_nodes_only_in_second, + storage_nodes_with_differences= ?diff.storage_nodes_with_differences, + "Found differences in TrieUpdates"); + } else { + debug!(target: "engine::tree", block=?sealed_block.num_hash(), "TrieUpdates match exactly"); + } } Err(e) => { info!(target: "engine::tree", error=?e, "on state root task wait_for_result") @@ -2656,6 +2669,142 @@ pub enum AdvancePersistenceError { Provider(#[from] ProviderError), } +#[derive(Debug, Default)] +struct TrieUpdatesDiff { + pub account_nodes_only_in_first: HashSet, + pub account_nodes_only_in_second: HashSet, + pub account_nodes_with_different_values: HashSet, + pub removed_nodes_only_in_first: HashSet, + pub removed_nodes_only_in_second: HashSet, + pub storage_tries_only_in_first: HashSet, + pub storage_tries_only_in_second: HashSet, + pub storage_tries_with_differences: HashMap, +} + +#[derive(Debug, Default)] +struct StorageTrieUpdatesDiff { + pub is_deleted_differs: bool, + pub storage_nodes_only_in_first: HashSet, + pub storage_nodes_only_in_second: HashSet, + pub storage_nodes_with_different_values: HashSet, + pub removed_nodes_only_in_first: HashSet, + pub removed_nodes_only_in_second: HashSet, +} + +fn compare_trie_updates(first: &TrieUpdates, second: &TrieUpdates) -> TrieUpdatesDiff { + let mut diff = TrieUpdatesDiff::default(); + + // compare account nodes + for key in first.account_nodes.keys() { + if !second.account_nodes.contains_key(key) { + diff.account_nodes_only_in_first.insert(key.clone()); + } else if first.account_nodes.get(key) != second.account_nodes.get(key) { + diff.account_nodes_with_different_values.insert(key.clone()); + } + } + for key in second.account_nodes.keys() { + if !first.account_nodes.contains_key(key) { + diff.account_nodes_only_in_second.insert(key.clone()); + } + } + + // compare removed nodes + for node in &first.removed_nodes { + if !second.removed_nodes.contains(node) { + diff.removed_nodes_only_in_first.insert(node.clone()); + } + } + for node in &second.removed_nodes { + if !first.removed_nodes.contains(node) { + diff.removed_nodes_only_in_second.insert(node.clone()); + } + } + + // compare storage tries + for key in first.storage_tries.keys() { + if second.storage_tries.contains_key(key) { + let storage_diff = compare_storage_trie_updates( + first.storage_tries.get(key).unwrap(), + second.storage_tries.get(key).unwrap(), + ); + if storage_diff.has_differences() { + diff.storage_tries_with_differences.insert(*key, storage_diff); + } + } else { + diff.storage_tries_only_in_first.insert(*key); + } + } + for key in second.storage_tries.keys() { + if !first.storage_tries.contains_key(key) { + diff.storage_tries_only_in_second.insert(*key); + } + } + + diff +} + +fn compare_storage_trie_updates( + first: &StorageTrieUpdates, + second: &StorageTrieUpdates, +) -> StorageTrieUpdatesDiff { + let mut diff = StorageTrieUpdatesDiff { + is_deleted_differs: first.is_deleted != second.is_deleted, + ..Default::default() + }; + + // compare storage nodes + for key in first.storage_nodes.keys() { + if !second.storage_nodes.contains_key(key) { + diff.storage_nodes_only_in_first.insert(key.clone()); + } else if first.storage_nodes.get(key) != second.storage_nodes.get(key) { + diff.storage_nodes_with_different_values.insert(key.clone()); + } + } + for key in second.storage_nodes.keys() { + if !first.storage_nodes.contains_key(key) { + diff.storage_nodes_only_in_second.insert(key.clone()); + } + } + + // compare removed nodes + for node in &first.removed_nodes { + if !second.removed_nodes.contains(node) { + diff.removed_nodes_only_in_first.insert(node.clone()); + } + } + for node in &second.removed_nodes { + if !first.removed_nodes.contains(node) { + diff.removed_nodes_only_in_second.insert(node.clone()); + } + } + + diff +} + +impl StorageTrieUpdatesDiff { + fn has_differences(&self) -> bool { + self.is_deleted_differs || + !self.storage_nodes_only_in_first.is_empty() || + !self.storage_nodes_only_in_second.is_empty() || + !self.storage_nodes_with_different_values.is_empty() || + !self.removed_nodes_only_in_first.is_empty() || + !self.removed_nodes_only_in_second.is_empty() + } +} + +impl TrieUpdatesDiff { + fn has_differences(&self) -> bool { + !self.account_nodes_only_in_first.is_empty() || + !self.account_nodes_only_in_second.is_empty() || + !self.account_nodes_with_different_values.is_empty() || + !self.removed_nodes_only_in_first.is_empty() || + !self.removed_nodes_only_in_second.is_empty() || + !self.storage_tries_only_in_first.is_empty() || + !self.storage_tries_only_in_second.is_empty() || + !self.storage_tries_with_differences.is_empty() + } +} + #[cfg(test)] mod tests { use super::*; From af16dc0d62e70dc87ab150833c6fa6381a55e502 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 29 Nov 2024 13:31:54 +0100 Subject: [PATCH 04/12] fix variable names --- crates/engine/tree/src/tree/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index dec083f851cd..29b764689187 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2303,9 +2303,9 @@ where if diff.has_differences() { info!(target: "engine::tree", block=?sealed_block.num_hash(), - storage_nodes_only_in_first= ?diff.storage_nodes_only_in_first, - storage_nodes_only_in_second= ?diff.storage_nodes_only_in_second, - storage_nodes_with_differences= ?diff.storage_nodes_with_differences, + storage_tries_only_in_first= ?diff.storage_tries_only_in_first, + storage_tries_only_in_second= ?diff.storage_tries_only_in_second, + storage_tries_with_differences= ?diff.storage_tries_with_differences, "Found differences in TrieUpdates"); } else { debug!(target: "engine::tree", block=?sealed_block.num_hash(), "TrieUpdates match exactly"); From 33f768018e29cf62d68fd4b53bb926d3c481fad9 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 3 Dec 2024 16:59:42 +0100 Subject: [PATCH 05/12] show account node and storage trie update one diff per line --- crates/engine/tree/src/tree/mod.rs | 36 ++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 29b764689187..20cc90ae21ec 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2299,14 +2299,36 @@ where match state_root_handle.wait_for_result() { Ok(state_root_task_result) => { info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_result=?state_root_task_result.0, regular_state_root_result = ?result.0); - let diff = compare_trie_updates(&state_root_task_result.1, &result.1); + let task_trie_updates = &state_root_task_result.1; + let regular_trie_updates = &result.1; + let diff = compare_trie_updates(task_trie_updates, regular_trie_updates); if diff.has_differences() { - info!(target: "engine::tree", - block=?sealed_block.num_hash(), - storage_tries_only_in_first= ?diff.storage_tries_only_in_first, - storage_tries_only_in_second= ?diff.storage_tries_only_in_second, - storage_tries_with_differences= ?diff.storage_tries_with_differences, - "Found differences in TrieUpdates"); + for path in diff.account_nodes_with_different_values { + let task_entry = task_trie_updates.account_nodes.get(&path); + let regular_entry = regular_trie_updates.account_nodes.get(&path); + if task_entry != regular_entry { + debug!(target: "engine::tree", ?path, ?task_entry, ?regular_entry, "Difference in account node updates"); + } + } + for address in diff.storage_tries_with_differences.keys() { + let task = task_trie_updates.storage_tries.get(address); + let regular = regular_trie_updates.storage_tries.get(address); + for path in task + .map_or_else(HashSet::new, |tries| { + tries.storage_nodes.keys().collect() + }) + .union(®ular.map_or_else(HashSet::new, |tries| { + tries.storage_nodes.keys().collect() + })) + { + let task_entry = task.map(|tries| tries.storage_nodes.get(*path)); + let regular_entry = + regular.map(|tries| tries.storage_nodes.get(*path)); + if task_entry != regular_entry { + debug!(target: "engine::tree", ?address, ?path, ?task_entry, ?regular_entry, "Difference in storage trie updates"); + } + } + } } else { debug!(target: "engine::tree", block=?sealed_block.num_hash(), "TrieUpdates match exactly"); } From 1da09f03f337558a72d5df92a46139551d9c060b Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 6 Dec 2024 19:26:19 +0100 Subject: [PATCH 06/12] use StateHookSender --- crates/engine/tree/src/tree/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 20cc90ae21ec..ae64e5043fc3 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -49,11 +49,11 @@ use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; use reth_trie::{ updates::{StorageTrieUpdates, TrieUpdates}, - HashedPostState, Nibbles, TrieInput, + Nibbles, TrieInput, }; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; use revm_primitives::EvmState; -use root::{StateRootConfig, StateRootMessage, StateRootTask}; +use root::{StateHookSender, StateRootConfig, StateRootMessage, StateRootTask}; use std::{ cmp::Ordering, collections::{btree_map, hash_map, BTreeMap, VecDeque}, @@ -2239,8 +2239,9 @@ where let state_root_task = StateRootTask::new(state_root_config, state_root_tx.clone(), state_root_rx); let state_root_handle = state_root_task.spawn(); + let state_hook_sender = StateHookSender::new(state_root_tx); let state_hook = move |state: &EvmState| { - let _ = state_root_tx.send(StateRootMessage::StateUpdate(state.clone())); + let _ = state_hook_sender.send(StateRootMessage::StateUpdate(state.clone())); }; let output = self.metrics.executor.execute_metered( @@ -2845,7 +2846,7 @@ mod tests { use reth_primitives::{Block, BlockExt, EthPrimitives}; use reth_provider::test_utils::MockEthProvider; use reth_rpc_types_compat::engine::{block_to_payload_v1, payload::block_to_payload_v3}; - use reth_trie::updates::TrieUpdates; + use reth_trie::{updates::TrieUpdates, HashedPostState}; use std::{ str::FromStr, sync::mpsc::{channel, Sender}, From 4246a342e2f16295b7c144e76c9ff43e52291f6f Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Mon, 9 Dec 2024 12:32:01 +0100 Subject: [PATCH 07/12] adapt to new StateRootTask constructor --- crates/engine/tree/src/tree/mod.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index ae64e5043fc3..1b27ab78d316 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -52,8 +52,7 @@ use reth_trie::{ Nibbles, TrieInput, }; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; -use revm_primitives::EvmState; -use root::{StateHookSender, StateRootConfig, StateRootMessage, StateRootTask}; +use root::{StateRootConfig, StateRootTask}; use std::{ cmp::Ordering, collections::{btree_map, hash_map, BTreeMap, VecDeque}, @@ -2228,21 +2227,15 @@ where let exec_time = Instant::now(); - let (state_root_tx, state_root_rx) = std::sync::mpsc::channel(); - let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; let input = self .compute_trie_input(consistent_view.clone(), block.parent_hash) .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; let state_root_config = StateRootConfig { consistent_view, input: Arc::new(input) }; - let state_root_task = - StateRootTask::new(state_root_config, state_root_tx.clone(), state_root_rx); + let state_root_task = StateRootTask::new(state_root_config); + let state_hook = state_root_task.state_hook(); let state_root_handle = state_root_task.spawn(); - let state_hook_sender = StateHookSender::new(state_root_tx); - let state_hook = move |state: &EvmState| { - let _ = state_hook_sender.send(StateRootMessage::StateUpdate(state.clone())); - }; let output = self.metrics.executor.execute_metered( executor, From 5c332da0b123309aeb00f5e88513eb3de251d07b Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Wed, 11 Dec 2024 16:31:02 +0100 Subject: [PATCH 08/12] fix parent_hash --- crates/engine/tree/src/tree/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 1b27ab78d316..8b2c97fc14ef 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2230,7 +2230,7 @@ where let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; let input = self - .compute_trie_input(consistent_view.clone(), block.parent_hash) + .compute_trie_input(consistent_view.clone(), block.header().parent_hash()) .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; let state_root_config = StateRootConfig { consistent_view, input: Arc::new(input) }; let state_root_task = StateRootTask::new(state_root_config); @@ -2274,7 +2274,7 @@ where if !persistence_in_progress { let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; let mut input = self - .compute_trie_input(consistent_view.clone(), block.parent_hash) + .compute_trie_input(consistent_view.clone(), block.header().parent_hash()) .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; // Extend with block we are validating root for. input.append_ref(&hashed_state); From 3746dc98ee6b1a5bdd45034095b63ade6380a7bc Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 13 Dec 2024 11:31:19 +0100 Subject: [PATCH 09/12] update to use std::thread::scope --- crates/engine/tree/src/tree/mod.rs | 270 +++++++++++++++++------------ 1 file changed, 163 insertions(+), 107 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 8b2c97fc14ef..c315250c260f 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -6,7 +6,7 @@ use crate::{ tree::metrics::EngineApiMetrics, }; use alloy_consensus::BlockHeader; -use alloy_eips::BlockNumHash; +use alloy_eips::{eip7685::Requests, BlockNumHash}; use alloy_primitives::{ map::{HashMap, HashSet}, BlockNumber, B256, U256, @@ -31,7 +31,10 @@ use reth_engine_primitives::{ EngineValidator, ForkchoiceStateTracker, OnForkChoiceUpdated, }; use reth_errors::{ConsensusError, ProviderResult}; -use reth_evm::execute::BlockExecutorProvider; +use reth_evm::{ + execute::{BlockExecutionOutput, BlockExecutorProvider}, + system_calls::OnStateHook, +}; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_builder_primitives::PayloadBuilder; use reth_payload_primitives::PayloadBuilderAttributes; @@ -41,17 +44,23 @@ use reth_primitives::{ }; use reth_primitives_traits::Block; use reth_provider::{ - providers::ConsistentDbView, BlockReader, DatabaseProviderFactory, ExecutionOutcome, - HashedPostStateProvider, ProviderError, StateCommitmentProvider, StateProviderBox, - StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, + providers::ConsistentDbView, BlockReader, DBProvider, DatabaseProviderFactory, + ExecutionOutcome, HashedPostStateProvider, ProviderError, StateCommitmentProvider, + StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, }; -use reth_revm::database::StateProviderDatabase; +use reth_revm::{database::StateProviderDatabase, db::BundleState}; use reth_stages_api::ControlFlow; use reth_trie::{ - updates::{StorageTrieUpdates, TrieUpdates}, - Nibbles, TrieInput, + hashed_cursor::HashedPostStateCursorFactory, + prefix_set::TriePrefixSetsMut, + proof::ProofBlindedProviderFactory, + trie_cursor::InMemoryTrieCursorFactory, + updates::{StorageTrieUpdates, TrieUpdates, TrieUpdatesSorted}, + HashedPostState, HashedPostStateSorted, Nibbles, TrieInput, }; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; +use revm_primitives::EvmState; use root::{StateRootConfig, StateRootTask}; use std::{ cmp::Ordering, @@ -81,6 +90,13 @@ pub use reth_engine_primitives::InvalidBlockHook; pub mod root; +struct StateHookContext

{ + provider_ro: P, + nodes_sorted: TrieUpdatesSorted, + state_sorted: HashedPostStateSorted, + prefix_sets: Arc, +} + /// Keeps track of the state of the tree. /// /// ## Invariants @@ -2227,114 +2243,148 @@ where let exec_time = Instant::now(); - let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + let persistence_not_in_progress = !self.persistence_state.in_progress(); - let input = self - .compute_trie_input(consistent_view.clone(), block.header().parent_hash()) - .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; - let state_root_config = StateRootConfig { consistent_view, input: Arc::new(input) }; - let state_root_task = StateRootTask::new(state_root_config); - let state_hook = state_root_task.state_hook(); - let state_root_handle = state_root_task.spawn(); - - let output = self.metrics.executor.execute_metered( - executor, - (&block, U256::MAX).into(), - Box::new(state_hook), - )?; - - trace!(target: "engine::tree", elapsed=?exec_time.elapsed(), ?block_number, "Executed block"); - - if let Err(err) = self.consensus.validate_block_post_execution( - &block, - PostExecutionInput::new(&output.receipts, &output.requests), - ) { - // call post-block hook - self.invalid_block_hook.on_invalid_block( - &parent_block, - &block.seal_slow(), - &output, - None, - ); - return Err(err.into()) - } - - let hashed_state = self.provider.hashed_post_state(&output.state); - - trace!(target: "engine::tree", block=?sealed_block.num_hash(), "Calculating block state root"); - let root_time = Instant::now(); - let mut state_root_result = None; - - // We attempt to compute state root in parallel if we are currently not persisting anything - // to database. This is safe, because the database state cannot change until we - // finish parallel computation. It is important that nothing is being persisted as - // we are computing in parallel, because we initialize a different database transaction - // per thread and it might end up with a different view of the database. - let persistence_in_progress = self.persistence_state.in_progress(); - if !persistence_in_progress { - let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; - let mut input = self - .compute_trie_input(consistent_view.clone(), block.header().parent_hash()) - .map_err(|e| InsertBlockErrorKindTwo::Other(Box::new(e)))?; - // Extend with block we are validating root for. - input.append_ref(&hashed_state); - - state_root_result = match self.compute_state_root_parallel(consistent_view, input) { - Ok((state_root, trie_output)) => Some((state_root, trie_output)), - Err(ParallelStateRootError::Provider(ProviderError::ConsistentView(error))) => { - debug!(target: "engine", %error, "Parallel state root computation failed consistency check, falling back"); - None - } - Err(error) => return Err(InsertBlockErrorKindTwo::Other(Box::new(error))), + let state_root_result = match std::thread::scope(|scope| { + let (state_root_handle, state_hook) = if persistence_not_in_progress { + let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + + let input = Arc::new( + self.compute_trie_input(consistent_view.clone(), block.header().parent_hash()) + .unwrap(), + ); + let state_root_config = StateRootConfig { + consistent_view: consistent_view.clone(), + input: input.clone(), + }; + + let provider_ro = consistent_view.provider_ro()?; + let nodes_sorted = input.nodes.clone().into_sorted(); + let state_sorted = input.state.clone().into_sorted(); + let prefix_sets = Arc::new(input.prefix_sets.clone()); + + let context = + StateHookContext { provider_ro, nodes_sorted, state_sorted, prefix_sets }; + + let context = Box::leak(Box::new(context)); + + let blinded_provider_factory = ProofBlindedProviderFactory::new( + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::new(context.provider_ro.tx_ref()), + &context.nodes_sorted, + ), + HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(context.provider_ro.tx_ref()), + &context.state_sorted, + ), + context.prefix_sets.clone(), + ); + + let state_root_task = + StateRootTask::new(state_root_config, blinded_provider_factory); + let state_hook = state_root_task.state_hook(); + (Some(state_root_task.spawn(scope)), Box::new(state_hook) as Box) + } else { + (None, Box::new(|_state: &EvmState| {}) as Box) }; - } - let (state_root, trie_output) = if let Some(result) = state_root_result { - match state_root_handle.wait_for_result() { - Ok(state_root_task_result) => { - info!(target: "engine::tree", block=?sealed_block.num_hash(), state_root_task_result=?state_root_task_result.0, regular_state_root_result = ?result.0); - let task_trie_updates = &state_root_task_result.1; - let regular_trie_updates = &result.1; - let diff = compare_trie_updates(task_trie_updates, regular_trie_updates); - if diff.has_differences() { - for path in diff.account_nodes_with_different_values { - let task_entry = task_trie_updates.account_nodes.get(&path); - let regular_entry = regular_trie_updates.account_nodes.get(&path); - if task_entry != regular_entry { - debug!(target: "engine::tree", ?path, ?task_entry, ?regular_entry, "Difference in account node updates"); - } + let output = self.metrics.executor.execute_metered( + executor, + (&block, U256::MAX).into(), + state_hook, + )?; + + trace!(target: "engine::tree", elapsed=?exec_time.elapsed(), ?block_number, "Executed block"); + + if let Err(err) = self.consensus.validate_block_post_execution( + &block, + PostExecutionInput::new(&output.receipts, &output.requests), + ) { + // call post-block hook + self.invalid_block_hook.on_invalid_block( + &parent_block, + &block.clone().seal_slow(), + &output, + None, + ); + return Err(err.into()) + } + + let hashed_state = self.provider.hashed_post_state(&output.state); + + trace!(target: "engine::tree", block=?sealed_block.num_hash(), "Calculating block state root"); + let root_time = Instant::now(); + + // We attempt to compute state root in parallel if we are currently not persisting + // anything to database. This is safe, because the database state cannot + // change until we finish parallel computation. It is important that nothing + // is being persisted as we are computing in parallel, because we initialize + // a different database transaction per thread and it might end up with a + // different view of the database. + if persistence_not_in_progress { + if let Some(state_root_handle) = state_root_handle { + match state_root_handle.wait_for_result() { + Ok((task_state_root, _task_trie_updates)) => { + info!( + target: "engine::tree", + block = ?sealed_block.num_hash(), + ?task_state_root, + "State root task finished" + ); } - for address in diff.storage_tries_with_differences.keys() { - let task = task_trie_updates.storage_tries.get(address); - let regular = regular_trie_updates.storage_tries.get(address); - for path in task - .map_or_else(HashSet::new, |tries| { - tries.storage_nodes.keys().collect() - }) - .union(®ular.map_or_else(HashSet::new, |tries| { - tries.storage_nodes.keys().collect() - })) - { - let task_entry = task.map(|tries| tries.storage_nodes.get(*path)); - let regular_entry = - regular.map(|tries| tries.storage_nodes.get(*path)); - if task_entry != regular_entry { - debug!(target: "engine::tree", ?address, ?path, ?task_entry, ?regular_entry, "Difference in storage trie updates"); - } - } + Err(error) => { + info!(target: "engine::tree", ?error, "Failed to wait for state root task + result"); } - } else { - debug!(target: "engine::tree", block=?sealed_block.num_hash(), "TrieUpdates match exactly"); } } - Err(e) => { - info!(target: "engine::tree", error=?e, "on state root task wait_for_result") + + match self.compute_state_root_parallel(block.header().parent_hash(), &hashed_state) + { + Ok(result) => { + info!( + target: "engine::tree", + block = ?sealed_block.num_hash(), + regular_state_root = ?result.0, + "Regular root task finished" + ); + Ok(Some((result.0, result.1, hashed_state, output, root_time))) + } + Err(ParallelStateRootError::Provider(ProviderError::ConsistentView(error))) => { + debug!(target: "engine", %error, "Parallel state root computation failed consistency check, falling back"); + Ok(None) + } + Err(error) => return Err(InsertBlockErrorKindTwo::Other(Box::new(error))), } + } else { + Ok(None) } + }) { + Ok(Some(res)) => Some(res), + Ok(None) => None, + Err(e) => return Err(e), + }; + + let (state_root, trie_output, hashed_state, output, root_time) = if let Some(result) = + state_root_result + { result } else { - debug!(target: "engine::tree", block=?sealed_block.num_hash(), persistence_in_progress, "Failed to compute state root in parallel"); - state_provider.state_root_with_updates(hashed_state.clone())? + debug!(target: "engine::tree", block=?sealed_block.num_hash(), ?persistence_not_in_progress, "Failed to compute state root in parallel"); + let (root, updates) = + state_provider.state_root_with_updates(HashedPostState::default())?; + ( + root, + updates, + HashedPostState::default(), + BlockExecutionOutput { + state: BundleState::default(), + gas_used: 0, + receipts: vec![], + requests: Requests::default(), + }, + Instant::now(), + ) }; if state_root != block.header().state_root() { @@ -2422,9 +2472,15 @@ where /// should be used instead. fn compute_state_root_parallel( &self, - consistent_view: ConsistentDbView

, - input: TrieInput, + parent_hash: B256, + hashed_state: &HashedPostState, ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { + let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; + + let mut input = self.compute_trie_input(consistent_view.clone(), parent_hash)?; + // Extend with block we are validating root for. + input.append_ref(hashed_state); + ParallelStateRoot::new(consistent_view, input).incremental_root_with_updates() } From 89754169c2459911bc157559975116dd7ae319ee Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 13 Dec 2024 11:33:06 +0100 Subject: [PATCH 10/12] remove trie updates diff helpers --- crates/engine/tree/src/tree/mod.rs | 142 +---------------------------- 1 file changed, 3 insertions(+), 139 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index c315250c260f..3fe9298c0903 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -55,8 +55,8 @@ use reth_trie::{ prefix_set::TriePrefixSetsMut, proof::ProofBlindedProviderFactory, trie_cursor::InMemoryTrieCursorFactory, - updates::{StorageTrieUpdates, TrieUpdates, TrieUpdatesSorted}, - HashedPostState, HashedPostStateSorted, Nibbles, TrieInput, + updates::{TrieUpdates, TrieUpdatesSorted}, + HashedPostState, HashedPostStateSorted, TrieInput, }; use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; @@ -2354,7 +2354,7 @@ where debug!(target: "engine", %error, "Parallel state root computation failed consistency check, falling back"); Ok(None) } - Err(error) => return Err(InsertBlockErrorKindTwo::Other(Box::new(error))), + Err(error) => Err(InsertBlockErrorKindTwo::Other(Box::new(error))), } } else { Ok(None) @@ -2741,142 +2741,6 @@ pub enum AdvancePersistenceError { Provider(#[from] ProviderError), } -#[derive(Debug, Default)] -struct TrieUpdatesDiff { - pub account_nodes_only_in_first: HashSet, - pub account_nodes_only_in_second: HashSet, - pub account_nodes_with_different_values: HashSet, - pub removed_nodes_only_in_first: HashSet, - pub removed_nodes_only_in_second: HashSet, - pub storage_tries_only_in_first: HashSet, - pub storage_tries_only_in_second: HashSet, - pub storage_tries_with_differences: HashMap, -} - -#[derive(Debug, Default)] -struct StorageTrieUpdatesDiff { - pub is_deleted_differs: bool, - pub storage_nodes_only_in_first: HashSet, - pub storage_nodes_only_in_second: HashSet, - pub storage_nodes_with_different_values: HashSet, - pub removed_nodes_only_in_first: HashSet, - pub removed_nodes_only_in_second: HashSet, -} - -fn compare_trie_updates(first: &TrieUpdates, second: &TrieUpdates) -> TrieUpdatesDiff { - let mut diff = TrieUpdatesDiff::default(); - - // compare account nodes - for key in first.account_nodes.keys() { - if !second.account_nodes.contains_key(key) { - diff.account_nodes_only_in_first.insert(key.clone()); - } else if first.account_nodes.get(key) != second.account_nodes.get(key) { - diff.account_nodes_with_different_values.insert(key.clone()); - } - } - for key in second.account_nodes.keys() { - if !first.account_nodes.contains_key(key) { - diff.account_nodes_only_in_second.insert(key.clone()); - } - } - - // compare removed nodes - for node in &first.removed_nodes { - if !second.removed_nodes.contains(node) { - diff.removed_nodes_only_in_first.insert(node.clone()); - } - } - for node in &second.removed_nodes { - if !first.removed_nodes.contains(node) { - diff.removed_nodes_only_in_second.insert(node.clone()); - } - } - - // compare storage tries - for key in first.storage_tries.keys() { - if second.storage_tries.contains_key(key) { - let storage_diff = compare_storage_trie_updates( - first.storage_tries.get(key).unwrap(), - second.storage_tries.get(key).unwrap(), - ); - if storage_diff.has_differences() { - diff.storage_tries_with_differences.insert(*key, storage_diff); - } - } else { - diff.storage_tries_only_in_first.insert(*key); - } - } - for key in second.storage_tries.keys() { - if !first.storage_tries.contains_key(key) { - diff.storage_tries_only_in_second.insert(*key); - } - } - - diff -} - -fn compare_storage_trie_updates( - first: &StorageTrieUpdates, - second: &StorageTrieUpdates, -) -> StorageTrieUpdatesDiff { - let mut diff = StorageTrieUpdatesDiff { - is_deleted_differs: first.is_deleted != second.is_deleted, - ..Default::default() - }; - - // compare storage nodes - for key in first.storage_nodes.keys() { - if !second.storage_nodes.contains_key(key) { - diff.storage_nodes_only_in_first.insert(key.clone()); - } else if first.storage_nodes.get(key) != second.storage_nodes.get(key) { - diff.storage_nodes_with_different_values.insert(key.clone()); - } - } - for key in second.storage_nodes.keys() { - if !first.storage_nodes.contains_key(key) { - diff.storage_nodes_only_in_second.insert(key.clone()); - } - } - - // compare removed nodes - for node in &first.removed_nodes { - if !second.removed_nodes.contains(node) { - diff.removed_nodes_only_in_first.insert(node.clone()); - } - } - for node in &second.removed_nodes { - if !first.removed_nodes.contains(node) { - diff.removed_nodes_only_in_second.insert(node.clone()); - } - } - - diff -} - -impl StorageTrieUpdatesDiff { - fn has_differences(&self) -> bool { - self.is_deleted_differs || - !self.storage_nodes_only_in_first.is_empty() || - !self.storage_nodes_only_in_second.is_empty() || - !self.storage_nodes_with_different_values.is_empty() || - !self.removed_nodes_only_in_first.is_empty() || - !self.removed_nodes_only_in_second.is_empty() - } -} - -impl TrieUpdatesDiff { - fn has_differences(&self) -> bool { - !self.account_nodes_only_in_first.is_empty() || - !self.account_nodes_only_in_second.is_empty() || - !self.account_nodes_with_different_values.is_empty() || - !self.removed_nodes_only_in_first.is_empty() || - !self.removed_nodes_only_in_second.is_empty() || - !self.storage_tries_only_in_first.is_empty() || - !self.storage_tries_only_in_second.is_empty() || - !self.storage_tries_with_differences.is_empty() - } -} - #[cfg(test)] mod tests { use super::*; From fcd44746e34dccc941287dcd77ba366566272161 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 13 Dec 2024 11:40:19 +0100 Subject: [PATCH 11/12] comments --- crates/engine/tree/src/tree/mod.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 3fe9298c0903..5c049a6d7128 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -90,13 +90,6 @@ pub use reth_engine_primitives::InvalidBlockHook; pub mod root; -struct StateHookContext

{ - provider_ro: P, - nodes_sorted: TrieUpdatesSorted, - state_sorted: HashedPostStateSorted, - prefix_sets: Arc, -} - /// Keeps track of the state of the tree. /// /// ## Invariants @@ -482,6 +475,15 @@ pub enum TreeAction { }, } +/// Context used to keep alive the required values when returning a state hook +/// from a scoped thread. +struct StateHookContext

{ + provider_ro: P, + nodes_sorted: TrieUpdatesSorted, + state_sorted: HashedPostStateSorted, + prefix_sets: Arc, +} + /// The engine API tree handler implementation. /// /// This type is responsible for processing engine API requests, maintaining the canonical state and @@ -2263,9 +2265,12 @@ where let state_sorted = input.state.clone().into_sorted(); let prefix_sets = Arc::new(input.prefix_sets.clone()); + // context will hold the values that need to be kept alive let context = StateHookContext { provider_ro, nodes_sorted, state_sorted, prefix_sets }; + // it is ok to leak here because we are in a scoped thread, the + // memory will be freed when the thread completes let context = Box::leak(Box::new(context)); let blinded_provider_factory = ProofBlindedProviderFactory::new( From 1de046bc25228ca3b6bf4af7bfec0833bda4af21 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 13 Dec 2024 13:01:07 +0100 Subject: [PATCH 12/12] return hashed_state and output in all cases --- crates/engine/tree/src/tree/mod.rs | 51 ++++++++++++------------------ 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 5c049a6d7128..f41330724ecf 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -6,7 +6,7 @@ use crate::{ tree::metrics::EngineApiMetrics, }; use alloy_consensus::BlockHeader; -use alloy_eips::{eip7685::Requests, BlockNumHash}; +use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{HashMap, HashSet}, BlockNumber, B256, U256, @@ -31,10 +31,7 @@ use reth_engine_primitives::{ EngineValidator, ForkchoiceStateTracker, OnForkChoiceUpdated, }; use reth_errors::{ConsensusError, ProviderResult}; -use reth_evm::{ - execute::{BlockExecutionOutput, BlockExecutorProvider}, - system_calls::OnStateHook, -}; +use reth_evm::{execute::BlockExecutorProvider, system_calls::OnStateHook}; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_builder_primitives::PayloadBuilder; use reth_payload_primitives::PayloadBuilderAttributes; @@ -48,7 +45,7 @@ use reth_provider::{ ExecutionOutcome, HashedPostStateProvider, ProviderError, StateCommitmentProvider, StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, }; -use reth_revm::{database::StateProviderDatabase, db::BundleState}; +use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, @@ -2353,43 +2350,35 @@ where regular_state_root = ?result.0, "Regular root task finished" ); - Ok(Some((result.0, result.1, hashed_state, output, root_time))) + Ok((Some((result.0, result.1)), hashed_state, output, root_time)) } Err(ParallelStateRootError::Provider(ProviderError::ConsistentView(error))) => { debug!(target: "engine", %error, "Parallel state root computation failed consistency check, falling back"); - Ok(None) + Ok((None, hashed_state, output, root_time)) } Err(error) => Err(InsertBlockErrorKindTwo::Other(Box::new(error))), } } else { - Ok(None) + Ok((None, hashed_state, output, root_time)) } }) { - Ok(Some(res)) => Some(res), - Ok(None) => None, + Ok((Some(res), hashed_state, output, root_time)) => { + (Some(res), hashed_state, output, root_time) + } + Ok((None, hashed_state, output, root_time)) => (None, hashed_state, output, root_time), Err(e) => return Err(e), }; - let (state_root, trie_output, hashed_state, output, root_time) = if let Some(result) = - state_root_result - { - result - } else { - debug!(target: "engine::tree", block=?sealed_block.num_hash(), ?persistence_not_in_progress, "Failed to compute state root in parallel"); - let (root, updates) = - state_provider.state_root_with_updates(HashedPostState::default())?; - ( - root, - updates, - HashedPostState::default(), - BlockExecutionOutput { - state: BundleState::default(), - gas_used: 0, - receipts: vec![], - requests: Requests::default(), - }, - Instant::now(), - ) + let (state_root, trie_output, hashed_state, output, root_time) = match state_root_result { + (Some(res), hashed_state, output, root_time) => { + (res.0, res.1, hashed_state, output, root_time) + } + (None, hashed_state, output, root_time) => { + debug!(target: "engine::tree", block=?sealed_block.num_hash(), ?persistence_not_in_progress, "Failed to compute state root in parallel"); + let (root, updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + (root, updates, hashed_state, output, root_time) + } }; if state_root != block.header().state_root() {