diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f625a69ec..4a0b7d1d01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,11 @@ More expansive patch notes and explanations may be found in the specific [pathfi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.14.3] - 2024-09-23 -### Added +### Fixed -- Pathfinder now fetches data concurrently from the feeder gateway when catching up. The `--gateway.fetch-concurrency` CLI option can be used to limit how many blocks are fetched concurrently (the default is 8). +- Pathfinder occasionally corrupts its Merkle trie storage during reorgs and then stops later with a "Node X at height Y is missing" or "Stored node's hash is missing" error. ## [0.14.2] - 2024-09-03 diff --git a/Cargo.lock b/Cargo.lock index 5a4abc129d..646dc4deba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3942,7 +3942,7 @@ dependencies = [ [[package]] name = "gateway-test-utils" -version = "0.14.2" +version = "0.14.3" dependencies = [ "reqwest", "serde_json", @@ -6194,7 +6194,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "p2p" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "async-trait", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "p2p_proto" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "fake", @@ -6256,7 +6256,7 @@ dependencies = [ [[package]] name = "p2p_proto_derive" -version = "0.14.2" +version = "0.14.3" dependencies = [ "proc-macro2", "quote", @@ -6265,7 +6265,7 @@ dependencies = [ [[package]] name = "p2p_stream" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "async-trait", @@ -6393,7 +6393,7 @@ checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "pathfinder" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "assert_matches", @@ -6458,7 +6458,7 @@ dependencies = [ [[package]] name = "pathfinder-common" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "bitvec", @@ -6482,7 +6482,7 @@ dependencies = [ [[package]] name = "pathfinder-compiler" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "cairo-lang-starknet 1.0.0-alpha.6", @@ -6503,7 +6503,7 @@ dependencies = [ [[package]] name = "pathfinder-crypto" -version = "0.14.2" +version = "0.14.3" dependencies = [ "ark-ff", "assert_matches", @@ -6520,7 +6520,7 @@ dependencies = [ [[package]] name = "pathfinder-ethereum" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "async-trait", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "pathfinder-executor" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "blockifier", @@ -6560,7 +6560,7 @@ dependencies = [ [[package]] name = "pathfinder-merkle-tree" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "bitvec", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "pathfinder-retry" -version = "0.14.2" +version = "0.14.3" dependencies = [ "tokio", "tokio-retry", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "pathfinder-rpc" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "assert_matches", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "pathfinder-serde" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "num-bigint 0.4.6", @@ -6650,7 +6650,7 @@ dependencies = [ [[package]] name = "pathfinder-storage" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "assert_matches", @@ -8340,7 +8340,7 @@ dependencies = [ [[package]] name = "starknet-gateway-client" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "assert_matches", @@ -8373,7 +8373,7 @@ dependencies = [ [[package]] name = "starknet-gateway-test-fixtures" -version = "0.14.2" +version = "0.14.3" dependencies = [ "pathfinder-common", "pathfinder-crypto", @@ -8381,7 +8381,7 @@ dependencies = [ [[package]] name = "starknet-gateway-types" -version = "0.14.2" +version = "0.14.3" dependencies = [ "anyhow", "assert_matches", @@ -8583,7 +8583,7 @@ dependencies = [ [[package]] name = "tagged" -version = "0.14.2" +version = "0.14.3" dependencies = [ "fake", "pretty_assertions_sorted", @@ -8592,7 +8592,7 @@ dependencies = [ [[package]] name = "tagged-debug-derive" -version = "0.14.2" +version = "0.14.3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 3779a97d3d..a02629c2ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,11 +38,11 @@ lto = true opt-level = 3 [workspace.package] -authors = ["Equilibrium Labs "] +version = "0.14.3" edition = "2021" license = "MIT OR Apache-2.0" rust-version = "1.80" -version = "0.14.2" +authors = ["Equilibrium Labs "] [workspace.dependencies] anyhow = "1.0.75" diff --git a/crates/load-test/Cargo.lock b/crates/load-test/Cargo.lock index aea085a038..c316e4d9dd 100644 --- a/crates/load-test/Cargo.lock +++ b/crates/load-test/Cargo.lock @@ -976,7 +976,7 @@ dependencies = [ [[package]] name = "pathfinder-crypto" -version = "0.14.2" +version = "0.14.3" dependencies = [ "bitvec", "fake", diff --git a/crates/merkle-tree/src/contract_state.rs b/crates/merkle-tree/src/contract_state.rs index 33f7333131..4bde30d203 100644 --- a/crates/merkle-tree/src/contract_state.rs +++ b/crates/merkle-tree/src/contract_state.rs @@ -207,7 +207,7 @@ pub fn revert_contract_state( } else { transaction .contract_root(head, contract_address)? - .context("Fetching current contract root")? + .unwrap_or(ContractRoot::ZERO) }; let state_hash = if contract_address.is_system_contract() && root == ContractRoot::ZERO diff --git a/crates/pathfinder/src/state/sync/revert.rs b/crates/pathfinder/src/state/sync/revert.rs index 06d4580d8c..6abe0c2042 100644 --- a/crates/pathfinder/src/state/sync/revert.rs +++ b/crates/pathfinder/src/state/sync/revert.rs @@ -49,8 +49,7 @@ pub fn revert_starknet_state( target_block, target_header.class_commitment, )?; - - transaction.coalesce_trie_nodes(target_block) + transaction.coalesce_trie_removals(target_block) } /// Revert all contract/global storage trie updates. @@ -63,55 +62,52 @@ fn revert_contract_updates( target_block: BlockNumber, expected_storage_commitment: StorageCommitment, ) -> anyhow::Result<()> { - if !transaction.storage_root_exists(target_block)? { - let updates = transaction.reverse_contract_updates(head, target_block)?; - - let mut global_tree = StorageCommitmentTree::load(transaction, head) - .context("Loading global storage tree")?; - - for (contract_address, contract_update) in updates { - let state_hash = pathfinder_merkle_tree::contract_state::revert_contract_state( - transaction, - contract_address, - head, - target_block, - contract_update, - )?; - - transaction - .insert_contract_state_hash(target_block, contract_address, state_hash) - .context("Inserting reverted contract state hash")?; - - global_tree - .set(contract_address, state_hash) - .context("Updating contract state hash in global tree")?; - } - - tracing::debug!("Applied reverse updates, committing global state tree"); - - let (storage_commitment, trie_update) = global_tree - .commit() - .context("Committing global state tree")?; - - if expected_storage_commitment != storage_commitment { - anyhow::bail!( - "Storage commitment mismatch: expected {}, calculated {}", - expected_storage_commitment, - storage_commitment - ); - } - - let root_idx = transaction - .insert_storage_trie(&trie_update, target_block) - .context("Persisting storage trie")?; + let updates = transaction.reverse_contract_updates(head, target_block)?; + + let mut global_tree = + StorageCommitmentTree::load(transaction, head).context("Loading global storage tree")?; + + for (contract_address, contract_update) in updates { + let state_hash = pathfinder_merkle_tree::contract_state::revert_contract_state( + transaction, + contract_address, + head, + target_block, + contract_update, + )?; transaction - .insert_storage_root(target_block, root_idx) - .context("Inserting storage root index")?; - tracing::debug!(%target_block, %storage_commitment, "Committed global state tree"); - } else { - tracing::debug!(%target_block, "State tree root node exists"); + .insert_contract_state_hash(target_block, contract_address, state_hash) + .context("Inserting reverted contract state hash")?; + + global_tree + .set(contract_address, state_hash) + .context("Updating contract state hash in global tree")?; + } + + tracing::debug!("Applied reverse updates, committing global state tree"); + + let (storage_commitment, trie_update) = global_tree + .commit() + .context("Committing global state tree")?; + + if expected_storage_commitment != storage_commitment { + anyhow::bail!( + "Storage commitment mismatch: expected {}, calculated {}", + expected_storage_commitment, + storage_commitment + ); } + + let root_idx = transaction + .insert_storage_trie(&trie_update, target_block) + .context("Persisting storage trie")?; + + transaction + .insert_storage_root(target_block, root_idx) + .context("Inserting storage root index")?; + tracing::debug!(%target_block, %storage_commitment, "Committed global state tree"); + Ok(()) } @@ -122,51 +118,48 @@ fn revert_class_updates( target_block: BlockNumber, expected_class_commitment: ClassCommitment, ) -> anyhow::Result<()> { - if !transaction.class_root_exists(target_block)? { - let updates = transaction.reverse_sierra_class_updates(head, target_block)?; - - let mut class_tree = ClassCommitmentTree::load(transaction, head) - .context("Loading class commitment trie")?; - - for (class_hash, casm_update) in updates { - let new_value = match casm_update { - None => { - // The class must be removed - ClassCommitmentLeafHash::ZERO - } - Some(casm_hash) => { - // Class hash has changed. Note that the class commitment leaf must have already - // been added to storage. - pathfinder_common::calculate_class_commitment_leaf_hash(casm_hash) - } - }; - - class_tree - .set(class_hash, new_value) - .context("Updating class commitment trie")?; - } - - let (class_commitment, trie_update) = - class_tree.commit().context("Committing class trie")?; - - if expected_class_commitment != class_commitment { - anyhow::bail!( - "Storage commitment mismatch: expected {}, calculated {}", - expected_class_commitment, - class_commitment - ); - } - - let root_idx = transaction - .insert_class_trie(&trie_update, target_block) - .context("Persisting class trie")?; + let updates = transaction.reverse_sierra_class_updates(head, target_block)?; + + let mut class_tree = + ClassCommitmentTree::load(transaction, head).context("Loading class commitment trie")?; + + for (class_hash, casm_update) in updates { + let new_value = match casm_update { + None => { + // The class must be removed + ClassCommitmentLeafHash::ZERO + } + Some(casm_hash) => { + // Class hash has changed. Note that the class commitment leaf must have already + // been added to storage. + pathfinder_common::calculate_class_commitment_leaf_hash(casm_hash) + } + }; + + class_tree + .set(class_hash, new_value) + .context("Updating class commitment trie")?; + } - transaction - .insert_class_root(target_block, root_idx) - .context("Inserting class root index")?; + let (class_commitment, trie_update) = class_tree.commit().context("Committing class trie")?; - tracing::debug!(%target_block, %class_commitment, "Committed class trie"); + if expected_class_commitment != class_commitment { + anyhow::bail!( + "Storage commitment mismatch: expected {}, calculated {}", + expected_class_commitment, + class_commitment + ); } + let root_idx = transaction + .insert_class_trie(&trie_update, target_block) + .context("Persisting class trie")?; + + transaction + .insert_class_root(target_block, root_idx) + .context("Inserting class root index")?; + + tracing::debug!(%target_block, %class_commitment, "Committed class trie"); + Ok(()) } diff --git a/crates/storage/src/connection/trie.rs b/crates/storage/src/connection/trie.rs index 7c55c451e9..11fe530526 100644 --- a/crates/storage/src/connection/trie.rs +++ b/crates/storage/src/connection/trie.rs @@ -366,7 +366,7 @@ impl Transaction<'_> { Ok(()) } - pub fn coalesce_trie_nodes(&self, target_block: BlockNumber) -> anyhow::Result<()> { + pub fn coalesce_trie_removals(&self, target_block: BlockNumber) -> anyhow::Result<()> { self.coalesce_removed_trie_nodes(target_block, "trie_contracts")?; self.coalesce_removed_trie_nodes(target_block, "trie_storage")?; self.coalesce_removed_trie_nodes(target_block, "trie_class")