Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: add BlockBodyIndicesProvider trait #13347

Merged
merged 2 commits into from
Dec 12, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions crates/stages/stages/src/stages/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@ mod tests {
use reth_provider::{
providers::{StaticFileProvider, StaticFileWriter},
test_utils::MockNodeTypesWithDB,
AccountExtReader, BlockReader, DatabaseProviderFactory, ProviderFactory, ProviderResult,
ReceiptProvider, StageCheckpointWriter, StaticFileProviderFactory, StorageReader,
AccountExtReader, BlockBodyIndicesProvider, DatabaseProviderFactory,
ProviderFactory, ProviderResult, ReceiptProvider, StageCheckpointWriter,
StaticFileProviderFactory, StorageReader,
};
use reth_prune_types::{PruneMode, PruneModes};
use reth_stages_api::{
Expand Down
4 changes: 2 additions & 2 deletions crates/stages/stages/src/stages/sender_recovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,8 @@ mod tests {
use reth_primitives::{SealedBlock, TransactionSigned};
use reth_primitives_traits::SignedTransaction;
use reth_provider::{
providers::StaticFileWriter, DatabaseProviderFactory, PruneCheckpointWriter,
StaticFileProviderFactory, TransactionsProvider,
providers::StaticFileWriter, BlockBodyIndicesProvider, DatabaseProviderFactory,
PruneCheckpointWriter, StaticFileProviderFactory, TransactionsProvider,
};
use reth_prune_types::{PruneCheckpoint, PruneMode};
use reth_stages_api::StageUnitCheckpoint;
Expand Down
3 changes: 2 additions & 1 deletion crates/stages/stages/src/stages/tx_lookup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,8 @@ mod tests {
use assert_matches::assert_matches;
use reth_primitives::SealedBlock;
use reth_provider::{
providers::StaticFileWriter, DatabaseProviderFactory, StaticFileProviderFactory,
providers::StaticFileWriter, BlockBodyIndicesProvider, DatabaseProviderFactory,
StaticFileProviderFactory,
};
use reth_stages_api::StageUnitCheckpoint;
use reth_testing_utils::generators::{
Expand Down
28 changes: 15 additions & 13 deletions crates/storage/provider/src/providers/blockchain_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ use reth_primitives_traits::BlockBody as _;
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
DBProvider, NodePrimitivesProvider, OmmersProvider, StateCommitmentProvider,
StorageChangeSetReader,
BlockBodyIndicesProvider, DBProvider, NodePrimitivesProvider, OmmersProvider,
StateCommitmentProvider, StorageChangeSetReader,
};
use reth_storage_errors::provider::ProviderResult;
use reth_trie::HashedPostState;
Expand Down Expand Up @@ -315,13 +315,6 @@ impl<N: ProviderNodeTypes> BlockReader for BlockchainProvider2<N> {
Ok(self.canonical_in_memory_state.pending_block_and_receipts())
}

fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.consistent_provider()?.block_body_indices(number)
}

/// Returns the block with senders with matching number or hash from database.
///
/// **NOTE: If [`TransactionVariant::NoHash`] is provided then the transactions have invalid
Expand Down Expand Up @@ -481,6 +474,15 @@ impl<N: ProviderNodeTypes> OmmersProvider for BlockchainProvider2<N> {
}
}

impl<N: ProviderNodeTypes> BlockBodyIndicesProvider for BlockchainProvider2<N> {
fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.consistent_provider()?.block_body_indices(number)
}
}

impl<N: ProviderNodeTypes> StageCheckpointReader for BlockchainProvider2<N> {
fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult<Option<StageCheckpoint>> {
self.consistent_provider()?.get_stage_checkpoint(id)
Expand Down Expand Up @@ -817,10 +819,10 @@ mod tests {
use reth_primitives::{BlockExt, Receipt, SealedBlock, StaticFileSegment};
use reth_primitives_traits::{BlockBody as _, SignedTransaction};
use reth_storage_api::{
BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource,
ChangeSetReader, DatabaseProviderFactory, HeaderProvider, OmmersProvider, ReceiptProvider,
ReceiptProviderIdExt, StateProviderFactory, TransactionVariant, TransactionsProvider,
WithdrawalsProvider,
BlockBodyIndicesProvider, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader,
BlockReaderIdExt, BlockSource, ChangeSetReader, DatabaseProviderFactory, HeaderProvider,
OmmersProvider, ReceiptProvider, ReceiptProviderIdExt, StateProviderFactory,
TransactionVariant, TransactionsProvider, WithdrawalsProvider,
};
use reth_testing_utils::generators::{
self, random_block, random_block_range, random_changeset_range, random_eoa_accounts,
Expand Down
74 changes: 38 additions & 36 deletions crates/storage/provider/src/providers/consistent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ use reth_primitives_traits::BlockBody;
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
DatabaseProviderFactory, NodePrimitivesProvider, OmmersProvider, StateProvider,
StorageChangeSetReader,
BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, OmmersProvider,
StateProvider, StorageChangeSetReader,
};
use reth_storage_errors::provider::ProviderResult;
use revm::{
Expand Down Expand Up @@ -841,40 +841,6 @@ impl<N: ProviderNodeTypes> BlockReader for ConsistentProvider<N> {
Ok(self.canonical_in_memory_state.pending_block_and_receipts())
}

fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.get_in_memory_or_storage_by_block(
number.into(),
|db_provider| db_provider.block_body_indices(number),
|block_state| {
// Find the last block indices on database
let last_storage_block_number = block_state.anchor().number;
let mut stored_indices = self
.storage_provider
.block_body_indices(last_storage_block_number)?
.ok_or(ProviderError::BlockBodyIndicesNotFound(last_storage_block_number))?;

// Prepare our block indices
stored_indices.first_tx_num = stored_indices.next_tx_num();
stored_indices.tx_count = 0;

// Iterate from the lowest block in memory until our target block
for state in block_state.chain().collect::<Vec<_>>().into_iter().rev() {
let block_tx_count = state.block_ref().block.body.transactions().len() as u64;
if state.block_ref().block().number() == number {
stored_indices.tx_count = block_tx_count;
} else {
stored_indices.first_tx_num += block_tx_count;
}
}

Ok(Some(stored_indices))
},
)
}

/// Returns the block with senders with matching number or hash from database.
///
/// **NOTE: If [`TransactionVariant::NoHash`] is provided then the transactions have invalid
Expand Down Expand Up @@ -1219,6 +1185,42 @@ impl<N: ProviderNodeTypes> OmmersProvider for ConsistentProvider<N> {
}
}

impl<N: ProviderNodeTypes> BlockBodyIndicesProvider for ConsistentProvider<N> {
fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.get_in_memory_or_storage_by_block(
number.into(),
|db_provider| db_provider.block_body_indices(number),
|block_state| {
// Find the last block indices on database
let last_storage_block_number = block_state.anchor().number;
let mut stored_indices = self
.storage_provider
.block_body_indices(last_storage_block_number)?
.ok_or(ProviderError::BlockBodyIndicesNotFound(last_storage_block_number))?;

// Prepare our block indices
stored_indices.first_tx_num = stored_indices.next_tx_num();
stored_indices.tx_count = 0;

// Iterate from the lowest block in memory until our target block
for state in block_state.chain().collect::<Vec<_>>().into_iter().rev() {
let block_tx_count = state.block_ref().block.body.transactions().len() as u64;
if state.block_ref().block().number() == number {
stored_indices.tx_count = block_tx_count;
} else {
stored_indices.first_tx_num += block_tx_count;
}
}

Ok(Some(stored_indices))
},
)
}
}

impl<N: ProviderNodeTypes> StageCheckpointReader for ConsistentProvider<N> {
fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult<Option<StageCheckpoint>> {
self.storage_provider.get_stage_checkpoint(id)
Expand Down
19 changes: 11 additions & 8 deletions crates/storage/provider/src/providers/database/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ use reth_primitives::{
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
NodePrimitivesProvider, OmmersProvider, StateCommitmentProvider, TryIntoHistoricalStateProvider,
BlockBodyIndicesProvider, NodePrimitivesProvider, OmmersProvider, StateCommitmentProvider,
TryIntoHistoricalStateProvider,
};
use reth_storage_errors::provider::ProviderResult;
use reth_trie::HashedPostState;
Expand Down Expand Up @@ -404,13 +405,6 @@ impl<N: ProviderNodeTypes> BlockReader for ProviderFactory<N> {
self.provider()?.pending_block_and_receipts()
}

fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.provider()?.block_body_indices(number)
}

fn block_with_senders(
&self,
id: BlockHashOrNumber,
Expand Down Expand Up @@ -578,6 +572,15 @@ impl<N: ProviderNodeTypes> OmmersProvider for ProviderFactory<N> {
}
}

impl<N: ProviderNodeTypes> BlockBodyIndicesProvider for ProviderFactory<N> {
fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.provider()?.block_body_indices(number)
}
}

impl<N: ProviderNodeTypes> StageCheckpointReader for ProviderFactory<N> {
fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult<Option<StageCheckpoint>> {
self.provider()?.get_stage_checkpoint(id)
Expand Down
16 changes: 10 additions & 6 deletions crates/storage/provider/src/providers/database/provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ use reth_primitives_traits::{Block as _, BlockBody as _, SignedTransaction};
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
BlockBodyReader, NodePrimitivesProvider, OmmersProvider, StateProvider, StorageChangeSetReader,
TryIntoHistoricalStateProvider,
BlockBodyIndicesProvider, BlockBodyReader, NodePrimitivesProvider, OmmersProvider,
StateProvider, StorageChangeSetReader, TryIntoHistoricalStateProvider,
};
use reth_storage_errors::provider::{ProviderResult, RootMismatch};
use reth_trie::{
Expand Down Expand Up @@ -1222,10 +1222,6 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockReader for DatabaseProvid
Ok(None)
}

fn block_body_indices(&self, num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
Ok(self.tx.get::<tables::BlockBodyIndices>(num)?)
}

/// Returns the block with senders with matching number or hash from database.
///
/// **NOTE: The transactions have invalid hashes, since they would need to be calculated on the
Expand Down Expand Up @@ -1637,6 +1633,14 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> OmmersProvider for DatabasePro
}
}

impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockBodyIndicesProvider
for DatabaseProvider<TX, N>
{
fn block_body_indices(&self, num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
Ok(self.tx.get::<tables::BlockBodyIndices>(num)?)
}
}

impl<TX: DbTx + 'static, N: NodeTypesForProvider> EvmEnvProvider<HeaderTy<N>>
for DatabaseProvider<TX, N>
{
Expand Down
18 changes: 10 additions & 8 deletions crates/storage/provider/src/providers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ use reth_primitives::{
};
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{CanonChainTracker, OmmersProvider};
use reth_storage_api::{BlockBodyIndicesProvider, CanonChainTracker, OmmersProvider};
use reth_storage_errors::provider::ProviderResult;
use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg};
use std::{
Expand Down Expand Up @@ -402,13 +402,6 @@ impl<N: TreeNodeTypes> BlockReader for BlockchainProvider<N> {
Ok(self.tree.pending_block_and_receipts())
}

fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.database.block_body_indices(number)
}

/// Returns the block with senders with matching number or hash from database.
///
/// **NOTE: If [`TransactionVariant::NoHash`] is provided then the transactions have invalid
Expand Down Expand Up @@ -586,6 +579,15 @@ impl<N: TreeNodeTypes> OmmersProvider for BlockchainProvider<N> {
}
}

impl<N: ProviderNodeTypes> BlockBodyIndicesProvider for BlockchainProvider<N> {
fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.database.block_body_indices(number)
}
}

impl<N: ProviderNodeTypes> StageCheckpointReader for BlockchainProvider<N> {
fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult<Option<StageCheckpoint>> {
self.database.provider()?.get_stage_checkpoint(id)
Expand Down
14 changes: 8 additions & 6 deletions crates/storage/provider/src/providers/static_file/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ use reth_primitives::{
};
use reth_primitives_traits::SignedTransaction;
use reth_stages_types::{PipelineTarget, StageId};
use reth_storage_api::{DBProvider, OmmersProvider};
use reth_storage_api::{BlockBodyIndicesProvider, DBProvider, OmmersProvider};
use reth_storage_errors::provider::{ProviderError, ProviderResult};
use std::{
collections::{hash_map::Entry, BTreeMap, HashMap},
Expand Down Expand Up @@ -1628,11 +1628,6 @@ impl<N: FullNodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>>
Err(ProviderError::UnsupportedProvider)
}

fn block_body_indices(&self, _num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
// Required data not present in static_files
Err(ProviderError::UnsupportedProvider)
}

fn block_with_senders(
&self,
_id: BlockHashOrNumber,
Expand Down Expand Up @@ -1694,6 +1689,13 @@ impl<N: FullNodePrimitives<BlockHeader: Value>> OmmersProvider for StaticFilePro
}
}

impl<N: Send + Sync> BlockBodyIndicesProvider for StaticFileProvider<N> {
fn block_body_indices(&self, _num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
// Required data not present in static_files
Err(ProviderError::UnsupportedProvider)
}
}

impl<N: NodePrimitives> StatsReader for StaticFileProvider<N> {
fn count_entries<T: Table>(&self) -> ProviderResult<usize> {
match T::NAME {
Expand Down
14 changes: 8 additions & 6 deletions crates/storage/provider/src/test_utils/mock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use reth_primitives::{
use reth_primitives_traits::SignedTransaction;
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
DatabaseProviderFactory, HashedPostStateProvider, OmmersProvider, StageCheckpointReader,
StateCommitmentProvider, StateProofProvider, StorageRootProvider,
BlockBodyIndicesProvider, DatabaseProviderFactory, HashedPostStateProvider, OmmersProvider,
StageCheckpointReader, StateCommitmentProvider, StateProofProvider, StorageRootProvider,
};
use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult};
use reth_trie::{
Expand Down Expand Up @@ -510,10 +510,6 @@ impl BlockReader for MockEthProvider {
Ok(None)
}

fn block_body_indices(&self, _num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
Ok(None)
}

fn block_with_senders(
&self,
_id: BlockHashOrNumber,
Expand Down Expand Up @@ -796,6 +792,12 @@ impl OmmersProvider for MockEthProvider {
}
}

impl BlockBodyIndicesProvider for MockEthProvider {
fn block_body_indices(&self, _num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
Ok(None)
}
}

impl ChangeSetReader for MockEthProvider {
fn account_block_changeset(
&self,
Expand Down
Loading