From 3e102227fd91f25a5e490324140d13418d1ca540 Mon Sep 17 00:00:00 2001 From: joshie <93316087+joshieDo@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:28:08 +0000 Subject: [PATCH] producer & pruner --- crates/cli/commands/src/db/get.rs | 28 +- crates/prune/prune/src/segments/mod.rs | 2 +- crates/prune/prune/src/segments/set.rs | 7 +- .../src/segments/static_file/block_meta.rs | 359 ++++++++++++++++++ .../prune/src/segments/static_file/mod.rs | 2 + crates/prune/types/src/segment.rs | 11 +- .../static-file/src/segments/block_meta.rs | 61 +++ .../static-file/src/segments/mod.rs | 3 + .../static-file/src/static_file_producer.rs | 23 +- crates/static-file/types/src/lib.rs | 4 +- crates/storage/db/src/static_file/masks.rs | 9 +- .../provider/src/providers/database/mod.rs | 7 +- .../src/providers/database/provider.rs | 30 +- .../src/providers/static_file/manager.rs | 5 +- .../src/providers/static_file/writer.rs | 14 +- 15 files changed, 524 insertions(+), 41 deletions(-) create mode 100644 crates/prune/prune/src/segments/static_file/block_meta.rs create mode 100644 crates/static-file/static-file/src/segments/block_meta.rs diff --git a/crates/cli/commands/src/db/get.rs b/crates/cli/commands/src/db/get.rs index 5fb234f0b89c..0b4a18284090 100644 --- a/crates/cli/commands/src/db/get.rs +++ b/crates/cli/commands/src/db/get.rs @@ -3,9 +3,11 @@ use alloy_primitives::{hex, BlockHash}; use clap::Parser; use reth_db::{ static_file::{ - ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask, + AllBlockMetaMask, ColumnSelectorOne, ColumnSelectorThree, ColumnSelectorTwo, + HeaderWithHashMask, ReceiptMask, TransactionMask, }, - tables, RawKey, RawTable, Receipts, TableViewer, Transactions, + tables, BlockBodyIndices, BlockOmmers, BlockWithdrawals, RawKey, RawTable, Receipts, + TableViewer, Transactions, }; use reth_db_api::table::{Decompress, DupSort, Table}; use reth_db_common::DbTool; @@ -72,7 +74,10 @@ impl Command { StaticFileSegment::Receipts => { (table_key::(&key)?, >>::MASK) } - StaticFileSegment::BlockMeta => todo!(), + StaticFileSegment::BlockMeta => ( + table_key::(&key)?, + >::MASK, + ), }; let content = tool.provider_factory.static_file_provider().find_static_file( @@ -115,7 +120,22 @@ impl Command { println!("{}", serde_json::to_string_pretty(&receipt)?); } StaticFileSegment::BlockMeta => { - todo!() + let indices = <::Value>::decompress( + content[0].as_slice(), + )?; + let ommers = <::Value>::decompress( + content[1].as_slice(), + )?; + let withdrawals = + <::Value>::decompress( + content[2].as_slice(), + )?; + println!( + "BlockIndices\n{}\nOmmers\n{}\nWithdrawals\n{}", + serde_json::to_string_pretty(&indices)?, + serde_json::to_string_pretty(&ommers)?, + serde_json::to_string_pretty(&withdrawals)? + ); } } } diff --git a/crates/prune/prune/src/segments/mod.rs b/crates/prune/prune/src/segments/mod.rs index 9f9e989dc06a..e45f49f0c67c 100644 --- a/crates/prune/prune/src/segments/mod.rs +++ b/crates/prune/prune/src/segments/mod.rs @@ -9,7 +9,7 @@ use reth_provider::{errors::provider::ProviderResult, BlockReader, PruneCheckpoi use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput}; pub use set::SegmentSet; pub use static_file::{ - Headers as StaticFileHeaders, Receipts as StaticFileReceipts, + BlockMeta as StaticFileBlockMeta, Headers as StaticFileHeaders, Receipts as StaticFileReceipts, Transactions as StaticFileTransactions, }; use std::{fmt::Debug, ops::RangeInclusive}; diff --git a/crates/prune/prune/src/segments/set.rs b/crates/prune/prune/src/segments/set.rs index d7bbee1042ba..12e1b14a0596 100644 --- a/crates/prune/prune/src/segments/set.rs +++ b/crates/prune/prune/src/segments/set.rs @@ -1,3 +1,4 @@ +use super::{StaticFileBlockMeta, StaticFileHeaders, StaticFileReceipts, StaticFileTransactions}; use crate::segments::{ AccountHistory, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory, TransactionLookup, UserReceipts, @@ -11,8 +12,6 @@ use reth_provider::{ }; use reth_prune_types::PruneModes; -use super::{StaticFileHeaders, StaticFileReceipts, StaticFileTransactions}; - /// Collection of [`Segment`]. Thread-safe, allocated on the heap. #[derive(Debug)] pub struct SegmentSet { @@ -73,7 +72,9 @@ where // Static file transactions .segment(StaticFileTransactions::new(static_file_provider.clone())) // Static file receipts - .segment(StaticFileReceipts::new(static_file_provider)) + .segment(StaticFileReceipts::new(static_file_provider.clone())) + // Static file block meta + .segment(StaticFileBlockMeta::new(static_file_provider)) // Account history .segment_opt(account_history.map(AccountHistory::new)) // Storage history diff --git a/crates/prune/prune/src/segments/static_file/block_meta.rs b/crates/prune/prune/src/segments/static_file/block_meta.rs new file mode 100644 index 000000000000..0cea195cb7a7 --- /dev/null +++ b/crates/prune/prune/src/segments/static_file/block_meta.rs @@ -0,0 +1,359 @@ +use crate::{ + db_ext::DbTxPruneExt, + segments::{PruneInput, Segment}, + PruneLimiter, PrunerError, +}; +use alloy_primitives::BlockNumber; +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, RangeWalker}, + tables, + transaction::DbTxMut, +}; +use reth_primitives_traits::NodePrimitives; +use reth_provider::{ + providers::StaticFileProvider, DBProvider, NodePrimitivesProvider, StaticFileProviderFactory, +}; +use reth_prune_types::{ + PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint, +}; +use reth_static_file_types::StaticFileSegment; +use std::num::NonZeroUsize; +use tracing::trace; + +/// Number of tables to prune in one step +const TABLES_TO_PRUNE: usize = 3; + +#[derive(Debug)] +pub struct BlockMeta { + static_file_provider: StaticFileProvider, +} + +impl BlockMeta { + pub const fn new(static_file_provider: StaticFileProvider) -> Self { + Self { static_file_provider } + } +} + +impl> Segment + for BlockMeta +{ + fn segment(&self) -> PruneSegment { + PruneSegment::BlockMeta + } + + fn mode(&self) -> Option { + self.static_file_provider + .get_highest_static_file_block(StaticFileSegment::BlockMeta) + .map(PruneMode::before_inclusive) + } + + fn purpose(&self) -> PrunePurpose { + PrunePurpose::StaticFile + } + + fn prune(&self, provider: &Provider, input: PruneInput) -> Result { + let (block_range_start, block_range_end) = match input.get_next_block_range() { + Some(range) => (*range.start(), *range.end()), + None => { + trace!(target: "pruner", "No headers to prune"); + return Ok(SegmentOutput::done()) + } + }; + + let last_pruned_block = + if block_range_start == 0 { None } else { Some(block_range_start - 1) }; + + let range = last_pruned_block.map_or(0, |block| block + 1)..=block_range_end; + + let mut indices_cursor = provider.tx_ref().cursor_write::()?; + let ommers_cursor = + provider.tx_ref().cursor_write::::BlockHeader>>()?; + let withdrawals_cursor = provider.tx_ref().cursor_write::()?; + + let mut limiter = input.limiter.floor_deleted_entries_limit_to_multiple_of( + NonZeroUsize::new(TABLES_TO_PRUNE).unwrap(), + ); + + let tables_iter = BlockMetaTablesIter::new( + provider, + &mut limiter, + indices_cursor.walk_range(range)?, + ommers_cursor, + withdrawals_cursor, + ); + + let mut last_pruned_block: Option = None; + let mut pruned = 0; + for res in tables_iter { + let BlockMetaTablesIterItem { pruned_block, entries_pruned } = res?; + last_pruned_block = Some(pruned_block); + pruned += entries_pruned; + } + + let done = last_pruned_block == Some(block_range_end); + let progress = limiter.progress(done); + + Ok(SegmentOutput { + progress, + pruned, + checkpoint: Some(SegmentOutputCheckpoint { + block_number: last_pruned_block, + tx_number: None, + }), + }) + } +} +type Walker<'a, Provider, T> = + RangeWalker<'a, T, <::Tx as DbTxMut>::CursorMut>; + +#[allow(missing_debug_implementations)] +struct BlockMetaTablesIter<'a, Provider, C1, C2> +where + Provider: NodePrimitivesProvider + DBProvider, +{ + provider: &'a Provider, + limiter: &'a mut PruneLimiter, + indices_walker: Walker<'a, Provider, tables::BlockBodyIndices>, + ommers_cursor: C1, + withdrawals_cursor: C2, +} + +struct BlockMetaTablesIterItem { + pruned_block: BlockNumber, + entries_pruned: usize, +} + +impl<'a, Provider, C1, C2> BlockMetaTablesIter<'a, Provider, C1, C2> +where + Provider: NodePrimitivesProvider + DBProvider, +{ + fn new( + provider: &'a Provider, + limiter: &'a mut PruneLimiter, + indices_cursor: Walker<'a, Provider, tables::BlockBodyIndices>, + ommers_cursor: C1, + withdrawals_cursor: C2, + ) -> Self { + Self { + provider, + limiter, + indices_walker: indices_cursor, + ommers_cursor, + withdrawals_cursor, + } + } +} + +impl Iterator for BlockMetaTablesIter<'_, Provider, C1, C2> +where + Provider: NodePrimitivesProvider + DBProvider, + C1: DbCursorRW::BlockHeader>> + + DbCursorRO::BlockHeader>>, + C2: DbCursorRW + DbCursorRO, +{ + type Item = Result; + fn next(&mut self) -> Option { + if self.limiter.is_limit_reached() { + return None + } + + let mut entries_pruned = 0; + let mut pruned_block_indices = None; + + if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( + &mut self.indices_walker, + self.limiter, + &mut |_| false, + &mut |(block, _)| { + pruned_block_indices = Some(block); + }, + ) { + return Some(Err(err.into())) + } + + if let Some(block) = &pruned_block_indices.clone() { + entries_pruned += 1; + match self.ommers_cursor.seek_exact(*block) { + Ok(v) if v.is_some() => { + if let Err(err) = self.ommers_cursor.delete_current() { + return Some(Err(err.into())) + } + entries_pruned += 1; + self.limiter.increment_deleted_entries_count(); + } + Err(err) => return Some(Err(err.into())), + Ok(_) => {} + }; + } + + if let Some(block) = &pruned_block_indices.clone() { + match self.withdrawals_cursor.seek_exact(*block) { + Ok(v) if v.is_some() => { + if let Err(err) = self.withdrawals_cursor.delete_current() { + return Some(Err(err.into())) + } + entries_pruned += 1; + self.limiter.increment_deleted_entries_count(); + } + Err(err) => return Some(Err(err.into())), + Ok(_) => {} + }; + } + + pruned_block_indices + .map(move |block| Ok(BlockMetaTablesIterItem { pruned_block: block, entries_pruned })) + } +} + +#[cfg(test)] +mod tests { + use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput}; + use alloy_eips::eip4895::{Withdrawal, Withdrawals}; + use alloy_primitives::{BlockNumber, B256}; + use assert_matches::assert_matches; + use reth_db::{ + models::{StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals}, + tables, + transaction::DbTxMut, + }; + use reth_db_api::transaction::DbTx; + use reth_provider::{ + DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, + StaticFileProviderFactory, + }; + use reth_prune_types::{ + PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, SegmentOutputCheckpoint, + }; + use reth_stages::test_utils::TestStageDB; + use reth_testing_utils::{generators, generators::random_header_range}; + use tracing::trace; + + #[test] + fn prune() { + reth_tracing::init_test_tracing(); + + let db = TestStageDB::default(); + let mut rng = generators::rng(); + + let headers = random_header_range(&mut rng, 0..100, B256::ZERO); + let tx = db.factory.provider_rw().unwrap().into_tx(); + + for header in &headers { + // One tx per block + tx.put::( + header.number, + StoredBlockBodyIndices { + first_tx_num: header.number.saturating_sub(1), + tx_count: 1, + }, + ) + .unwrap(); + + // Not all blocks have ommers, + if header.number % 2 == 0 { + tx.put::( + header.number, + StoredBlockOmmers { ommers: vec![header.clone_header()] }, + ) + .unwrap(); + } + + // Not all blocks have withdrawals + if header.number % 3 == 0 { + tx.put::( + header.number, + StoredBlockWithdrawals { + withdrawals: Withdrawals::new(vec![Withdrawal::default()]), + }, + ) + .unwrap(); + } + } + tx.commit().unwrap(); + + let initial_ommer_entries = headers.len() / 2; + let initial_withdrawal_entries = headers.len() / 3 + 1; + + assert_eq!(db.table::().unwrap().len(), headers.len()); + // We have only inserted every 2 blocks + assert_eq!(db.table::().unwrap().len(), initial_ommer_entries); + // We have only inserted every 3 blocks + assert_eq!( + db.table::().unwrap().len(), + initial_withdrawal_entries + ); + + let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { + let segment = super::BlockMeta::new(db.factory.static_file_provider()); + let input = PruneInput { + previous_checkpoint: db + .factory + .provider() + .unwrap() + .get_prune_checkpoint(PruneSegment::BlockMeta) + .unwrap(), + to_block, + limiter: PruneLimiter::default().set_deleted_entries_limit(5), + }; + + let provider = db.factory.database_provider_rw().unwrap(); + let result = segment.prune(&provider, input).unwrap(); + + trace!(target: "pruner::test", + expected_prune_progress=?expected_result.0, + expected_pruned=?expected_result.1, + result=?result, + "SegmentOutput" + ); + + assert_matches!( + result, + SegmentOutput {progress, pruned, checkpoint: Some(_)} + if (progress, pruned) == expected_result + ); + + provider + .save_prune_checkpoint( + PruneSegment::BlockMeta, + result.checkpoint.unwrap().as_prune_checkpoint(PruneMode::Before(to_block + 1)), + ) + .unwrap(); + provider.commit().expect("commit"); + }; + + test_prune( + 3, + (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 3), + ); + test_prune( + 3, + (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 3), + ); + test_prune(3, (PruneProgress::Finished, 2)); + } + + #[test] + fn prune_cannot_be_done() { + let db = TestStageDB::default(); + + let limiter = PruneLimiter::default().set_deleted_entries_limit(0); + + let input = PruneInput { + previous_checkpoint: None, + to_block: 1, + // Less than total number of tables for `BlockMeta` segment + limiter, + }; + + let provider = db.factory.database_provider_rw().unwrap(); + let segment = super::BlockMeta::new(db.factory.static_file_provider()); + let result = segment.prune(&provider, input).unwrap(); + assert_eq!( + result, + SegmentOutput::not_done( + PruneInterruptReason::DeletedEntriesLimitReached, + Some(SegmentOutputCheckpoint::default()) + ) + ); + } +} diff --git a/crates/prune/prune/src/segments/static_file/mod.rs b/crates/prune/prune/src/segments/static_file/mod.rs index cb9dc79c6cdc..57ff173cd5d0 100644 --- a/crates/prune/prune/src/segments/static_file/mod.rs +++ b/crates/prune/prune/src/segments/static_file/mod.rs @@ -1,7 +1,9 @@ +mod block_meta; mod headers; mod receipts; mod transactions; +pub use block_meta::BlockMeta; pub use headers::Headers; pub use receipts::Receipts; pub use transactions::Transactions; diff --git a/crates/prune/types/src/segment.rs b/crates/prune/types/src/segment.rs index 6d44f88a881e..1b9d92eaa142 100644 --- a/crates/prune/types/src/segment.rs +++ b/crates/prune/types/src/segment.rs @@ -39,15 +39,20 @@ pub enum PruneSegment { Headers, /// Prune segment responsible for the `Transactions` table. Transactions, + /// Prune segment responsible for the `BlockBodyIndices`, `BlockOmmers` and `BlockWithdrawals` + /// table. + BlockMeta, } impl PruneSegment { /// Returns minimum number of blocks to left in the database for this segment. pub const fn min_blocks(&self, purpose: PrunePurpose) -> u64 { match self { - Self::SenderRecovery | Self::TransactionLookup | Self::Headers | Self::Transactions => { - 0 - } + Self::SenderRecovery | + Self::TransactionLookup | + Self::Headers | + Self::BlockMeta | + Self::Transactions => 0, Self::Receipts if purpose.is_static_file() => 0, Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => { MINIMUM_PRUNING_DISTANCE diff --git a/crates/static-file/static-file/src/segments/block_meta.rs b/crates/static-file/static-file/src/segments/block_meta.rs new file mode 100644 index 000000000000..22b195d51817 --- /dev/null +++ b/crates/static-file/static-file/src/segments/block_meta.rs @@ -0,0 +1,61 @@ +use crate::segments::Segment; +use alloy_primitives::BlockNumber; +use reth_codecs::Compact; +use reth_db::{table::Value, tables}; +use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; +use reth_primitives_traits::NodePrimitives; +use reth_provider::{providers::StaticFileWriter, DBProvider, StaticFileProviderFactory}; +use reth_static_file_types::StaticFileSegment; +use reth_storage_errors::provider::ProviderResult; +use std::ops::RangeInclusive; + +/// Static File segment responsible for [`StaticFileSegment::BlockMeta`] part of data. +#[derive(Debug, Default)] +pub struct BlockMeta; + +impl Segment for BlockMeta +where + Provider: StaticFileProviderFactory> + + DBProvider, +{ + fn segment(&self) -> StaticFileSegment { + StaticFileSegment::BlockMeta + } + + fn copy_to_static_files( + &self, + provider: Provider, + block_range: RangeInclusive, + ) -> ProviderResult<()> { + let static_file_provider = provider.static_file_provider(); + let mut static_file_writer = + static_file_provider.get_writer(*block_range.start(), StaticFileSegment::BlockMeta)?; + + let mut indices_cursor = provider.tx_ref().cursor_read::()?; + let indices_walker = indices_cursor.walk_range(block_range)?; + + let mut ommers_cursor = provider + .tx_ref() + .cursor_read::::BlockHeader>>( + )?; + + let mut withdrawals_cursor = provider.tx_ref().cursor_read::()?; + + for entry in indices_walker { + let (block_number, indices) = entry?; + let ommers = + ommers_cursor.seek_exact(block_number)?.map(|(_, l)| l).unwrap_or_default(); + let withdrawals = + withdrawals_cursor.seek_exact(block_number)?.map(|(_, l)| l).unwrap_or_default(); + + static_file_writer.append_eth_block_meta( + &indices, + &ommers, + &withdrawals, + block_number, + )?; + } + + Ok(()) + } +} diff --git a/crates/static-file/static-file/src/segments/mod.rs b/crates/static-file/static-file/src/segments/mod.rs index fc79effdd5ac..215f044ed6fc 100644 --- a/crates/static-file/static-file/src/segments/mod.rs +++ b/crates/static-file/static-file/src/segments/mod.rs @@ -9,6 +9,9 @@ pub use headers::Headers; mod receipts; pub use receipts::Receipts; +mod block_meta; +pub use block_meta::BlockMeta; + use alloy_primitives::BlockNumber; use reth_provider::StaticFileProviderFactory; use reth_static_file_types::StaticFileSegment; diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index 90d3b692b763..a43831ce5256 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -140,6 +140,9 @@ where if let Some(block_range) = targets.receipts.clone() { segments.push((Box::new(segments::Receipts), block_range)); } + if let Some(block_range) = targets.block_meta.clone() { + segments.push((Box::new(segments::BlockMeta), block_range)); + } segments.par_iter().try_for_each(|(segment, block_range)| -> ProviderResult<()> { debug!(target: "static_file", segment = %segment.segment(), ?block_range, "StaticFileProducer segment"); @@ -326,7 +329,7 @@ mod tests { headers: Some(1), receipts: Some(1), transactions: Some(1), - block_meta: None, + block_meta: Some(1), }) .expect("get static file targets"); assert_eq!( @@ -335,7 +338,7 @@ mod tests { headers: Some(0..=1), receipts: Some(0..=1), transactions: Some(0..=1), - block_meta: None + block_meta: Some(0..=1) } ); assert_matches!(static_file_producer.run(targets), Ok(_)); @@ -345,7 +348,7 @@ mod tests { headers: Some(1), receipts: Some(1), transactions: Some(1), - block_meta: None + block_meta: Some(1) } ); @@ -354,7 +357,7 @@ mod tests { headers: Some(3), receipts: Some(3), transactions: Some(3), - block_meta: None, + block_meta: Some(3), }) .expect("get static file targets"); assert_eq!( @@ -363,7 +366,7 @@ mod tests { headers: Some(2..=3), receipts: Some(2..=3), transactions: Some(2..=3), - block_meta: None + block_meta: Some(2..=3) } ); assert_matches!(static_file_producer.run(targets), Ok(_)); @@ -373,7 +376,7 @@ mod tests { headers: Some(3), receipts: Some(3), transactions: Some(3), - block_meta: None + block_meta: Some(3) } ); @@ -382,7 +385,7 @@ mod tests { headers: Some(4), receipts: Some(4), transactions: Some(4), - block_meta: None, + block_meta: Some(4), }) .expect("get static file targets"); assert_eq!( @@ -391,7 +394,7 @@ mod tests { headers: Some(4..=4), receipts: Some(4..=4), transactions: Some(4..=4), - block_meta: None + block_meta: Some(4..=4) } ); assert_matches!( @@ -404,7 +407,7 @@ mod tests { headers: Some(3), receipts: Some(3), transactions: Some(3), - block_meta: None + block_meta: Some(3) } ); } @@ -433,7 +436,7 @@ mod tests { headers: Some(1), receipts: Some(1), transactions: Some(1), - block_meta: None, + block_meta: Some(1), }) .expect("get static file targets"); assert_matches!(locked_producer.run(targets.clone()), Ok(_)); diff --git a/crates/static-file/types/src/lib.rs b/crates/static-file/types/src/lib.rs index bf52b4dee36e..42e1224a0917 100644 --- a/crates/static-file/types/src/lib.rs +++ b/crates/static-file/types/src/lib.rs @@ -30,10 +30,10 @@ pub struct HighestStaticFiles { /// Highest static file block of receipts, inclusive. /// If [`None`], no static file is available. pub receipts: Option, - /// Highest static file block of transactions, inclusive. + /// Highest static file block of block meta, inclusive. /// If [`None`], no static file is available. pub transactions: Option, - /// Highest static file block of transactions, inclusive. + /// Highest static file block of block meta, inclusive. /// If [`None`], no static file is available. pub block_meta: Option, } diff --git a/crates/storage/db/src/static_file/masks.rs b/crates/storage/db/src/static_file/masks.rs index f89a0eac1d4e..dc30b2191b58 100644 --- a/crates/storage/db/src/static_file/masks.rs +++ b/crates/storage/db/src/static_file/masks.rs @@ -1,6 +1,9 @@ use crate::{ add_static_file_mask, - static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo}, + static_file::{ + mask::{ColumnSelectorOne, ColumnSelectorTwo}, + ColumnSelectorThree, + }, BlockBodyIndices, BlockWithdrawals, HeaderTerminalDifficulties, }; use alloy_primitives::BlockHash; @@ -56,3 +59,7 @@ add_static_file_mask! { #[doc = "Mask for a `StoredBlockWithdrawals` from BlockMeta static file segment"] WithdrawalsMask, ::Value, 0b100 } +add_static_file_mask! { + #[doc = "Mask for selecting all columns from BlockMeta static file segment"] + AllBlockMetaMask, ::Value, StoredBlockOmmers::, ::Value, 0b111 +} diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 57ec9c059708..705db199f41d 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -566,7 +566,12 @@ impl BlockBodyIndicesProvider for ProviderFactory { &self, number: BlockNumber, ) -> ProviderResult> { - self.provider()?.block_body_indices(number) + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::BlockMeta, + number, + |static_file| static_file.block_body_indices(number), + || self.provider()?.block_body_indices(number), + ) } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 89b2ae5b6001..04cd7198a783 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1577,14 +1577,21 @@ impl> Withdrawals ) -> ProviderResult> { if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { if let Some(number) = self.convert_hash_or_number(id)? { - // If we are past shanghai, then all blocks should have a withdrawal list, even if - // empty - let withdrawals = self - .tx - .get::(number) - .map(|w| w.map(|w| w.withdrawals))? - .unwrap_or_default(); - return Ok(Some(withdrawals)) + return self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::BlockMeta, + number, + |static_file| static_file.withdrawals_by_block(number.into(), timestamp), + || { + // If we are past shanghai, then all blocks should have a withdrawal list, + // even if empty + let withdrawals = self + .tx + .get::(number) + .map(|w| w.map(|w| w.withdrawals))? + .unwrap_or_default(); + Ok(Some(withdrawals)) + }, + ) } } Ok(None) @@ -1617,7 +1624,12 @@ impl BlockBodyIndicesProvider for DatabaseProvider { fn block_body_indices(&self, num: u64) -> ProviderResult> { - Ok(self.tx.get::(num)?) + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::BlockMeta, + num, + |static_file| static_file.block_body_indices(num), + || Ok(self.tx.get::(num)?), + ) } } diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 40f15c1f52c3..4ae46ba8e70a 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -903,8 +903,9 @@ impl StaticFileProvider { ); let mut writer = self.latest_writer(segment)?; if segment.is_headers() { - // TODO(joshie): is_block_meta writer.prune_headers(highest_static_file_block - checkpoint_block_number)?; + } else if segment.is_block_meta() { + writer.prune_block_meta(highest_static_file_block - checkpoint_block_number)?; } else if let Some(block) = provider.block_body_indices(checkpoint_block_number)? { // todo joshie: is querying block_body_indices a potential issue once bbi is moved // to sf as well @@ -1025,7 +1026,7 @@ impl StaticFileProvider { "Could not find block or tx number on a range request" ); - let err = if segment.is_headers() { + let err = if segment.is_block_based() { ProviderError::MissingStaticFileBlock(segment, number) } else { ProviderError::MissingStaticFileTx(segment, number) diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index c558f43c7e58..42929008fd8a 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -68,7 +68,7 @@ impl StaticFileWriters { } pub(crate) fn commit(&self) -> ProviderResult<()> { - for writer_lock in [&self.headers, &self.transactions, &self.receipts] { + for writer_lock in [&self.headers, &self.block_meta, &self.transactions, &self.receipts] { let mut writer = writer_lock.write(); if let Some(writer) = writer.as_mut() { writer.commit()?; @@ -204,7 +204,7 @@ impl StaticFileProviderRW { /// [`NippyJarWriter`] for more on healing. fn ensure_end_range_consistency(&mut self) -> ProviderResult<()> { // If we have lost rows (in this run or previous), we need to update the [SegmentHeader]. - let expected_rows = if self.user_header().segment().is_headers() { + let expected_rows = if self.user_header().segment().is_block_based() { self.user_header().block_len().unwrap_or_default() } else { self.user_header().tx_len().unwrap_or_default() @@ -234,7 +234,7 @@ impl StaticFileProviderRW { StaticFileSegment::Receipts => { self.prune_receipt_data(to_delete, last_block_number.expect("should exist"))? } - StaticFileSegment::BlockMeta => todo!(), + StaticFileSegment::BlockMeta => self.prune_block_meta(to_delete)?, } } @@ -410,12 +410,12 @@ impl StaticFileProviderRW { let block_start = self.writer.user_header().expected_block_start(); // We only delete the file if it's NOT the first static file AND: - // * it's a Header segment OR + // * it's a block based segment OR // * it's a tx-based segment AND `last_block` is lower than the first block of this // file's block range. Otherwise, having no rows simply means that this block // range has no transactions, but the file should remain. if block_start != 0 && - (segment.is_headers() || last_block.is_some_and(|b| b < block_start)) + (segment.is_block_based() || last_block.is_some_and(|b| b < block_start)) { self.delete_current_and_open_previous()?; } else { @@ -892,5 +892,9 @@ fn create_jar( jar = jar.with_lz4(); } + if segment.is_block_meta() { + jar = jar.with_lz4(); + } + jar }